skip indexation while token is not a word

This commit is contained in:
many 2020-12-17 17:18:11 +01:00
parent 2852349e68
commit aeb676e757
No known key found for this signature in database
GPG Key ID: 2CEF23B75189EACA

View File

@ -138,6 +138,7 @@ where
fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<Item = (usize, Token<'a>)> {
tokens
.skip_while(|token| !token.is_word())
.scan((0, None), |(offset, prev_kind), token| {
match token.kind {
TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {