skip indexation while token is not a word

2020-12-17 17:18:11 +01:00 · 2020-12-17 17:18:11 +01:00 · aeb676e757
parent 2852349e68
commit aeb676e757
1 changed files with 1 additions and 0 deletions
--- a/meilisearch-core/src/raw_indexer.rs
+++ b/meilisearch-core/src/raw_indexer.rs
@ -138,6 +138,7 @@ where

 fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<Item = (usize, Token<'a>)> {
    tokens
+        .skip_while(|token| !token.is_word())
        .scan((0, None), |(offset, prev_kind), token| {
                match token.kind {
                    TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {