mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-05 20:48:58 +01:00
skip indexation while token is not a word
This commit is contained in:
parent
2852349e68
commit
aeb676e757
@ -138,6 +138,7 @@ where
|
||||
|
||||
fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<Item = (usize, Token<'a>)> {
|
||||
tokens
|
||||
.skip_while(|token| !token.is_word())
|
||||
.scan((0, None), |(offset, prev_kind), token| {
|
||||
match token.kind {
|
||||
TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {
|
||||
|
Loading…
Reference in New Issue
Block a user