searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 06:44:27 +01:00
Code Issues Releases Wiki Activity

Ignore tokens with empty normalized value during indexing process

Browse Source
This commit is contained in:
ManyTheFish 2023-09-05 15:44:14 +02:00
parent 8ac5b765bc
commit 66aa6d5871
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -226,9 +226,9 @@ fn process_tokens<'a>(
) -> impl Iterator<Item = (usize, Token<'a>)> { ) -> impl Iterator<Item = (usize, Token<'a>)> {
tokens tokens
.skip_while(|token| token.is_separator()) .skip_while(|token| token.is_separator())
.scan((0, None), |(offset, prev_kind), token| { .scan((0, None), |(offset, prev_kind), mut token| {
match token.kind { match token.kind {
TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => { TokenKind::Word | TokenKind::StopWord if !token.lemma().is_empty() => {
*offset += match *prev_kind { *offset += match *prev_kind {
Some(TokenKind::Separator(SeparatorKind::Hard)) => 8, Some(TokenKind::Separator(SeparatorKind::Hard)) => 8,
Some(_) => 1, Some(_) => 1,
@ -244,7 +244,7 @@ fn process_tokens<'a>(
{ {
*prev_kind = Some(token.kind); *prev_kind = Some(token.kind);
} }
_ => (), _ => token.kind = TokenKind::Unknown,
} }
Some((*offset, token)) Some((*offset, token))
}) })