searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-06-13 08:09:50 +02:00
Code Issues Releases Wiki Activity

Ignore tokens with empty normalized value during indexing process

Browse Source
This commit is contained in:
ManyTheFish 2023-09-05 15:44:14 +02:00
parent 8ac5b765bc
commit 66aa6d5871
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -226,9 +226,9 @@ fn process_tokens<'a>(
) -> impl Iterator<Item = (usize, Token<'a>)> {
tokens
.skip_while(|token| token.is_separator())
.scan((0, None), |(offset, prev_kind), token| {
.scan((0, None), |(offset, prev_kind), mut token| {
match token.kind {
TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {
TokenKind::Word | TokenKind::StopWord if !token.lemma().is_empty() => {
*offset += match *prev_kind {
Some(TokenKind::Separator(SeparatorKind::Hard)) => 8,
Some(_) => 1,
@ -244,7 +244,7 @@ fn process_tokens<'a>(
{
*prev_kind = Some(token.kind);
}
_ => (),
_ => token.kind = TokenKind::Unknown,
}
Some((*offset, token))
})