searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Skip script,language insertion if language is undetected

Browse Source
This commit is contained in:
f3r10 2022-10-19 07:09:05 -05:00 committed by ManyTheFish
parent 2d58b28f43
commit d8207356f4
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -71,12 +71,13 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
.take_while(|(p, _)| (*p as u32) < max_positions_per_attributes); .take_while(|(p, _)| (*p as u32) < max_positions_per_attributes);
for (index, token) in tokens { for (index, token) in tokens {
let script = token.script; if let Some(language) = token.language {
let language = token.language.unwrap_or_default(); let script = token.script;
let entry = script_language_pair let entry = script_language_pair
.entry((script, language)) .entry((script, language))
.or_insert_with(RoaringBitmap::new); .or_insert_with(RoaringBitmap::new);
entry.push(document_id); entry.push(document_id);
}
let token = token.lemma().trim(); let token = token.lemma().trim();
if !token.is_empty() && token.len() <= MAX_WORD_LENGTH { if !token.is_empty() && token.len() <= MAX_WORD_LENGTH {
key_buffer.truncate(mem::size_of::<u32>()); key_buffer.truncate(mem::size_of::<u32>());