searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-23 13:10:06 +01:00
Code Issues Releases Wiki Activity

Change order of arguments to tokenizer_builder

Browse Source
This commit is contained in:
Louis Dureuil 2023-10-30 16:26:29 +01:00
parent 9fedd8101a
commit be395c7944
No known key found for this signature in database
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -56,7 +56,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
let mut value_buffer = Vec::new();
// initialize tokenizer.
let mut builder = tokenizer_builder(stop_words, dictionary, allowed_separators, None);
let mut builder = tokenizer_builder(stop_words, allowed_separators, dictionary, None);
let tokenizer = builder.build();
// iterate over documents.
@ -247,8 +247,8 @@ fn lang_safe_tokens_from_document<'a>(
// build a new temporary tokenizer including the allow list.
let mut builder = tokenizer_builder(
stop_words,
dictionary,
allowed_separators,
dictionary,
Some(&script_language),
);
let tokenizer = builder.build();