searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-23 21:20:24 +01:00
Code Issues Releases Wiki Activity

Change order of arguments to tokenizer_builder

Browse Source
This commit is contained in:
Louis Dureuil 2023-10-30 16:26:29 +01:00
parent 9fedd8101a
commit be395c7944
No known key found for this signature in database
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -56,7 +56,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
let mut value_buffer = Vec::new(); let mut value_buffer = Vec::new();
// initialize tokenizer. // initialize tokenizer.
let mut builder = tokenizer_builder(stop_words, dictionary, allowed_separators, None); let mut builder = tokenizer_builder(stop_words, allowed_separators, dictionary, None);
let tokenizer = builder.build(); let tokenizer = builder.build();
// iterate over documents. // iterate over documents.
@ -247,8 +247,8 @@ fn lang_safe_tokens_from_document<'a>(
// build a new temporary tokenizer including the allow list. // build a new temporary tokenizer including the allow list.
let mut builder = tokenizer_builder( let mut builder = tokenizer_builder(
stop_words, stop_words,
dictionary,
allowed_separators, allowed_separators,
dictionary,
Some(&script_language), Some(&script_language),
); );
let tokenizer = builder.build(); let tokenizer = builder.build();