searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Make max_position_per_attributes changable

Browse source
This commit is contained in:
many 2021-10-06 12:11:07 +02:00
parent 360c5ff3df
commit c5a6075484
No known key found for this signature in database
GPG key ID: 2CEF23B75189EACA
6 changed files with 31 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View file

@ -23,7 +23,10 @@ pub fn extract_docid_word_positions<R: io::Read>(
indexer: GrenadParameters,
searchable_fields: &Option<HashSet<FieldId>>,
stop_words: Option<&fst::Set<&[u8]>>,
max_positions_per_attributes: Option<u32>,
) -> Result<(RoaringBitmap, grenad::Reader<File>)> {
let max_positions_per_attributes = max_positions_per_attributes
.map_or(MAX_POSITION_PER_ATTRIBUTE, |max| max.min(MAX_POSITION_PER_ATTRIBUTE));
let max_memory = indexer.max_memory_by_thread();
let mut documents_ids = RoaringBitmap::new();
@ -62,7 +65,7 @@ pub fn extract_docid_word_positions<R: io::Read>(
if let Some(field) = json_to_string(&value, &mut field_buffer) {
let analyzed = analyzer.analyze(field);
let tokens = process_tokens(analyzed.tokens())
.take_while(|(p, _)| (*p as u32) < MAX_POSITION_PER_ATTRIBUTE);
.take_while(|(p, _)| (*p as u32) < max_positions_per_attributes);
for (index, token) in tokens {
let token = token.text().trim();