searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 14:54:27 +01:00
Code Issues Releases Wiki Activity

Run the words prefixes update inside of the indexing documents update

Browse Source
This commit is contained in:
Kerollmops 2021-02-10 11:53:13 +01:00 committed by Clément Renault
parent 8788485924
commit a4a48be923
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 19 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
milli/src/update/index_documents/mod.rs
View File

@ -17,7 +17,7 @@ use rayon::prelude::*;
use serde::{Serialize, Deserialize}; use serde::{Serialize, Deserialize};
use crate::index::Index; use crate::index::Index;
use crate::update::{Facets, UpdateIndexingStep}; use crate::update::{Facets, WordsPrefixes, UpdateIndexingStep};
use self::store::{Store, Readers}; use self::store::{Store, Readers};
pub use self::merge_function::{ pub use self::merge_function::{
main_merge, word_docids_merge, words_pairs_proximities_docids_merge, main_merge, word_docids_merge, words_pairs_proximities_docids_merge,
@ -259,6 +259,8 @@ pub struct IndexDocuments<'t, 'u, 'i, 'a> {
pub(crate) thread_pool: Option<&'a ThreadPool>, pub(crate) thread_pool: Option<&'a ThreadPool>,
facet_level_group_size: Option<NonZeroUsize>, facet_level_group_size: Option<NonZeroUsize>,
facet_min_level_size: Option<NonZeroUsize>, facet_min_level_size: Option<NonZeroUsize>,
words_prefix_threshold: Option<f64>,
max_prefix_length: Option<usize>,
update_method: IndexDocumentsMethod, update_method: IndexDocumentsMethod,
update_format: UpdateFormat, update_format: UpdateFormat,
autogenerate_docids: bool, autogenerate_docids: bool,
@ -284,6 +286,8 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
thread_pool: None, thread_pool: None,
facet_level_group_size: None, facet_level_group_size: None,
facet_min_level_size: None, facet_min_level_size: None,
words_prefix_threshold: None,
max_prefix_length: None,
update_method: IndexDocumentsMethod::ReplaceDocuments, update_method: IndexDocumentsMethod::ReplaceDocuments,
update_format: UpdateFormat::Json, update_format: UpdateFormat::Json,
autogenerate_docids: true, autogenerate_docids: true,
@ -667,6 +671,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
}); });
} }
// Run the facets update operation.
let mut builder = Facets::new(self.wtxn, self.index, self.update_id); let mut builder = Facets::new(self.wtxn, self.index, self.update_id);
builder.chunk_compression_type = self.chunk_compression_type; builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level; builder.chunk_compression_level = self.chunk_compression_level;
@ -679,6 +684,19 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
} }
builder.execute()?; builder.execute()?;
// Run the words prefixes update operation.
let mut builder = WordsPrefixes::new(self.wtxn, self.index, self.update_id);
builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level;
builder.chunk_fusing_shrink_size = self.chunk_fusing_shrink_size;
if let Some(value) = self.words_prefix_threshold {
builder.threshold(value);
}
if let Some(value) = self.max_prefix_length {
builder.max_prefix_length(value);
}
builder.execute()?;
debug_assert_eq!(database_count, total_databases); debug_assert_eq!(database_count, total_databases);
info!("Transform output indexed in {:.02?}", before_indexing.elapsed()); info!("Transform output indexed in {:.02?}", before_indexing.elapsed());