diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 3b3c94230..7310260f6 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -504,18 +504,22 @@ fn import_dump( let network = dump_reader.network()?.cloned().unwrap_or_default(); index_scheduler.put_network(network)?; - let mut indexer_config = index_scheduler.indexer_config().clone_no_threadpool(); - - // 3.1 Use all cpus to index the import dump - indexer_config.thread_pool = { - let all_cpus = num_cpus::get(); - - let temp_pool = ThreadPoolNoAbortBuilder::new() - .thread_name(|index| format!("indexing-thread:{index}")) - .num_threads(all_cpus) - .build()?; - - Some(temp_pool) + // 3.1 Use all cpus to process dump if max_indexing_threads not configured + let backup_config; + let indexer_config = if index_scheduler.indexer_config().max_threads.is_none() { + let mut _config = index_scheduler.indexer_config().clone_no_threadpool(); + _config.thread_pool = { + Some( + ThreadPoolNoAbortBuilder::new() + .thread_name(|index| format!("indexing-thread:{index}")) + .num_threads(num_cpus::get()) + .build()?, + ) + }; + backup_config = _config; + &backup_config + } else { + index_scheduler.indexer_config() }; // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might @@ -533,7 +537,7 @@ fn import_dump( let mut wtxn = index.write_txn()?; - let mut builder = milli::update::Settings::new(&mut wtxn, &index, &indexer_config); + let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config); // 4.1 Import the primary key if there is one. if let Some(ref primary_key) = metadata.primary_key { builder.set_primary_key(primary_key.to_string()); @@ -568,7 +572,7 @@ fn import_dump( let builder = milli::update::IndexDocuments::new( &mut wtxn, &index, - &indexer_config, + indexer_config, IndexDocumentsConfig { update_method: IndexDocumentsMethod::ReplaceDocuments, ..Default::default() diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index c71bf16c0..259fd501f 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -746,10 +746,12 @@ impl IndexerOpts { max_indexing_memory.to_string(), ); } - export_to_env_if_not_present( - MEILI_MAX_INDEXING_THREADS, - max_indexing_threads.0.to_string(), - ); + if let Some(max_indexing_threads) = max_indexing_threads.0 { + export_to_env_if_not_present( + MEILI_MAX_INDEXING_THREADS, + max_indexing_threads.to_string(), + ); + } } } @@ -757,14 +759,18 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { type Error = anyhow::Error; fn try_from(other: &IndexerOpts) -> Result { + // use 1/2 cpu threads if no value specified + let max_indexing_threads = other.max_indexing_threads.unwrap_or(num_cpus::get() / 2); + let thread_pool = ThreadPoolNoAbortBuilder::new() .thread_name(|index| format!("indexing-thread:{index}")) - .num_threads(*other.max_indexing_threads) + .num_threads(max_indexing_threads) .build()?; Ok(Self { log_every_n: Some(DEFAULT_LOG_EVERY_N), max_memory: other.max_indexing_memory.map(|b| b.as_u64() as usize), + max_threads: *other.max_indexing_threads, thread_pool: Some(thread_pool), max_positions_per_attributes: None, skip_index_budget: other.skip_index_budget, @@ -828,31 +834,31 @@ fn total_memory_bytes() -> Option { } } -#[derive(Debug, Clone, Copy, Deserialize, Serialize)] -pub struct MaxThreads(usize); +#[derive(Default, Debug, Clone, Copy, Deserialize, Serialize)] +pub struct MaxThreads(Option); impl FromStr for MaxThreads { type Err = ParseIntError; - fn from_str(s: &str) -> Result { - usize::from_str(s).map(Self) - } -} - -impl Default for MaxThreads { - fn default() -> Self { - MaxThreads(num_cpus::get() / 2) + fn from_str(s: &str) -> Result { + if s.is_empty() { + return Ok(MaxThreads::default()); + } + usize::from_str(s).map(Some).map(MaxThreads) } } impl fmt::Display for MaxThreads { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) + match self.0 { + Some(threads) => write!(f, "{}", threads), + None => Ok(()), + } } } impl Deref for MaxThreads { - type Target = usize; + type Target = Option; fn deref(&self) -> &Self::Target { &self.0 diff --git a/crates/milli/src/update/indexer_config.rs b/crates/milli/src/update/indexer_config.rs index f9503c48e..e19649a0d 100644 --- a/crates/milli/src/update/indexer_config.rs +++ b/crates/milli/src/update/indexer_config.rs @@ -9,6 +9,7 @@ pub struct IndexerConfig { pub max_nb_chunks: Option, pub documents_chunk_size: Option, pub max_memory: Option, + pub max_threads: Option, pub chunk_compression_type: CompressionType, pub chunk_compression_level: Option, pub thread_pool: Option, @@ -32,6 +33,7 @@ impl IndexerConfig { max_nb_chunks: self.max_nb_chunks, documents_chunk_size: self.documents_chunk_size, max_memory: self.max_memory, + max_threads: self.max_threads, chunk_compression_type: self.chunk_compression_type, chunk_compression_level: self.chunk_compression_level, max_positions_per_attributes: self.max_positions_per_attributes, @@ -48,6 +50,7 @@ impl Default for IndexerConfig { max_nb_chunks: None, documents_chunk_size: None, max_memory: None, + max_threads: None, chunk_compression_type: CompressionType::None, chunk_compression_level: None, thread_pool: None,