mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-06-15 20:42:24 +02:00
refactor: helper methods for pool and max threads
This commit is contained in:
parent
806e983aa5
commit
865f24cfef
@ -506,10 +506,8 @@ fn import_dump(
|
|||||||
let base_config = index_scheduler.indexer_config();
|
let base_config = index_scheduler.indexer_config();
|
||||||
|
|
||||||
let indexer_config = if base_config.max_threads.is_none() {
|
let indexer_config = if base_config.max_threads.is_none() {
|
||||||
let thread_pool = ThreadPoolNoAbortBuilder::new()
|
let thread_pool =
|
||||||
.thread_name(|index| format!("indexing-thread:{index}"))
|
ThreadPoolNoAbortBuilder::new_for_indexing().num_threads(num_cpus::get()).build()?;
|
||||||
.num_threads(num_cpus::get())
|
|
||||||
.build()?;
|
|
||||||
|
|
||||||
let _config = IndexerConfig { thread_pool, ..*base_config };
|
let _config = IndexerConfig { thread_pool, ..*base_config };
|
||||||
backup_config = _config;
|
backup_config = _config;
|
||||||
|
@ -759,12 +759,8 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
|
|||||||
type Error = anyhow::Error;
|
type Error = anyhow::Error;
|
||||||
|
|
||||||
fn try_from(other: &IndexerOpts) -> Result<Self, Self::Error> {
|
fn try_from(other: &IndexerOpts) -> Result<Self, Self::Error> {
|
||||||
// use 1/2 cpu threads if no value specified
|
let thread_pool = ThreadPoolNoAbortBuilder::new_for_indexing()
|
||||||
let max_indexing_threads = other.max_indexing_threads.unwrap_or_else(|| num_cpus::get() / 2);
|
.num_threads(other.max_indexing_threads.unwrap_or_else(|| num_cpus::get() / 2))
|
||||||
|
|
||||||
let thread_pool = ThreadPoolNoAbortBuilder::new()
|
|
||||||
.thread_name(|index| format!("indexing-thread:{index}"))
|
|
||||||
.num_threads(max_indexing_threads)
|
|
||||||
.build()?;
|
.build()?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@ -841,7 +837,7 @@ impl FromStr for MaxThreads {
|
|||||||
type Err = ParseIntError;
|
type Err = ParseIntError;
|
||||||
|
|
||||||
fn from_str(s: &str) -> Result<MaxThreads, Self::Err> {
|
fn from_str(s: &str) -> Result<MaxThreads, Self::Err> {
|
||||||
if s.is_empty() {
|
if s.is_empty() || s == "unlimited" {
|
||||||
return Ok(MaxThreads::default());
|
return Ok(MaxThreads::default());
|
||||||
}
|
}
|
||||||
usize::from_str(s).map(Some).map(MaxThreads)
|
usize::from_str(s).map(Some).map(MaxThreads)
|
||||||
|
@ -54,6 +54,10 @@ impl ThreadPoolNoAbortBuilder {
|
|||||||
ThreadPoolNoAbortBuilder::default()
|
ThreadPoolNoAbortBuilder::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn new_for_indexing() -> ThreadPoolNoAbortBuilder {
|
||||||
|
ThreadPoolNoAbortBuilder::default().thread_name(|index| format!("indexing-thread:{index}"))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn thread_name<F>(mut self, closure: F) -> Self
|
pub fn thread_name<F>(mut self, closure: F) -> Self
|
||||||
where
|
where
|
||||||
F: FnMut(usize) -> String + 'static,
|
F: FnMut(usize) -> String + 'static,
|
||||||
|
@ -28,24 +28,29 @@ impl IndexerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// By default use only 1 thread for indexing in tests
|
||||||
|
#[cfg(test)]
|
||||||
|
fn default_thread_pool_and_threads() -> (ThreadPoolNoAbort, Option<usize>) {
|
||||||
|
let pool = ThreadPoolNoAbortBuilder::new_for_indexing()
|
||||||
|
.num_threads(1)
|
||||||
|
.build()
|
||||||
|
.expect("failed to build default rayon thread pool");
|
||||||
|
|
||||||
|
(pool, Some(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
fn default_thread_pool_and_threads() -> (ThreadPoolNoAbort, Option<usize>) {
|
||||||
|
let pool = ThreadPoolNoAbortBuilder::new_for_indexing()
|
||||||
|
.build()
|
||||||
|
.expect("failed to build default rayon thread pool");
|
||||||
|
|
||||||
|
(pool, None)
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for IndexerConfig {
|
impl Default for IndexerConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
#[allow(unused_mut)]
|
let (thread_pool, max_threads) = default_thread_pool_and_threads();
|
||||||
let mut pool_builder = ThreadPoolNoAbortBuilder::new();
|
|
||||||
|
|
||||||
#[allow(unused_mut, unused_assignments)]
|
|
||||||
let mut max_threads = None;
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
{
|
|
||||||
pool_builder = pool_builder.num_threads(1);
|
|
||||||
max_threads = Some(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
let thread_pool = pool_builder
|
|
||||||
.thread_name(|index| format!("indexing-thread:{index}"))
|
|
||||||
.build()
|
|
||||||
.expect("failed to build default rayon thread pool");
|
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
max_threads,
|
max_threads,
|
||||||
|
Loading…
x
Reference in New Issue
Block a user