searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 12:27:13 +02:00
Code Issues Releases Wiki Activity

Create temp threadpool with all CPUs in dump

Browse source
This commit is contained in:
nnethercott 2025-04-27 00:51:26 +02:00
parent 9fd9fcb03e
commit 648b2876f6
6 changed files with 40 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

19
crates/meilisearch/src/lib.rs
View file

@ -38,6 +38,7 @@ use meilisearch_auth::{open_auth_store_env, AuthController};
use meilisearch_types::milli::constants::VERSION_MAJOR;
use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
use meilisearch_types::milli::ThreadPoolNoAbortBuilder;
use meilisearch_types::settings::apply_settings_to_builder;
use meilisearch_types::tasks::KindWithContent;
use meilisearch_types::versioning::{
@ -505,6 +506,18 @@ fn import_dump(
let indexer_config = index_scheduler.indexer_config();
// Use all cpus to index a dump
let pool_before = {
let all_cpus = num_cpus::get();
let temp_pool = ThreadPoolNoAbortBuilder::new()
.thread_name(|index| format!("indexing-thread:{index}"))
.num_threads(all_cpus)
.build()?;
indexer_config.thread_pool.write().unwrap().replace(temp_pool)
};
// /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might
// try to process tasks while we're trying to import the indexes.
@ -576,6 +589,12 @@ fn import_dump(
index_scheduler.refresh_index_stats(&uid)?;
}
// Restore original thread pool after dump
{
let mut guard = indexer_config.thread_pool.write().unwrap();
*guard = pool_before;
}
// 5. Import the queue
let mut index_scheduler_dump = index_scheduler.register_dumped_task()?;
// 5.1. Import the batches