searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-02-05 10:03:27 +01:00
Code Issues Releases Wiki Activity

Give more RAM to bbqueue.

Browse Source 
- bbqueue buffers used to have (5% * 2%) / num_threads
- they now have 5% / num_threads
This commit is contained in:
Louis Dureuil 2025-01-23 10:55:03 +01:00
parent 34dea863e5
commit 36b6e94b29
No known key found for this signature in database
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
crates/milli/src/update/new/indexer/mod.rs
View File

@ -93,17 +93,25 @@ where
..grenad_parameters ..grenad_parameters
}; };
// We compute and remove the allocated BBQueues buffers capacity from the indexing memory. // 5% percent of the allocated memory for the extractors, or min 100MiB
let minimum_capacity = 50 * 1024 * 1024 * pool.current_num_threads(); // 50 MiB // 5% percent of the allocated memory for the bbqueues, or min 50MiB
//
// Minimum capacity for bbqueues
let minimum_total_bbbuffer_capacity = 50 * 1024 * 1024 * pool.current_num_threads(); // 50 MiB
let minimum_total_extractors_capacity = minimum_total_bbbuffer_capacity * 2;
let (grenad_parameters, total_bbbuffer_capacity) = grenad_parameters.max_memory.map_or( let (grenad_parameters, total_bbbuffer_capacity) = grenad_parameters.max_memory.map_or(
(grenad_parameters, 2 * minimum_capacity), // 100 MiB by thread by default (
GrenadParameters {
max_memory: Some(minimum_total_extractors_capacity),
..grenad_parameters
},
minimum_total_bbbuffer_capacity,
), // 100 MiB by thread by default
|max_memory| { |max_memory| {
// 2% of the indexing memory let total_bbbuffer_capacity = max_memory.max(minimum_total_bbbuffer_capacity);
let total_bbbuffer_capacity = (max_memory / 100 / 2).max(minimum_capacity);
let new_grenad_parameters = GrenadParameters { let new_grenad_parameters = GrenadParameters {
max_memory: Some( max_memory: Some(max_memory.max(minimum_total_extractors_capacity)),
max_memory.saturating_sub(total_bbbuffer_capacity).max(100 * 1024 * 1024),
),
..grenad_parameters ..grenad_parameters
}; };
(new_grenad_parameters, total_bbbuffer_capacity) (new_grenad_parameters, total_bbbuffer_capacity)