From 9eeb75d501b2ce9fd58ab2ad131059aee3c8dcc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 6 Feb 2024 10:47:04 +0100 Subject: [PATCH] Clamp the max memory of the grenad sorters to a reasonable maximum --- .../index_documents/helpers/grenad_helpers.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/milli/src/update/index_documents/helpers/grenad_helpers.rs b/milli/src/update/index_documents/helpers/grenad_helpers.rs index e1b27baa2..dde03c73d 100644 --- a/milli/src/update/index_documents/helpers/grenad_helpers.rs +++ b/milli/src/update/index_documents/helpers/grenad_helpers.rs @@ -9,6 +9,10 @@ use super::{ClonableMmap, MergeFn}; use crate::update::index_documents::valid_lmdb_key; use crate::Result; +/// This is something reasonable given the fact +/// that there is one grenad sorter by thread. +const MAX_GRENAD_SORTER_USAGE: usize = 200 * 1024 * 1024; // 200 MiB + pub type CursorClonableMmap = io::Cursor; pub fn create_writer( @@ -24,6 +28,9 @@ pub fn create_writer( builder.build(BufWriter::new(file)) } +/// A helper function that creates a grenad sorter +/// with the given parameters. The max memory is +/// clamped to something reasonable. pub fn create_sorter( sort_algorithm: grenad::SortAlgorithm, merge: MergeFn, @@ -41,7 +48,7 @@ pub fn create_sorter( builder.max_nb_chunks(nb_chunks); } if let Some(memory) = max_memory { - builder.dump_threshold(memory); + builder.dump_threshold(memory.min(MAX_GRENAD_SORTER_USAGE)); builder.allow_realloc(false); } builder.sort_algorithm(sort_algorithm); @@ -187,10 +194,15 @@ impl Default for GrenadParameters { impl GrenadParameters { /// This function use the number of threads in the current threadpool to compute the value. + /// /// This should be called inside of a rayon thread pool, - /// Otherwise, it will take the global number of threads. + /// otherwise, it will take the global number of threads. + /// + /// The max memory cannot exceed a given reasonable value. pub fn max_memory_by_thread(&self) -> Option { - self.max_memory.map(|max_memory| max_memory / rayon::current_num_threads()) + self.max_memory.map(|max_memory| { + (max_memory / rayon::current_num_threads()).min(MAX_GRENAD_SORTER_USAGE) + }) } }