diff --git a/milli/src/update/new/extract/searchable/mod.rs b/milli/src/update/new/extract/searchable/mod.rs index ba4731d73..a7498d0d9 100644 --- a/milli/src/update/new/extract/searchable/mod.rs +++ b/milli/src/update/new/extract/searchable/mod.rs @@ -66,7 +66,7 @@ pub trait SearchableExtractor { fields_ids_map.clone(), CboCachedSorter::new( // TODO use a better value - 100.try_into().unwrap(), + 1_000_000.try_into().unwrap(), create_sorter( grenad::SortAlgorithm::Stable, MergeDeladdCboRoaringBitmaps, diff --git a/milli/src/update/new/indexer/mod.rs b/milli/src/update/new/indexer/mod.rs index d721a5511..0865374c8 100644 --- a/milli/src/update/new/indexer/mod.rs +++ b/milli/src/update/new/indexer/mod.rs @@ -54,7 +54,7 @@ where PI: IntoParallelIterator> + Send, PI::Iter: Clone, { - let (merger_sender, writer_receiver) = merger_writer_channel(100); + let (merger_sender, writer_receiver) = merger_writer_channel(10_000); // This channel acts as a rendezvous point to ensure that we are one task ahead let (extractor_sender, merger_receiver) = extractors_merger_channels(0); @@ -89,10 +89,16 @@ where Ok(()) as Result<_> })?; + const TEN_GIB: usize = 10 * 1024 * 1024 * 1024; + let max_memory = TEN_GIB / dbg!(rayon::current_num_threads()); + let grenad_parameters = GrenadParameters { + max_memory: Some(max_memory), + ..GrenadParameters::default() + }; extract_and_send_docids::( index, &global_fields_ids_map, - GrenadParameters::default(), + grenad_parameters, document_changes.clone(), &extractor_sender, )?; @@ -100,7 +106,7 @@ where extract_and_send_docids::( index, &global_fields_ids_map, - GrenadParameters::default(), + grenad_parameters, document_changes.clone(), &extractor_sender, )?; @@ -108,7 +114,7 @@ where extract_and_send_docids::( index, &global_fields_ids_map, - GrenadParameters::default(), + grenad_parameters, document_changes.clone(), &extractor_sender, )?; @@ -116,7 +122,7 @@ where extract_and_send_docids::( index, &global_fields_ids_map, - GrenadParameters::default(), + grenad_parameters, document_changes.clone(), &extractor_sender, )?; @@ -135,7 +141,7 @@ where >( index, &global_fields_ids_map, - GrenadParameters::default(), + grenad_parameters, document_changes.clone(), &extractor_sender, )?;