From f6b3d1f9a5acf74bc88b32c1fb2bf9970cdd85d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Thu, 5 Sep 2024 15:12:07 +0200
Subject: [PATCH] Increase some channel sizes

---
 milli/src/update/new/extract/searchable/mod.rs |  2 +-
 milli/src/update/new/indexer/mod.rs            | 18 ++++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/milli/src/update/new/extract/searchable/mod.rs b/milli/src/update/new/extract/searchable/mod.rs
index ba4731d73..a7498d0d9 100644
--- a/milli/src/update/new/extract/searchable/mod.rs
+++ b/milli/src/update/new/extract/searchable/mod.rs
@@ -66,7 +66,7 @@ pub trait SearchableExtractor {
                 fields_ids_map.clone(),
                 CboCachedSorter::new(
                     // TODO use a better value
-                    100.try_into().unwrap(),
+                    1_000_000.try_into().unwrap(),
                     create_sorter(
                         grenad::SortAlgorithm::Stable,
                         MergeDeladdCboRoaringBitmaps,
diff --git a/milli/src/update/new/indexer/mod.rs b/milli/src/update/new/indexer/mod.rs
index d721a5511..0865374c8 100644
--- a/milli/src/update/new/indexer/mod.rs
+++ b/milli/src/update/new/indexer/mod.rs
@@ -54,7 +54,7 @@ where
     PI: IntoParallelIterator<Item = Result<DocumentChange>> + Send,
     PI::Iter: Clone,
 {
-    let (merger_sender, writer_receiver) = merger_writer_channel(100);
+    let (merger_sender, writer_receiver) = merger_writer_channel(10_000);
     // This channel acts as a rendezvous point to ensure that we are one task ahead
     let (extractor_sender, merger_receiver) = extractors_merger_channels(0);
 
@@ -89,10 +89,16 @@ where
                         Ok(()) as Result<_>
                     })?;
 
+                    const TEN_GIB: usize = 10 * 1024 * 1024 * 1024;
+                    let max_memory = TEN_GIB / dbg!(rayon::current_num_threads());
+                    let grenad_parameters = GrenadParameters {
+                        max_memory: Some(max_memory),
+                        ..GrenadParameters::default()
+                    };
                     extract_and_send_docids::<WordDocidsExtractor, WordDocids>(
                         index,
                         &global_fields_ids_map,
-                        GrenadParameters::default(),
+                        grenad_parameters,
                         document_changes.clone(),
                         &extractor_sender,
                     )?;
@@ -100,7 +106,7 @@ where
                     extract_and_send_docids::<WordFidDocidsExtractor, WordFidDocids>(
                         index,
                         &global_fields_ids_map,
-                        GrenadParameters::default(),
+                        grenad_parameters,
                         document_changes.clone(),
                         &extractor_sender,
                     )?;
@@ -108,7 +114,7 @@ where
                     extract_and_send_docids::<ExactWordDocidsExtractor, ExactWordDocids>(
                         index,
                         &global_fields_ids_map,
-                        GrenadParameters::default(),
+                        grenad_parameters,
                         document_changes.clone(),
                         &extractor_sender,
                     )?;
@@ -116,7 +122,7 @@ where
                     extract_and_send_docids::<WordPositionDocidsExtractor, WordPositionDocids>(
                         index,
                         &global_fields_ids_map,
-                        GrenadParameters::default(),
+                        grenad_parameters,
                         document_changes.clone(),
                         &extractor_sender,
                     )?;
@@ -135,7 +141,7 @@ where
                     >(
                         index,
                         &global_fields_ids_map,
-                        GrenadParameters::default(),
+                        grenad_parameters,
                         document_changes.clone(),
                         &extractor_sender,
                     )?;