set the memory in arroy

crates/milli/src/update/index_documents/mod.rs

@@ -520,7 +520,15 @@ where
 
             pool.install(|| {
                 let mut writer = ArroyWrapper::new(vector_arroy, embedder_index, was_quantized);
-                writer.build_and_quantize(wtxn, &mut rng, dimension, is_quantizing, cancel)?;
+                writer.build_and_quantize(
+                    wtxn,
+                    &mut rng,
+                    dimension,
+                    is_quantizing,
+                    // Arroy should only use 50% of the memory
+                    self.indexer_config.max_memory.map(|mm| mm / 2),
+                    cancel,
+                )?;
                 Result::Ok(())
             })
             .map_err(InternalError::from)??;

crates/milli/src/update/new/indexer/mod.rs

@@ -70,6 +70,8 @@ where
         max_memory: grenad_parameters.max_memory.map(|mm| mm * 5 / 100),
         ..grenad_parameters
     };
+    // Arroy should use 50% of the grenad memory instead of 5%
+    let arroy_memory = grenad_parameters.max_memory.map(|mm| mm * 10);
 
     // 5% percent of the allocated memory for the extractors, or min 100MiB
     // 5% percent of the allocated memory for the bbqueues, or min 50MiB
@@ -200,6 +202,7 @@ where
                 index,
                 wtxn,
                 index_embeddings,
+                arroy_memory,
                 &mut arroy_writers,
                 &indexing_context.must_stop_processing,
             )

crates/milli/src/update/new/indexer/write.rs

@@ -101,6 +101,7 @@ pub fn build_vectors<MSP>(
     index: &Index,
     wtxn: &mut RwTxn<'_>,
     index_embeddings: Vec<IndexEmbeddingConfig>,
+    arroy_memory: Option<usize>,
     arroy_writers: &mut HashMap<u8, (&str, &Embedder, ArroyWrapper, usize)>,
     must_stop_processing: &MSP,
 ) -> Result<()>
@@ -114,7 +115,14 @@ where
     let mut rng = rand::rngs::StdRng::seed_from_u64(42);
     for (_index, (_embedder_name, _embedder, writer, dimensions)) in arroy_writers {
         let dimensions = *dimensions;
-        writer.build_and_quantize(wtxn, &mut rng, dimensions, false, must_stop_processing)?;
+        writer.build_and_quantize(
+            wtxn,
+            &mut rng,
+            dimensions,
+            false,
+            arroy_memory,
+            must_stop_processing,
+        )?;
     }
 
     index.put_embedding_configs(wtxn, index_embeddings)?;