refactor indexer mod

crates/milli/src/update/new/extract/faceted/extract_facets.rs

@@ -28,7 +28,7 @@ use crate::{DocumentId, FieldId, Index, Result, MAX_FACET_VALUE_LENGTH};
 pub struct FacetedExtractorData<'a, 'b> {
     attributes_to_extract: &'a [&'a str],
     sender: &'a FieldIdDocidFacetSender<'a, 'b>,
-    grenad_parameters: GrenadParameters,
+    grenad_parameters: &'a GrenadParameters,
     buckets: usize,
 }
 
@@ -374,7 +374,6 @@ fn truncate_str(s: &str) -> &str {
 impl FacetedDocidsExtractor {
     #[tracing::instrument(level = "trace", skip_all, target = "indexing::extract::faceted")]
     pub fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
-        grenad_parameters: GrenadParameters,
         document_changes: &DC,
         indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
         extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
@@ -398,7 +397,7 @@ impl FacetedDocidsExtractor {
 
             let extractor = FacetedExtractorData {
                 attributes_to_extract: &attributes_to_extract,
-                grenad_parameters,
+                grenad_parameters: indexing_context.grenad_parameters,
                 buckets: rayon::current_num_threads(),
                 sender,
             };

crates/milli/src/update/new/extract/mod.rs

@@ -18,12 +18,10 @@ pub use vectors::EmbeddingExtractor;
 use super::indexer::document_changes::{DocumentChanges, IndexingContext};
 use super::steps::IndexingStep;
 use super::thread_local::{FullySend, ThreadLocal};
-use crate::update::GrenadParameters;
 use crate::Result;
 
 pub trait DocidsExtractor {
     fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
-        grenad_parameters: GrenadParameters,
         document_changes: &DC,
         indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
         extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,

crates/milli/src/update/new/extract/searchable/extract_word_docids.rs

@@ -208,7 +208,7 @@ impl<'extractor> WordDocidsCaches<'extractor> {
 
 pub struct WordDocidsExtractorData<'a> {
     tokenizer: &'a DocumentTokenizer<'a>,
-    grenad_parameters: GrenadParameters,
+    grenad_parameters: &'a GrenadParameters,
     buckets: usize,
 }
 
@@ -240,7 +240,6 @@ pub struct WordDocidsExtractors;
 
 impl WordDocidsExtractors {
     pub fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
-        grenad_parameters: GrenadParameters,
         document_changes: &DC,
         indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
         extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
@@ -288,7 +287,7 @@ impl WordDocidsExtractors {
 
             let extractor = WordDocidsExtractorData {
                 tokenizer: &document_tokenizer,
-                grenad_parameters,
+                grenad_parameters: indexing_context.grenad_parameters,
                 buckets: rayon::current_num_threads(),
             };
 

crates/milli/src/update/new/extract/searchable/mod.rs

@@ -24,7 +24,7 @@ use crate::{Index, Result, MAX_POSITION_PER_ATTRIBUTE};
 
 pub struct SearchableExtractorData<'a, EX: SearchableExtractor> {
     tokenizer: &'a DocumentTokenizer<'a>,
-    grenad_parameters: GrenadParameters,
+    grenad_parameters: &'a GrenadParameters,
     buckets: usize,
     _ex: PhantomData<EX>,
 }
@@ -57,7 +57,6 @@ impl<'a, 'extractor, EX: SearchableExtractor + Sync> Extractor<'extractor>
 
 pub trait SearchableExtractor: Sized + Sync {
     fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
-        grenad_parameters: GrenadParameters,
         document_changes: &DC,
         indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
         extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
@@ -96,7 +95,7 @@ pub trait SearchableExtractor: Sized + Sync {
 
         let extractor_data: SearchableExtractorData<Self> = SearchableExtractorData {
             tokenizer: &document_tokenizer,
-            grenad_parameters,
+            grenad_parameters: indexing_context.grenad_parameters,
             buckets: rayon::current_num_threads(),
             _ex: PhantomData,
         };
@@ -134,7 +133,6 @@ pub trait SearchableExtractor: Sized + Sync {
 
 impl<T: SearchableExtractor> DocidsExtractor for T {
     fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
-        grenad_parameters: GrenadParameters,
         document_changes: &DC,
         indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
         extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
@@ -143,12 +141,6 @@ impl<T: SearchableExtractor> DocidsExtractor for T {
     where
         MSP: Fn() -> bool + Sync,
     {
-        Self::run_extraction(
-            grenad_parameters,
-            document_changes,
-            indexing_context,
-            extractor_allocs,
-            step,
-        )
+        Self::run_extraction(document_changes, indexing_context, extractor_allocs, step)
     }
 }