diff --git a/crates/milli/src/update/new/extract/documents.rs b/crates/milli/src/update/new/extract/documents.rs index 37d867b31..f5dcb3eff 100644 --- a/crates/milli/src/update/new/extract/documents.rs +++ b/crates/milli/src/update/new/extract/documents.rs @@ -9,9 +9,7 @@ use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::update::new::channel::{DocumentsSender, ExtractorBbqueueSender}; use crate::update::new::document::{write_to_obkv, Document as _}; use crate::update::new::document_change::DatabaseDocument; -use crate::update::new::indexer::document_changes::{ - DocumentChangeContext, Extractor, IndexingContext, -}; +use crate::update::new::indexer::document_changes::{DocumentContext, Extractor, IndexingContext}; use crate::update::new::indexer::settings_changes::{ settings_change_extract, DatabaseDocuments, SettingsChangeExtractor, }; @@ -50,7 +48,7 @@ impl<'extractor> Extractor<'extractor> for DocumentsExtractor<'_, '_> { fn process<'doc>( &self, changes: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> Result<()> { let mut document_buffer = bumpalo::collections::Vec::new_in(&context.doc_alloc); let mut document_extractor_data = context.data.0.borrow_mut_or_yield(); @@ -196,7 +194,7 @@ impl<'extractor> SettingsChangeExtractor<'extractor> for SettingsChangeDocumentE fn process<'doc>( &self, documents: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> Result<()> { let mut document_buffer = bumpalo::collections::Vec::new_in(&context.doc_alloc); diff --git a/crates/milli/src/update/new/extract/faceted/extract_facets.rs b/crates/milli/src/update/new/extract/faceted/extract_facets.rs index 517ef3f2d..39e68fb6d 100644 --- a/crates/milli/src/update/new/extract/faceted/extract_facets.rs +++ b/crates/milli/src/update/new/extract/faceted/extract_facets.rs @@ -17,7 +17,7 @@ use crate::update::del_add::DelAdd; use crate::update::new::channel::FieldIdDocidFacetSender; use crate::update::new::extract::perm_json_p; use crate::update::new::indexer::document_changes::{ - extract, DocumentChangeContext, DocumentChanges, Extractor, IndexingContext, + extract, DocumentContext, DocumentChanges, Extractor, IndexingContext, }; use crate::update::new::ref_cell_ext::RefCellExt as _; use crate::update::new::steps::IndexingStep; @@ -51,7 +51,7 @@ impl<'extractor> Extractor<'extractor> for FacetedExtractorData<'_, '_> { fn process<'doc>( &self, changes: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> Result<()> { for change in changes { let change = change?; @@ -75,7 +75,7 @@ pub struct FacetedDocidsExtractor; impl FacetedDocidsExtractor { #[allow(clippy::too_many_arguments)] fn extract_document_change( - context: &DocumentChangeContext>, + context: &DocumentContext>, filterable_attributes: &[FilterableAttributesRule], sortable_fields: &HashSet, asc_desc_fields: &HashSet, diff --git a/crates/milli/src/update/new/extract/geo/mod.rs b/crates/milli/src/update/new/extract/geo/mod.rs index b2ccc1b2b..927434ff6 100644 --- a/crates/milli/src/update/new/extract/geo/mod.rs +++ b/crates/milli/src/update/new/extract/geo/mod.rs @@ -11,7 +11,7 @@ use serde_json::Value; use crate::error::GeoError; use crate::update::new::document::Document; -use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor}; +use crate::update::new::indexer::document_changes::{DocumentContext, Extractor}; use crate::update::new::ref_cell_ext::RefCellExt as _; use crate::update::new::thread_local::MostlySend; use crate::update::new::DocumentChange; @@ -150,7 +150,7 @@ impl<'extractor> Extractor<'extractor> for GeoExtractor { fn process<'doc>( &'doc self, changes: impl Iterator>>, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, ) -> Result<()> { let rtxn = &context.rtxn; let index = context.index; diff --git a/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs b/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs index 046116939..35bc9f063 100644 --- a/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs +++ b/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs @@ -11,7 +11,7 @@ use super::tokenize_document::{tokenizer_builder, DocumentTokenizer}; use crate::update::new::extract::cache::BalancedCaches; use crate::update::new::extract::perm_json_p::contained_in; use crate::update::new::indexer::document_changes::{ - extract, DocumentChangeContext, DocumentChanges, Extractor, IndexingContext, + extract, DocumentContext, DocumentChanges, Extractor, IndexingContext, }; use crate::update::new::ref_cell_ext::RefCellExt as _; use crate::update::new::steps::IndexingStep; @@ -226,7 +226,7 @@ impl<'extractor> Extractor<'extractor> for WordDocidsExtractorData<'_> { fn process<'doc>( &self, changes: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> Result<()> { for change in changes { let change = change?; @@ -305,7 +305,7 @@ impl WordDocidsExtractors { } fn extract_document_change( - context: &DocumentChangeContext>>, + context: &DocumentContext>>, document_tokenizer: &DocumentTokenizer, searchable_attributes: Option<&[&str]>, document_change: DocumentChange, diff --git a/crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs b/crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs index 3b358800f..dffde06c7 100644 --- a/crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs +++ b/crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs @@ -10,7 +10,7 @@ use crate::proximity::{index_proximity, MAX_DISTANCE}; use crate::update::new::document::Document; use crate::update::new::extract::cache::BalancedCaches; use crate::update::new::indexer::document_changes::{ - extract, DocumentChangeContext, DocumentChanges, Extractor, IndexingContext, + extract, DocumentContext, DocumentChanges, Extractor, IndexingContext, }; use crate::update::new::ref_cell_ext::RefCellExt as _; use crate::update::new::steps::IndexingStep; @@ -39,7 +39,7 @@ impl<'extractor> Extractor<'extractor> for WordPairProximityDocidsExtractorData< fn process<'doc>( &self, changes: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> Result<()> { for change in changes { let change = change?; @@ -116,7 +116,7 @@ impl WordPairProximityDocidsExtractor { // and to store the docids of the documents that have a number of words in a given field // equal to or under than MAX_COUNTED_WORDS. fn extract_document_change( - context: &DocumentChangeContext>, + context: &DocumentContext>, document_tokenizer: &DocumentTokenizer, searchable_attributes: Option<&[&str]>, document_change: DocumentChange, diff --git a/crates/milli/src/update/new/extract/vectors/mod.rs b/crates/milli/src/update/new/extract/vectors/mod.rs index 73a418b19..c5e56356d 100644 --- a/crates/milli/src/update/new/extract/vectors/mod.rs +++ b/crates/milli/src/update/new/extract/vectors/mod.rs @@ -11,7 +11,7 @@ use crate::progress::EmbedderStats; use crate::prompt::Prompt; use crate::update::new::channel::EmbeddingSender; use crate::update::new::document_change::DatabaseDocument; -use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor}; +use crate::update::new::indexer::document_changes::{DocumentContext, Extractor}; use crate::update::new::indexer::settings_changes::SettingsChangeExtractor; use crate::update::new::thread_local::MostlySend; use crate::update::new::vector_document::VectorDocument; @@ -60,7 +60,7 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> { fn process<'doc>( &'doc self, changes: impl Iterator>>, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, ) -> crate::Result<()> { let embedders = self.embedders.inner_as_ref(); let mut unused_vectors_distribution = @@ -341,7 +341,7 @@ impl<'extractor> SettingsChangeExtractor<'extractor> for SettingsChangeEmbedding fn process<'doc>( &'doc self, documents: impl Iterator>>, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, ) -> crate::Result<()> { let embedders = self.embedders.inner_as_ref(); let old_embedders = self.old_embedders.inner_as_ref(); diff --git a/crates/milli/src/update/new/indexer/document_changes.rs b/crates/milli/src/update/new/indexer/document_changes.rs index ca5bc8dc5..3069ab29b 100644 --- a/crates/milli/src/update/new/indexer/document_changes.rs +++ b/crates/milli/src/update/new/indexer/document_changes.rs @@ -15,7 +15,7 @@ use crate::update::new::thread_local::{FullySend, MostlySend, ThreadLocal}; use crate::update::GrenadParameters; use crate::{FieldsIdsMap, GlobalFieldsIdsMap, Index, InternalError, Result}; -pub struct DocumentChangeContext< +pub struct DocumentContext< 'doc, // covariant lifetime of a single `process` call 'extractor: 'doc, // invariant lifetime of the extractor_allocs 'fid: 'doc, // invariant lifetime of the new_fields_ids_map @@ -56,7 +56,7 @@ impl< 'fid: 'doc, // invariant lifetime of fields ids map 'indexer: 'doc, // covariant lifetime of objects that survive a `process` call T: MostlySend, - > DocumentChangeContext<'doc, 'extractor, 'fid, 'indexer, T> + > DocumentContext<'doc, 'extractor, 'fid, 'indexer, T> { #[allow(clippy::too_many_arguments)] pub fn new( @@ -84,7 +84,7 @@ impl< let data = datastore.get_or_try(move || init_data(&extractor_alloc.0))?; let txn = index.read_txn()?; - Ok(DocumentChangeContext { + Ok(DocumentContext { index, rtxn: txn, db_fields_ids_map, @@ -106,7 +106,7 @@ pub trait Extractor<'extractor>: Sync { fn process<'doc>( &'doc self, changes: impl Iterator>>, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, ) -> Result<()>; } @@ -125,7 +125,7 @@ pub trait DocumentChanges<'pl // lifetime of the underlying payload fn item_to_document_change<'doc, // lifetime of a single `process` call T: MostlySend>( &'doc self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, item: &'doc Self::Item, ) -> Result>> where 'pl: 'doc // the payload must survive the process calls ; @@ -224,7 +224,7 @@ where let pi = document_changes.iter(CHUNK_SIZE); pi.try_arc_for_each_try_init( || { - DocumentChangeContext::new( + DocumentContext::new( index, db_fields_ids_map, new_fields_ids_map, diff --git a/crates/milli/src/update/new/indexer/document_deletion.rs b/crates/milli/src/update/new/indexer/document_deletion.rs index 114ce0a69..292cdc36e 100644 --- a/crates/milli/src/update/new/indexer/document_deletion.rs +++ b/crates/milli/src/update/new/indexer/document_deletion.rs @@ -4,7 +4,7 @@ use rayon::iter::IndexedParallelIterator; use rayon::slice::ParallelSlice as _; use roaring::RoaringBitmap; -use super::document_changes::{DocumentChangeContext, DocumentChanges}; +use super::document_changes::{DocumentContext, DocumentChanges}; use crate::documents::PrimaryKey; use crate::update::new::thread_local::MostlySend; use crate::update::new::{DatabaseDocument, DocumentChange}; @@ -58,7 +58,7 @@ impl<'pl> DocumentChanges<'pl> for DocumentDeletionChanges<'pl> { T: MostlySend, >( &'doc self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, docid: &'doc Self::Item, ) -> Result>> where @@ -94,7 +94,7 @@ mod test { use crate::index::tests::TempIndex; use crate::progress::Progress; use crate::update::new::indexer::document_changes::{ - extract, DocumentChangeContext, Extractor, IndexingContext, + extract, DocumentContext, Extractor, IndexingContext, }; use crate::update::new::indexer::DocumentDeletion; use crate::update::new::steps::IndexingStep; @@ -125,7 +125,7 @@ mod test { fn process<'doc>( &self, changes: impl Iterator>>, - context: &DocumentChangeContext, + context: &DocumentContext, ) -> crate::Result<()> { for change in changes { let change = change?; diff --git a/crates/milli/src/update/new/indexer/document_operation.rs b/crates/milli/src/update/new/indexer/document_operation.rs index 70dc5f35c..4bcfb2d47 100644 --- a/crates/milli/src/update/new/indexer/document_operation.rs +++ b/crates/milli/src/update/new/indexer/document_operation.rs @@ -12,7 +12,7 @@ use serde_json::value::RawValue; use serde_json::Deserializer; use super::super::document_change::DocumentChange; -use super::document_changes::{DocumentChangeContext, DocumentChanges}; +use super::document_changes::{DocumentContext, DocumentChanges}; use super::guess_primary_key::retrieve_or_guess_primary_key; use crate::documents::PrimaryKey; use crate::progress::{AtomicPayloadStep, Progress}; @@ -411,7 +411,7 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> { fn item_to_document_change<'doc, T: MostlySend + 'doc>( &'doc self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, item: &'doc Self::Item, ) -> Result>> where diff --git a/crates/milli/src/update/new/indexer/partial_dump.rs b/crates/milli/src/update/new/indexer/partial_dump.rs index 6e4abd898..614c61353 100644 --- a/crates/milli/src/update/new/indexer/partial_dump.rs +++ b/crates/milli/src/update/new/indexer/partial_dump.rs @@ -5,7 +5,7 @@ use rayon::iter::IndexedParallelIterator; use rustc_hash::FxBuildHasher; use serde_json::value::RawValue; -use super::document_changes::{DocumentChangeContext, DocumentChanges}; +use super::document_changes::{DocumentContext, DocumentChanges}; use crate::documents::PrimaryKey; use crate::update::concurrent_available_ids::ConcurrentAvailableIds; use crate::update::new::document::Versions; @@ -55,7 +55,7 @@ where fn item_to_document_change<'doc, T: MostlySend + 'doc>( &'doc self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, document: &'doc Self::Item, ) -> Result>> where diff --git a/crates/milli/src/update/new/indexer/settings_changes.rs b/crates/milli/src/update/new/indexer/settings_changes.rs index f92935399..90c451534 100644 --- a/crates/milli/src/update/new/indexer/settings_changes.rs +++ b/crates/milli/src/update/new/indexer/settings_changes.rs @@ -9,7 +9,7 @@ use super::document_changes::IndexingContext; use crate::documents::PrimaryKey; use crate::progress::AtomicDocumentStep; use crate::update::new::document_change::DatabaseDocument; -use crate::update::new::indexer::document_changes::DocumentChangeContext; +use crate::update::new::indexer::document_changes::DocumentContext; use crate::update::new::parallel_iterator_ext::ParallelIteratorExt as _; use crate::update::new::steps::IndexingStep; use crate::update::new::thread_local::{FullySend, MostlySend, ThreadLocal}; @@ -24,7 +24,7 @@ pub trait SettingsChangeExtractor<'extractor>: Sync { fn process<'doc>( &'doc self, changes: impl Iterator>>, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, ) -> Result<()>; } pub struct DatabaseDocuments<'indexer> { @@ -46,7 +46,7 @@ impl<'indexer> DatabaseDocuments<'indexer> { T: MostlySend, >( &'doc self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, docid: &'doc DocumentId, ) -> Result>> { let current = context.index.document(&context.rtxn, *docid)?; @@ -109,7 +109,7 @@ pub fn settings_change_extract< let pi = documents.iter(CHUNK_SIZE); pi.try_arc_for_each_try_init( || { - DocumentChangeContext::new( + DocumentContext::new( index, db_fields_ids_map, new_fields_ids_map, diff --git a/crates/milli/src/update/new/indexer/update_by_function.rs b/crates/milli/src/update/new/indexer/update_by_function.rs index 694645d28..b394757d1 100644 --- a/crates/milli/src/update/new/indexer/update_by_function.rs +++ b/crates/milli/src/update/new/indexer/update_by_function.rs @@ -5,7 +5,7 @@ use rhai::{Dynamic, Engine, OptimizationLevel, Scope, AST}; use roaring::RoaringBitmap; use rustc_hash::FxBuildHasher; -use super::document_changes::DocumentChangeContext; +use super::document_changes::DocumentContext; use super::DocumentChanges; use crate::documents::Error::InvalidDocumentFormat; use crate::documents::PrimaryKey; @@ -86,13 +86,13 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> { fn item_to_document_change<'doc, T: MostlySend + 'doc>( &self, - context: &'doc DocumentChangeContext, + context: &'doc DocumentContext, docid: &'doc Self::Item, ) -> Result>> where 'index: 'doc, { - let DocumentChangeContext { + let DocumentContext { index, db_fields_ids_map, rtxn: txn,