searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Remove lots of Arcs

Browse source
This commit is contained in:
Mubelotix 2025-06-26 12:07:48 +02:00
parent ef007d547d
commit 29f6eeff8f
No known key found for this signature in database
GPG key ID: 0406DF6C3A69B942
22 changed files with 112 additions and 118 deletions
Download patch file Download diff file Expand all files Collapse all files

8
crates/milli/src/update/index_documents/extract/extract_vector_points.rs
View file

@ -684,7 +684,7 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
embedder: Arc<Embedder>,
embedder_name: &str,
possible_embedding_mistakes: &PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &EmbedderStats,
unused_vectors_distribution: &UnusedVectorsDistribution,
request_threads: &ThreadPoolNoAbort,
) -> Result<grenad::Reader<BufReader<File>>> {
@ -727,7 +727,7 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
std::mem::replace(&mut chunks, Vec::with_capacity(n_chunks)),
embedder_name,
possible_embedding_mistakes,
embedder_stats.clone(),
embedder_stats,
unused_vectors_distribution,
request_threads,
)?;
@ -750,7 +750,7 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
std::mem::take(&mut chunks),
embedder_name,
possible_embedding_mistakes,
embedder_stats.clone(),
embedder_stats,
unused_vectors_distribution,
request_threads,
)?;
@ -789,7 +789,7 @@ fn embed_chunks(
text_chunks: Vec<Vec<String>>,
embedder_name: &str,
possible_embedding_mistakes: &PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &EmbedderStats,
unused_vectors_distribution: &UnusedVectorsDistribution,
request_threads: &ThreadPoolNoAbort,
) -> Result<Vec<Vec<Embedding>>> {

6
crates/milli/src/update/index_documents/extract/mod.rs
View file

@ -50,7 +50,7 @@ pub(crate) fn data_from_obkv_documents(
settings_diff: Arc<InnerIndexSettingsDiff>,
max_positions_per_attributes: Option<u32>,
possible_embedding_mistakes: Arc<PossibleEmbeddingMistakes>,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
) -> Result<()> {
let (original_pipeline_result, flattened_pipeline_result): (Result<_>, Result<_>) = rayon::join(
|| {
@ -234,7 +234,7 @@ fn send_original_documents_data(
embedders_configs: Arc<Vec<IndexEmbeddingConfig>>,
settings_diff: Arc<InnerIndexSettingsDiff>,
possible_embedding_mistakes: Arc<PossibleEmbeddingMistakes>,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
) -> Result<()> {
let original_documents_chunk =
original_documents_chunk.and_then(|c| unsafe { as_cloneable_grenad(&c) })?;
@ -274,7 +274,7 @@ fn send_original_documents_data(
embedder.clone(),
&embedder_name,
&possible_embedding_mistakes,
embedder_stats.clone(),
&embedder_stats,
&unused_vectors_distribution,
request_threads(),
) {

30
crates/milli/src/update/index_documents/mod.rs
View file

@ -81,7 +81,7 @@ pub struct IndexDocuments<'t, 'i, 'a, FP, FA> {
added_documents: u64,
deleted_documents: u64,
embedders: EmbeddingConfigs,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
}
#[derive(Default, Debug, Clone)]
@ -104,7 +104,7 @@ where
config: IndexDocumentsConfig,
progress: FP,
should_abort: FA,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
) -> Result<IndexDocuments<'t, 'i, 'a, FP, FA>> {
let transform = Some(Transform::new(
wtxn,
@ -2030,7 +2030,7 @@ mod tests {
EmbeddingConfigs::default(),
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2118,7 +2118,7 @@ mod tests {
EmbeddingConfigs::default(),
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2304,7 +2304,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2367,7 +2367,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2421,7 +2421,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2474,7 +2474,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2529,7 +2529,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2589,7 +2589,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2642,7 +2642,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2695,7 +2695,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2894,7 +2894,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -2954,7 +2954,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();
@ -3011,7 +3011,7 @@ mod tests {
embedders,
&|| false,
&Progress::default(),
Default::default(),
&Default::default(),
)
.unwrap();
wtxn.commit().unwrap();

18
crates/milli/src/update/new/extract/vectors/mod.rs
View file

@ -1,4 +1,4 @@
use std::{cell::RefCell, sync::Arc};
use std::cell::RefCell;
use bumpalo::collections::Vec as BVec;
use bumpalo::Bump;
@ -23,7 +23,7 @@ pub struct EmbeddingExtractor<'a, 'b> {
embedders: &'a EmbeddingConfigs,
sender: EmbeddingSender<'a, 'b>,
possible_embedding_mistakes: PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &'a EmbedderStats,
threads: &'a ThreadPoolNoAbort,
}
@ -32,7 +32,7 @@ impl<'a, 'b> EmbeddingExtractor<'a, 'b> {
embedders: &'a EmbeddingConfigs,
sender: EmbeddingSender<'a, 'b>,
field_distribution: &'a FieldDistribution,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &'a EmbedderStats,
threads: &'a ThreadPoolNoAbort,
) -> Self {
let possible_embedding_mistakes = PossibleEmbeddingMistakes::new(field_distribution);
@ -78,7 +78,7 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
prompt,
context.data,
&self.possible_embedding_mistakes,
self.embedder_stats.clone(),
self.embedder_stats,
self.threads,
self.sender,
&context.doc_alloc,
@ -311,7 +311,7 @@ struct Chunks<'a, 'b, 'extractor> {
dimensions: usize,
prompt: &'a Prompt,
possible_embedding_mistakes: &'a PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &'a EmbedderStats,
user_provided: &'a RefCell<EmbeddingExtractorData<'extractor>>,
threads: &'a ThreadPoolNoAbort,
sender: EmbeddingSender<'a, 'b>,
@ -327,7 +327,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
prompt: &'a Prompt,
user_provided: &'a RefCell<EmbeddingExtractorData<'extractor>>,
possible_embedding_mistakes: &'a PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &'a EmbedderStats,
threads: &'a ThreadPoolNoAbort,
sender: EmbeddingSender<'a, 'b>,
doc_alloc: &'a Bump,
@ -378,7 +378,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
self.embedder_id,
self.embedder_name,
self.possible_embedding_mistakes,
self.embedder_stats.clone(),
self.embedder_stats,
unused_vectors_distribution,
self.threads,
self.sender,
@ -397,7 +397,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
self.embedder_id,
self.embedder_name,
self.possible_embedding_mistakes,
self.embedder_stats.clone(),
self.embedder_stats,
unused_vectors_distribution,
self.threads,
self.sender,
@ -416,7 +416,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
embedder_id: u8,
embedder_name: &str,
possible_embedding_mistakes: &PossibleEmbeddingMistakes,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &EmbedderStats,
unused_vectors_distribution: &UnusedVectorsDistributionBump,
threads: &ThreadPoolNoAbort,
sender: EmbeddingSender<'a, 'b>,

3
crates/milli/src/update/new/indexer/extract.rs
View file

@ -1,6 +1,5 @@
use std::collections::BTreeMap;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::sync::OnceLock;
use bumpalo::Bump;
@ -36,7 +35,7 @@ pub(super) fn extract_all<'pl, 'extractor, DC, MSP>(
mut index_embeddings: Vec<IndexEmbeddingConfig>,
document_ids: &mut RoaringBitmap,
modified_docids: &mut RoaringBitmap,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &EmbedderStats,
) -> Result<(FacetFieldIdsDelta, Vec<IndexEmbeddingConfig>)>
where
DC: DocumentChanges<'pl>,

3
crates/milli/src/update/new/indexer/mod.rs
View file

@ -1,5 +1,4 @@
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::sync::{Once, RwLock};
use std::thread::{self, Builder};
@ -56,7 +55,7 @@ pub fn index<'pl, 'indexer, 'index, DC, MSP>(
embedders: EmbeddingConfigs,
must_stop_processing: &'indexer MSP,
progress: &'indexer Progress,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: &'indexer EmbedderStats,
) -> Result<ChannelCongestion>
where
DC: DocumentChanges<'pl>,

4
crates/milli/src/update/settings.rs
View file

@ -475,7 +475,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
progress_callback: &FP,
should_abort: &FA,
settings_diff: InnerIndexSettingsDiff,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
) -> Result<()>
where
FP: Fn(UpdateIndexingStep) + Sync,
@ -1362,7 +1362,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
mut self,
progress_callback: FP,
should_abort: FA,
embedder_stats: Arc<EmbedderStats>,
embedder_stats: Arc<EmbedderStats>, // Cant change
) -> Result<()>
where
FP: Fn(UpdateIndexingStep) + Sync,