From 5716ab70f38c521f768f98f64aa32399f0fedb54 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 30 Jun 2025 00:07:32 +0200 Subject: [PATCH] EmbeddingConfigs -> RuntimeEmbedders --- crates/benchmarks/benches/indexing.rs | 64 +++++++++---------- crates/benchmarks/benches/utils.rs | 4 +- crates/fuzzers/src/bin/fuzz-indexing.rs | 4 +- crates/index-scheduler/src/lib.rs | 58 ++++++++++++----- .../src/scheduler/process_dump_creation.rs | 11 +--- .../src/scheduler/process_index_operation.rs | 9 ++- crates/index-scheduler/src/scheduler/test.rs | 2 +- crates/meilisearch/src/lib.rs | 2 +- .../src/routes/indexes/documents.rs | 13 +--- crates/meilisearch/src/search/mod.rs | 19 ++---- crates/meilitool/src/main.rs | 10 +-- .../milli/src/search/new/tests/integration.rs | 4 +- crates/milli/src/search/new/vector_sort.rs | 4 +- crates/milli/src/search/similar.rs | 11 ++-- crates/milli/src/test_index.rs | 4 +- .../milli/tests/search/facet_distribution.rs | 4 +- crates/milli/tests/search/mod.rs | 4 +- crates/milli/tests/search/query_criteria.rs | 4 +- crates/milli/tests/search/typo_tolerance.rs | 4 +- 19 files changed, 118 insertions(+), 117 deletions(-) diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 16e7a2f81..4083b69dd 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -11,7 +11,7 @@ use milli::heed::{EnvOpenOptions, RwTxn}; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::{FilterableAttributesRule, Index}; use rand::seq::SliceRandom; use rand_chacha::rand_core::SeedableRng; @@ -166,7 +166,7 @@ fn indexing_songs_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -233,7 +233,7 @@ fn reindexing_songs_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -278,7 +278,7 @@ fn reindexing_songs_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -347,7 +347,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -424,7 +424,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -469,7 +469,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -510,7 +510,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -578,7 +578,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -645,7 +645,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -712,7 +712,7 @@ fn indexing_wiki(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -778,7 +778,7 @@ fn reindexing_wiki(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -823,7 +823,7 @@ fn reindexing_wiki(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -891,7 +891,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -968,7 +968,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1014,7 +1014,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1056,7 +1056,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1123,7 +1123,7 @@ fn indexing_movies_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1189,7 +1189,7 @@ fn reindexing_movies_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1234,7 +1234,7 @@ fn reindexing_movies_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1302,7 +1302,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), @@ -1351,7 +1351,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec Index { new_fields_ids_map, primary_key, &document_changes, - EmbeddingConfigs::default(), + RuntimeEmbedders::default(), &|| false, &Progress::default(), &Default::default(), diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index 0632b7846..ec1f96fd5 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -13,7 +13,7 @@ use milli::heed::EnvOpenOptions; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::IndexerConfig; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::Index; use serde_json::Value; use tempfile::TempDir; @@ -89,7 +89,7 @@ fn main() { let mut new_fields_ids_map = db_fields_ids_map.clone(); let indexer_alloc = Bump::new(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let mut operations = Vec::new(); diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 505ce23f8..f551652c1 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -57,12 +57,15 @@ use meilisearch_types::features::{ use meilisearch_types::heed::byteorder::BE; use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128}; use meilisearch_types::heed::{self, Database, Env, RoTxn, WithoutTls}; -use meilisearch_types::milli::index::IndexEmbeddingConfig; use meilisearch_types::milli::update::IndexerConfig; -use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs}; +use meilisearch_types::milli::vector::json_template::JsonTemplate; +use meilisearch_types::milli::vector::{ + Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment, +}; use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task}; +use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; pub use queue::Query; use queue::Queue; @@ -851,29 +854,42 @@ impl IndexScheduler { &self, index_uid: String, embedding_configs: Vec, - ) -> Result { + ) -> Result { let res: Result<_> = embedding_configs .into_iter() .map( |IndexEmbeddingConfig { name, config: milli::vector::EmbeddingConfig { embedder_options, prompt, quantized }, - .. - }| { - let prompt = Arc::new( - prompt - .try_into() - .map_err(meilisearch_types::milli::Error::from) - .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?, - ); + fragments, + }| + -> Result<(String, Arc)> { + let document_template = prompt + .try_into() + .map_err(meilisearch_types::milli::Error::from) + .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?; + + let fragments = fragments + .into_inner() + .into_iter() + .map(|fragment| { + let value = embedder_options.fragment(&fragment.name).unwrap(); + let template = JsonTemplate::new(value.clone()).unwrap(); + RuntimeFragment { name: fragment.name, id: fragment.id, template } + }) + .collect(); // optimistically return existing embedder { let embedders = self.embedders.read().unwrap(); if let Some(embedder) = embedders.get(&embedder_options) { - return Ok(( - name, - (embedder.clone(), prompt, quantized.unwrap_or_default()), - )); + let runtime = Arc::new(RuntimeEmbedder { + embedder: embedder.clone(), + document_template, + fragments, + is_quantized: quantized.unwrap_or_default(), + }); + + return Ok((name, runtime)); } } @@ -889,11 +905,19 @@ impl IndexScheduler { let mut embedders = self.embedders.write().unwrap(); embedders.insert(embedder_options, embedder.clone()); } - Ok((name, (embedder, prompt, quantized.unwrap_or_default()))) + + let runtime = Arc::new(RuntimeEmbedder { + embedder: embedder.clone(), + document_template, + fragments, + is_quantized: quantized.unwrap_or_default(), + }); + + Ok((name, runtime)) }, ) .collect(); - res.map(EmbeddingConfigs::new) + res.map(RuntimeEmbedders::new) } pub fn chat_settings(&self, uid: &str) -> Result> { diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index a6d785b2f..ec1be0e93 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -165,9 +165,6 @@ impl IndexScheduler { let fields_ids_map = index.fields_ids_map(&rtxn)?; let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - let embedding_configs = index - .embedding_configs(&rtxn) - .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; let nb_documents = index .number_of_documents(&rtxn) @@ -221,16 +218,12 @@ impl IndexScheduler { return Err(Error::from_milli(user_err, Some(uid.to_string()))); }; - for (embedder_name, embeddings) in embeddings { - let user_provided = embedding_configs - .iter() - .find(|conf| conf.name == embedder_name) - .is_some_and(|conf| conf.user_provided.contains(id)); + for (embedder_name, (embeddings, regenerate)) in embeddings { let embeddings = ExplicitVectors { embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings, )), - regenerate: !user_provided, + regenerate, }; vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap()); } diff --git a/crates/index-scheduler/src/scheduler/process_index_operation.rs b/crates/index-scheduler/src/scheduler/process_index_operation.rs index 04aaf9a84..62d0e6545 100644 --- a/crates/index-scheduler/src/scheduler/process_index_operation.rs +++ b/crates/index-scheduler/src/scheduler/process_index_operation.rs @@ -89,8 +89,9 @@ impl IndexScheduler { let mut content_files_iter = content_files.iter(); let mut indexer = indexer::DocumentOperation::new(); let embedders = index + .embedding_configs() .embedding_configs(index_wtxn) - .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; + .map_err(|e| Error::from_milli(e.into(), Some(index_uid.clone())))?; let embedders = self.embedders(index_uid.clone(), embedders)?; for operation in operations { match operation { @@ -274,8 +275,9 @@ impl IndexScheduler { }) .unwrap()?; let embedders = index + .embedding_configs() .embedding_configs(index_wtxn) - .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?; + .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?; let embedders = self.embedders(index_uid.clone(), embedders)?; progress.update_progress(DocumentEditionProgress::Indexing); @@ -423,8 +425,9 @@ impl IndexScheduler { indexer.delete_documents_by_docids(to_delete); let document_changes = indexer.into_changes(&indexer_alloc, primary_key); let embedders = index + .embedding_configs() .embedding_configs(index_wtxn) - .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?; + .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?; let embedders = self.embedders(index_uid.clone(), embedders)?; progress.update_progress(DocumentDeletionProgress::Indexing); diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index ee26165c7..2c492525f 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -3,11 +3,11 @@ use std::collections::BTreeMap; use big_s::S; use meili_snap::{json_string, snapshot}; use meilisearch_auth::AuthFilter; -use meilisearch_types::milli::index::IndexEmbeddingConfig; use meilisearch_types::milli::update::IndexDocumentsMethod::*; use meilisearch_types::milli::{self}; use meilisearch_types::settings::SettingEmbeddingSettings; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; +use milli::vector::db::IndexEmbeddingConfig; use roaring::RoaringBitmap; use crate::insta_snapshot::snapshot_index_scheduler; diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 871bd688e..e1acef2ce 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -563,7 +563,7 @@ fn import_dump( let reader = BufReader::new(file); let reader = DocumentsBatchReader::from_reader(reader)?; - let embedder_configs = index.embedding_configs(&wtxn)?; + let embedder_configs = index.embedding_configs().embedding_configs(&wtxn)?; let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?; let builder = milli::update::IndexDocuments::new( diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 50eec46fe..a93d736f7 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1452,7 +1452,6 @@ fn some_documents<'a, 't: 'a>( ) -> Result> + 'a, ResponseError> { let fields_ids_map = index.fields_ids_map(rtxn)?; let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - let embedding_configs = index.embedding_configs(rtxn)?; Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| { ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> { @@ -1468,15 +1467,9 @@ fn some_documents<'a, 't: 'a>( Some(Value::Object(map)) => map, _ => Default::default(), }; - for (name, vector) in index.embeddings(rtxn, key)? { - let user_provided = embedding_configs - .iter() - .find(|conf| conf.name == name) - .is_some_and(|conf| conf.user_provided.contains(key)); - let embeddings = ExplicitVectors { - embeddings: Some(vector.into()), - regenerate: !user_provided, - }; + for (name, (vector, regenerate)) in index.embeddings(rtxn, key)? { + let embeddings = + ExplicitVectors { embeddings: Some(vector.into()), regenerate }; vectors.insert( name, serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?, diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 5e543c53f..61ef3f813 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -399,10 +399,10 @@ impl SearchKind { route: Route, ) -> Result<(String, Arc, bool), ResponseError> { let rtxn = index.read_txn()?; - let embedder_configs = index.embedding_configs(&rtxn)?; + let embedder_configs = index.embedding_configs().embedding_configs(&rtxn)?; let embedders = index_scheduler.embedders(index_uid, embedder_configs)?; - let (embedder, _, quantized) = embedders + let (embedder, quantized) = embedders .get(embedder_name) .ok_or(match route { Route::Search | Route::MultiSearch => { @@ -412,6 +412,7 @@ impl SearchKind { milli::UserError::InvalidSimilarEmbedder(embedder_name.to_owned()) } }) + .map(|runtime| (runtime.embedder.clone(), runtime.is_quantized)) .map_err(milli::Error::from)?; if let Some(vector_len) = vector_len { @@ -1328,7 +1329,6 @@ struct HitMaker<'a> { vectors_fid: Option, retrieve_vectors: RetrieveVectors, to_retrieve_ids: BTreeSet, - embedding_configs: Vec, formatter_builder: MatcherBuilder<'a>, formatted_options: BTreeMap, show_ranking_score: bool, @@ -1443,8 +1443,6 @@ impl<'a> HitMaker<'a> { &displayed_ids, ); - let embedding_configs = index.embedding_configs(rtxn)?; - Ok(Self { index, rtxn, @@ -1453,7 +1451,6 @@ impl<'a> HitMaker<'a> { vectors_fid, retrieve_vectors, to_retrieve_ids, - embedding_configs, formatter_builder, formatted_options, show_ranking_score: format.show_ranking_score, @@ -1499,14 +1496,8 @@ impl<'a> HitMaker<'a> { Some(Value::Object(map)) => map, _ => Default::default(), }; - for (name, vector) in self.index.embeddings(self.rtxn, id)? { - let user_provided = self - .embedding_configs - .iter() - .find(|conf| conf.name == name) - .is_some_and(|conf| conf.user_provided.contains(id)); - let embeddings = - ExplicitVectors { embeddings: Some(vector.into()), regenerate: !user_provided }; + for (name, (vector, regenerate)) in self.index.embeddings(self.rtxn, id)? { + let embeddings = ExplicitVectors { embeddings: Some(vector.into()), regenerate }; vectors.insert( name, serde_json::to_value(embeddings).map_err(InternalError::SerdeJson)?, diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index dd1213782..b967e620c 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -545,7 +545,6 @@ fn export_documents( let rtxn = index.read_txn()?; let fields_ids_map = index.fields_ids_map(&rtxn)?; let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - let embedding_configs = index.embedding_configs(&rtxn)?; if let Some(offset) = offset { eprintln!("Skipping {offset} documents"); @@ -592,17 +591,12 @@ fn export_documents( .into()); }; - for (embedder_name, embeddings) in embeddings { - let user_provided = embedding_configs - .iter() - .find(|conf| conf.name == embedder_name) - .is_some_and(|conf| conf.user_provided.contains(id)); - + for (embedder_name, (embeddings, regenerate)) in embeddings { let embeddings = ExplicitVectors { embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings, )), - regenerate: !user_provided, + regenerate, }; vectors .insert(embedder_name, serde_json::to_value(embeddings).unwrap()); diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index 9e2afca97..38f39e18b 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -8,7 +8,7 @@ use maplit::{btreemap, hashset}; use crate::progress::Progress; use crate::update::new::indexer; use crate::update::{IndexerConfig, Settings}; -use crate::vector::EmbeddingConfigs; +use crate::vector::RuntimeEmbedders; use crate::{db_snap, Criterion, FilterableAttributesRule, Index}; pub const CONTENT: &str = include_str!("../../../../tests/assets/test_set.ndjson"); use crate::constants::RESERVED_GEO_FIELD_NAME; @@ -55,7 +55,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let mut file = tempfile::tempfile().unwrap(); diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index 834f97384..2c201e899 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -32,8 +32,8 @@ impl VectorSort { ) -> Result { let embedder_index = ctx .index - .embedder_category_id - .get(ctx.txn, embedder_name)? + .embedding_configs() + .embedder_id(ctx.txn, embedder_name)? .ok_or_else(|| crate::UserError::InvalidSearchEmbedder(embedder_name.to_owned()))?; Ok(Self { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 759940f9c..903b5fcf9 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -64,10 +64,13 @@ impl<'a> Similar<'a> { let universe = universe; - let embedder_index = - self.index.embedder_category_id.get(self.rtxn, &self.embedder_name)?.ok_or_else( - || crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()), - )?; + let embedder_index = self + .index + .embedding_configs() + .embedder_id(self.rtxn, &self.embedder_name)? + .ok_or_else(|| { + crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) + })?; let reader = ArroyWrapper::new(self.index.vector_arroy, embedder_index, self.quantized); let results = reader.nns_by_item( diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index f2e34c615..cfd8c8492 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -18,7 +18,7 @@ use crate::update::{ self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings, }; use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; -use crate::vector::EmbeddingConfigs; +use crate::vector::RuntimeEmbedders; use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; pub(crate) struct TempIndex { @@ -223,7 +223,7 @@ fn aborting_indexation() { let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let payload = documents!([ { "id": 1, "name": "kevin" }, diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index d04db425e..cc1b85369 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -5,7 +5,7 @@ use milli::documents::mmap_from_objects; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::{FacetDistribution, FilterableAttributesRule, Index, Object, OrderBy}; use serde_json::{from_value, json}; @@ -35,7 +35,7 @@ fn test_facet_distribution_with_no_facet_values() { let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let doc1: Object = from_value( diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index 3ee78561d..fa03f1cc1 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -10,7 +10,7 @@ use maplit::{btreemap, hashset}; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::{ AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, TermsMatchingStrategy, }; @@ -74,7 +74,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let mut file = tempfile::tempfile().unwrap(); diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index cb0c23e42..3f8134085 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -8,7 +8,7 @@ use maplit::hashset; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::{AscDesc, Criterion, Index, Member, Search, SearchResult, TermsMatchingStrategy}; use rand::Rng; use Criterion::*; @@ -288,7 +288,7 @@ fn criteria_ascdesc() { let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); let mut file = tempfile::tempfile().unwrap(); diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index 49c9c7b5d..95ff85165 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -6,7 +6,7 @@ use milli::documents::mmap_from_objects; use milli::progress::Progress; use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; -use milli::vector::EmbeddingConfigs; +use milli::vector::RuntimeEmbedders; use milli::{Criterion, Index, Object, Search, TermsMatchingStrategy}; use serde_json::from_value; use tempfile::tempdir; @@ -123,7 +123,7 @@ fn test_typo_disabled_on_word() { let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let mut new_fields_ids_map = db_fields_ids_map.clone(); - let embedders = EmbeddingConfigs::default(); + let embedders = RuntimeEmbedders::default(); let mut indexer = indexer::DocumentOperation::new(); indexer.replace_documents(&documents).unwrap();