searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 04:17:10 +02:00
Code Issues Releases Wiki Activity

EmbeddingConfigs -> RuntimeEmbedders

Browse source
This commit is contained in:
Louis Dureuil 2025-06-30 00:07:32 +02:00
parent 422a786ffd
commit 5716ab70f3
No known key found for this signature in database
19 changed files with 118 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

64
crates/benchmarks/benches/indexing.rs
View file

@ -11,7 +11,7 @@ use milli::heed::{EnvOpenOptions, RwTxn};
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{FilterableAttributesRule, Index}; use milli::{FilterableAttributesRule, Index};
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
use rand_chacha::rand_core::SeedableRng; use rand_chacha::rand_core::SeedableRng;
@ -166,7 +166,7 @@ fn indexing_songs_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -233,7 +233,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -278,7 +278,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -347,7 +347,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -424,7 +424,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -469,7 +469,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -510,7 +510,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -578,7 +578,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -645,7 +645,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -712,7 +712,7 @@ fn indexing_wiki(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -778,7 +778,7 @@ fn reindexing_wiki(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -823,7 +823,7 @@ fn reindexing_wiki(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -891,7 +891,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -968,7 +968,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1014,7 +1014,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1056,7 +1056,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1123,7 +1123,7 @@ fn indexing_movies_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1189,7 +1189,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1234,7 +1234,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1302,7 +1302,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1351,7 +1351,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBi
new_fields_ids_map, new_fields_ids_map,
Some(primary_key), Some(primary_key),
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1416,7 +1416,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1461,7 +1461,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1502,7 +1502,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1592,7 +1592,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1683,7 +1683,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1766,7 +1766,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1833,7 +1833,7 @@ fn indexing_geo(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1899,7 +1899,7 @@ fn reindexing_geo(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -1944,7 +1944,7 @@ fn reindexing_geo(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),
@ -2012,7 +2012,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),

4
crates/benchmarks/benches/utils.rs
View file

@ -13,7 +13,7 @@ use milli::heed::EnvOpenOptions;
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy}; use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
use serde_json::Value; use serde_json::Value;
@ -125,7 +125,7 @@ pub fn base_setup(conf: &Conf) -> Index {
new_fields_ids_map, new_fields_ids_map,
primary_key, primary_key,
&document_changes, &document_changes,
EmbeddingConfigs::default(), RuntimeEmbedders::default(),
&|| false, &|| false,
&Progress::default(), &Progress::default(),
&Default::default(), &Default::default(),

4
crates/fuzzers/src/bin/fuzz-indexing.rs
View file

@ -13,7 +13,7 @@ use milli::heed::EnvOpenOptions;
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::IndexerConfig; use milli::update::IndexerConfig;
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::Index; use milli::Index;
use serde_json::Value; use serde_json::Value;
use tempfile::TempDir; use tempfile::TempDir;
@ -89,7 +89,7 @@ fn main() {
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let indexer_alloc = Bump::new(); let indexer_alloc = Bump::new();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let mut operations = Vec::new(); let mut operations = Vec::new();

58
crates/index-scheduler/src/lib.rs
View file

@ -57,12 +57,15 @@ use meilisearch_types::features::{
use meilisearch_types::heed::byteorder::BE; use meilisearch_types::heed::byteorder::BE;
use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128}; use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128};
use meilisearch_types::heed::{self, Database, Env, RoTxn, WithoutTls}; use meilisearch_types::heed::{self, Database, Env, RoTxn, WithoutTls};
use meilisearch_types::milli::index::IndexEmbeddingConfig;
use meilisearch_types::milli::update::IndexerConfig; use meilisearch_types::milli::update::IndexerConfig;
use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs}; use meilisearch_types::milli::vector::json_template::JsonTemplate;
use meilisearch_types::milli::vector::{
Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment,
};
use meilisearch_types::milli::{self, Index}; use meilisearch_types::milli::{self, Index};
use meilisearch_types::task_view::TaskView; use meilisearch_types::task_view::TaskView;
use meilisearch_types::tasks::{KindWithContent, Task}; use meilisearch_types::tasks::{KindWithContent, Task};
use milli::vector::db::IndexEmbeddingConfig;
use processing::ProcessingTasks; use processing::ProcessingTasks;
pub use queue::Query; pub use queue::Query;
use queue::Queue; use queue::Queue;
@ -851,29 +854,42 @@ impl IndexScheduler {
&self, &self,
index_uid: String, index_uid: String,
embedding_configs: Vec<IndexEmbeddingConfig>, embedding_configs: Vec<IndexEmbeddingConfig>,
) -> Result<EmbeddingConfigs> { ) -> Result<RuntimeEmbedders> {
let res: Result<_> = embedding_configs let res: Result<_> = embedding_configs
.into_iter() .into_iter()
.map( .map(
|IndexEmbeddingConfig { |IndexEmbeddingConfig {
name, name,
config: milli::vector::EmbeddingConfig { embedder_options, prompt, quantized }, config: milli::vector::EmbeddingConfig { embedder_options, prompt, quantized },
.. fragments,
}| { }|
let prompt = Arc::new( -> Result<(String, Arc<RuntimeEmbedder>)> {
prompt let document_template = prompt
.try_into() .try_into()
.map_err(meilisearch_types::milli::Error::from) .map_err(meilisearch_types::milli::Error::from)
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?, .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?;
);
let fragments = fragments
.into_inner()
.into_iter()
.map(|fragment| {
let value = embedder_options.fragment(&fragment.name).unwrap();
let template = JsonTemplate::new(value.clone()).unwrap();
RuntimeFragment { name: fragment.name, id: fragment.id, template }
})
.collect();
// optimistically return existing embedder // optimistically return existing embedder
{ {
let embedders = self.embedders.read().unwrap(); let embedders = self.embedders.read().unwrap();
if let Some(embedder) = embedders.get(&embedder_options) { if let Some(embedder) = embedders.get(&embedder_options) {
return Ok(( let runtime = Arc::new(RuntimeEmbedder {
name, embedder: embedder.clone(),
(embedder.clone(), prompt, quantized.unwrap_or_default()), document_template,
)); fragments,
is_quantized: quantized.unwrap_or_default(),
});
return Ok((name, runtime));
} }
} }
@ -889,11 +905,19 @@ impl IndexScheduler {
let mut embedders = self.embedders.write().unwrap(); let mut embedders = self.embedders.write().unwrap();
embedders.insert(embedder_options, embedder.clone()); embedders.insert(embedder_options, embedder.clone());
} }
Ok((name, (embedder, prompt, quantized.unwrap_or_default())))
let runtime = Arc::new(RuntimeEmbedder {
embedder: embedder.clone(),
document_template,
fragments,
is_quantized: quantized.unwrap_or_default(),
});
Ok((name, runtime))
}, },
) )
.collect(); .collect();
res.map(EmbeddingConfigs::new) res.map(RuntimeEmbedders::new)
} }
pub fn chat_settings(&self, uid: &str) -> Result<Option<ChatCompletionSettings>> { pub fn chat_settings(&self, uid: &str) -> Result<Option<ChatCompletionSettings>> {

11
crates/index-scheduler/src/scheduler/process_dump_creation.rs
View file

@ -165,9 +165,6 @@ impl IndexScheduler {
let fields_ids_map = index.fields_ids_map(&rtxn)?; let fields_ids_map = index.fields_ids_map(&rtxn)?;
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let embedding_configs = index
.embedding_configs(&rtxn)
.map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
let nb_documents = index let nb_documents = index
.number_of_documents(&rtxn) .number_of_documents(&rtxn)
@ -221,16 +218,12 @@ impl IndexScheduler {
return Err(Error::from_milli(user_err, Some(uid.to_string()))); return Err(Error::from_milli(user_err, Some(uid.to_string())));
}; };
for (embedder_name, embeddings) in embeddings { for (embedder_name, (embeddings, regenerate)) in embeddings {
let user_provided = embedding_configs
.iter()
.find(|conf| conf.name == embedder_name)
.is_some_and(|conf| conf.user_provided.contains(id));
let embeddings = ExplicitVectors { let embeddings = ExplicitVectors {
embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
embeddings, embeddings,
)), )),
regenerate: !user_provided, regenerate,
}; };
vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap()); vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap());
} }

9
crates/index-scheduler/src/scheduler/process_index_operation.rs
View file

@ -89,8 +89,9 @@ impl IndexScheduler {
let mut content_files_iter = content_files.iter(); let mut content_files_iter = content_files.iter();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let embedders = index let embedders = index
.embedding_configs()
.embedding_configs(index_wtxn) .embedding_configs(index_wtxn)
.map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; .map_err(|e| Error::from_milli(e.into(), Some(index_uid.clone())))?;
let embedders = self.embedders(index_uid.clone(), embedders)?; let embedders = self.embedders(index_uid.clone(), embedders)?;
for operation in operations { for operation in operations {
match operation { match operation {
@ -274,8 +275,9 @@ impl IndexScheduler {
}) })
.unwrap()?; .unwrap()?;
let embedders = index let embedders = index
.embedding_configs()
.embedding_configs(index_wtxn) .embedding_configs(index_wtxn)
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?; .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?;
let embedders = self.embedders(index_uid.clone(), embedders)?; let embedders = self.embedders(index_uid.clone(), embedders)?;
progress.update_progress(DocumentEditionProgress::Indexing); progress.update_progress(DocumentEditionProgress::Indexing);
@ -423,8 +425,9 @@ impl IndexScheduler {
indexer.delete_documents_by_docids(to_delete); indexer.delete_documents_by_docids(to_delete);
let document_changes = indexer.into_changes(&indexer_alloc, primary_key); let document_changes = indexer.into_changes(&indexer_alloc, primary_key);
let embedders = index let embedders = index
.embedding_configs()
.embedding_configs(index_wtxn) .embedding_configs(index_wtxn)
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?; .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?;
let embedders = self.embedders(index_uid.clone(), embedders)?; let embedders = self.embedders(index_uid.clone(), embedders)?;
progress.update_progress(DocumentDeletionProgress::Indexing); progress.update_progress(DocumentDeletionProgress::Indexing);

2
crates/index-scheduler/src/scheduler/test.rs
View file

@ -3,11 +3,11 @@ use std::collections::BTreeMap;
use big_s::S; use big_s::S;
use meili_snap::{json_string, snapshot}; use meili_snap::{json_string, snapshot};
use meilisearch_auth::AuthFilter; use meilisearch_auth::AuthFilter;
use meilisearch_types::milli::index::IndexEmbeddingConfig;
use meilisearch_types::milli::update::IndexDocumentsMethod::*; use meilisearch_types::milli::update::IndexDocumentsMethod::*;
use meilisearch_types::milli::{self}; use meilisearch_types::milli::{self};
use meilisearch_types::settings::SettingEmbeddingSettings; use meilisearch_types::settings::SettingEmbeddingSettings;
use meilisearch_types::tasks::{IndexSwap, KindWithContent}; use meilisearch_types::tasks::{IndexSwap, KindWithContent};
use milli::vector::db::IndexEmbeddingConfig;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use crate::insta_snapshot::snapshot_index_scheduler; use crate::insta_snapshot::snapshot_index_scheduler;

2
crates/meilisearch/src/lib.rs
View file

@ -563,7 +563,7 @@ fn import_dump(
let reader = BufReader::new(file); let reader = BufReader::new(file);
let reader = DocumentsBatchReader::from_reader(reader)?; let reader = DocumentsBatchReader::from_reader(reader)?;
let embedder_configs = index.embedding_configs(&wtxn)?; let embedder_configs = index.embedding_configs().embedding_configs(&wtxn)?;
let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?; let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?;
let builder = milli::update::IndexDocuments::new( let builder = milli::update::IndexDocuments::new(

13
crates/meilisearch/src/routes/indexes/documents.rs
View file

@ -1452,7 +1452,6 @@ fn some_documents<'a, 't: 'a>(
) -> Result<impl Iterator<Item = Result<Document, ResponseError>> + 'a, ResponseError> { ) -> Result<impl Iterator<Item = Result<Document, ResponseError>> + 'a, ResponseError> {
let fields_ids_map = index.fields_ids_map(rtxn)?; let fields_ids_map = index.fields_ids_map(rtxn)?;
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let embedding_configs = index.embedding_configs(rtxn)?;
Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| { Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| {
ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> { ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> {
@ -1468,15 +1467,9 @@ fn some_documents<'a, 't: 'a>(
Some(Value::Object(map)) => map, Some(Value::Object(map)) => map,
_ => Default::default(), _ => Default::default(),
}; };
for (name, vector) in index.embeddings(rtxn, key)? { for (name, (vector, regenerate)) in index.embeddings(rtxn, key)? {
let user_provided = embedding_configs let embeddings =
.iter() ExplicitVectors { embeddings: Some(vector.into()), regenerate };
.find(|conf| conf.name == name)
.is_some_and(|conf| conf.user_provided.contains(key));
let embeddings = ExplicitVectors {
embeddings: Some(vector.into()),
regenerate: !user_provided,
};
vectors.insert( vectors.insert(
name, name,
serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?, serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?,

19
crates/meilisearch/src/search/mod.rs
View file

@ -399,10 +399,10 @@ impl SearchKind {
route: Route, route: Route,
) -> Result<(String, Arc<Embedder>, bool), ResponseError> { ) -> Result<(String, Arc<Embedder>, bool), ResponseError> {
let rtxn = index.read_txn()?; let rtxn = index.read_txn()?;
let embedder_configs = index.embedding_configs(&rtxn)?; let embedder_configs = index.embedding_configs().embedding_configs(&rtxn)?;
let embedders = index_scheduler.embedders(index_uid, embedder_configs)?; let embedders = index_scheduler.embedders(index_uid, embedder_configs)?;
let (embedder, _, quantized) = embedders let (embedder, quantized) = embedders
.get(embedder_name) .get(embedder_name)
.ok_or(match route { .ok_or(match route {
Route::Search | Route::MultiSearch => { Route::Search | Route::MultiSearch => {
@ -412,6 +412,7 @@ impl SearchKind {
milli::UserError::InvalidSimilarEmbedder(embedder_name.to_owned()) milli::UserError::InvalidSimilarEmbedder(embedder_name.to_owned())
} }
}) })
.map(|runtime| (runtime.embedder.clone(), runtime.is_quantized))
.map_err(milli::Error::from)?; .map_err(milli::Error::from)?;
if let Some(vector_len) = vector_len { if let Some(vector_len) = vector_len {
@ -1328,7 +1329,6 @@ struct HitMaker<'a> {
vectors_fid: Option<FieldId>, vectors_fid: Option<FieldId>,
retrieve_vectors: RetrieveVectors, retrieve_vectors: RetrieveVectors,
to_retrieve_ids: BTreeSet<FieldId>, to_retrieve_ids: BTreeSet<FieldId>,
embedding_configs: Vec<index::IndexEmbeddingConfig>,
formatter_builder: MatcherBuilder<'a>, formatter_builder: MatcherBuilder<'a>,
formatted_options: BTreeMap<FieldId, FormatOptions>, formatted_options: BTreeMap<FieldId, FormatOptions>,
show_ranking_score: bool, show_ranking_score: bool,
@ -1443,8 +1443,6 @@ impl<'a> HitMaker<'a> {
&displayed_ids, &displayed_ids,
); );
let embedding_configs = index.embedding_configs(rtxn)?;
Ok(Self { Ok(Self {
index, index,
rtxn, rtxn,
@ -1453,7 +1451,6 @@ impl<'a> HitMaker<'a> {
vectors_fid, vectors_fid,
retrieve_vectors, retrieve_vectors,
to_retrieve_ids, to_retrieve_ids,
embedding_configs,
formatter_builder, formatter_builder,
formatted_options, formatted_options,
show_ranking_score: format.show_ranking_score, show_ranking_score: format.show_ranking_score,
@ -1499,14 +1496,8 @@ impl<'a> HitMaker<'a> {
Some(Value::Object(map)) => map, Some(Value::Object(map)) => map,
_ => Default::default(), _ => Default::default(),
}; };
for (name, vector) in self.index.embeddings(self.rtxn, id)? { for (name, (vector, regenerate)) in self.index.embeddings(self.rtxn, id)? {
let user_provided = self let embeddings = ExplicitVectors { embeddings: Some(vector.into()), regenerate };
.embedding_configs
.iter()
.find(|conf| conf.name == name)
.is_some_and(|conf| conf.user_provided.contains(id));
let embeddings =
ExplicitVectors { embeddings: Some(vector.into()), regenerate: !user_provided };
vectors.insert( vectors.insert(
name, name,
serde_json::to_value(embeddings).map_err(InternalError::SerdeJson)?, serde_json::to_value(embeddings).map_err(InternalError::SerdeJson)?,

10
crates/meilitool/src/main.rs
View file

@ -545,7 +545,6 @@ fn export_documents(
let rtxn = index.read_txn()?; let rtxn = index.read_txn()?;
let fields_ids_map = index.fields_ids_map(&rtxn)?; let fields_ids_map = index.fields_ids_map(&rtxn)?;
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let embedding_configs = index.embedding_configs(&rtxn)?;
if let Some(offset) = offset { if let Some(offset) = offset {
eprintln!("Skipping {offset} documents"); eprintln!("Skipping {offset} documents");
@ -592,17 +591,12 @@ fn export_documents(
.into()); .into());
}; };
for (embedder_name, embeddings) in embeddings { for (embedder_name, (embeddings, regenerate)) in embeddings {
let user_provided = embedding_configs
.iter()
.find(|conf| conf.name == embedder_name)
.is_some_and(|conf| conf.user_provided.contains(id));
let embeddings = ExplicitVectors { let embeddings = ExplicitVectors {
embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
embeddings, embeddings,
)), )),
regenerate: !user_provided, regenerate,
}; };
vectors vectors
.insert(embedder_name, serde_json::to_value(embeddings).unwrap()); .insert(embedder_name, serde_json::to_value(embeddings).unwrap());

4
crates/milli/src/search/new/tests/integration.rs
View file

@ -8,7 +8,7 @@ use maplit::{btreemap, hashset};
use crate::progress::Progress; use crate::progress::Progress;
use crate::update::new::indexer; use crate::update::new::indexer;
use crate::update::{IndexerConfig, Settings}; use crate::update::{IndexerConfig, Settings};
use crate::vector::EmbeddingConfigs; use crate::vector::RuntimeEmbedders;
use crate::{db_snap, Criterion, FilterableAttributesRule, Index}; use crate::{db_snap, Criterion, FilterableAttributesRule, Index};
pub const CONTENT: &str = include_str!("../../../../tests/assets/test_set.ndjson"); pub const CONTENT: &str = include_str!("../../../../tests/assets/test_set.ndjson");
use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::constants::RESERVED_GEO_FIELD_NAME;
@ -55,7 +55,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let mut file = tempfile::tempfile().unwrap(); let mut file = tempfile::tempfile().unwrap();

4
crates/milli/src/search/new/vector_sort.rs
View file

@ -32,8 +32,8 @@ impl<Q: RankingRuleQueryTrait> VectorSort<Q> {
) -> Result<Self> { ) -> Result<Self> {
let embedder_index = ctx let embedder_index = ctx
.index .index
.embedder_category_id .embedding_configs()
.get(ctx.txn, embedder_name)? .embedder_id(ctx.txn, embedder_name)?
.ok_or_else(|| crate::UserError::InvalidSearchEmbedder(embedder_name.to_owned()))?; .ok_or_else(|| crate::UserError::InvalidSearchEmbedder(embedder_name.to_owned()))?;
Ok(Self { Ok(Self {

11
crates/milli/src/search/similar.rs
View file

@ -64,10 +64,13 @@ impl<'a> Similar<'a> {
let universe = universe; let universe = universe;
let embedder_index = let embedder_index = self
self.index.embedder_category_id.get(self.rtxn, &self.embedder_name)?.ok_or_else( .index
|| crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()), .embedding_configs()
)?; .embedder_id(self.rtxn, &self.embedder_name)?
.ok_or_else(|| {
crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned())
})?;
let reader = ArroyWrapper::new(self.index.vector_arroy, embedder_index, self.quantized); let reader = ArroyWrapper::new(self.index.vector_arroy, embedder_index, self.quantized);
let results = reader.nns_by_item( let results = reader.nns_by_item(

4
crates/milli/src/test_index.rs
View file

@ -18,7 +18,7 @@ use crate::update::{
self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings, self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings,
}; };
use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; use crate::vector::settings::{EmbedderSource, EmbeddingSettings};
use crate::vector::EmbeddingConfigs; use crate::vector::RuntimeEmbedders;
use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult};
pub(crate) struct TempIndex { pub(crate) struct TempIndex {
@ -223,7 +223,7 @@ fn aborting_indexation() {
let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let payload = documents!([ let payload = documents!([
{ "id": 1, "name": "kevin" }, { "id": 1, "name": "kevin" },

4
crates/milli/tests/search/facet_distribution.rs
View file

@ -5,7 +5,7 @@ use milli::documents::mmap_from_objects;
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{FacetDistribution, FilterableAttributesRule, Index, Object, OrderBy}; use milli::{FacetDistribution, FilterableAttributesRule, Index, Object, OrderBy};
use serde_json::{from_value, json}; use serde_json::{from_value, json};
@ -35,7 +35,7 @@ fn test_facet_distribution_with_no_facet_values() {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let doc1: Object = from_value( let doc1: Object = from_value(

4
crates/milli/tests/search/mod.rs
View file

@ -10,7 +10,7 @@ use maplit::{btreemap, hashset};
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{ use milli::{
AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, TermsMatchingStrategy, AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, TermsMatchingStrategy,
}; };
@ -74,7 +74,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let mut file = tempfile::tempfile().unwrap(); let mut file = tempfile::tempfile().unwrap();

4
crates/milli/tests/search/query_criteria.rs
View file

@ -8,7 +8,7 @@ use maplit::hashset;
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{AscDesc, Criterion, Index, Member, Search, SearchResult, TermsMatchingStrategy}; use milli::{AscDesc, Criterion, Index, Member, Search, SearchResult, TermsMatchingStrategy};
use rand::Rng; use rand::Rng;
use Criterion::*; use Criterion::*;
@ -288,7 +288,7 @@ fn criteria_ascdesc() {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
let mut file = tempfile::tempfile().unwrap(); let mut file = tempfile::tempfile().unwrap();

4
crates/milli/tests/search/typo_tolerance.rs
View file

@ -6,7 +6,7 @@ use milli::documents::mmap_from_objects;
use milli::progress::Progress; use milli::progress::Progress;
use milli::update::new::indexer; use milli::update::new::indexer;
use milli::update::{IndexerConfig, Settings}; use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs; use milli::vector::RuntimeEmbedders;
use milli::{Criterion, Index, Object, Search, TermsMatchingStrategy}; use milli::{Criterion, Index, Object, Search, TermsMatchingStrategy};
use serde_json::from_value; use serde_json::from_value;
use tempfile::tempdir; use tempfile::tempdir;
@ -123,7 +123,7 @@ fn test_typo_disabled_on_word() {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default(); let embedders = RuntimeEmbedders::default();
let mut indexer = indexer::DocumentOperation::new(); let mut indexer = indexer::DocumentOperation::new();
indexer.replace_documents(&documents).unwrap(); indexer.replace_documents(&documents).unwrap();