From 31a793d226154dcecff10f8e761582b775665dd5 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 5 Jun 2024 16:10:56 +0200 Subject: [PATCH] fix the regeneration of the embeddings in the search --- index-scheduler/src/lib.rs | 2 +- meilisearch/src/search.rs | 12 +- meilisearch/tests/dumps/mod.rs | 25 +-- meilisearch/tests/search/hybrid.rs | 30 +-- meilisearch/tests/search/mod.rs | 70 +++--- meilisearch/tests/similar/mod.rs | 336 ++++++++++++++++------------- milli/src/vector/parsed_vectors.rs | 16 ++ 7 files changed, 281 insertions(+), 210 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 8d6237408..de263c50d 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -5366,7 +5366,7 @@ mod tests { template: "{{doc.doggo}}", }, }, - user_defined: RoaringBitmap<[1, 2]>, + user_provided: RoaringBitmap<[1, 2]>, }, ] "###); diff --git a/meilisearch/src/search.rs b/meilisearch/src/search.rs index a0c05b09e..ce712f17f 100644 --- a/meilisearch/src/search.rs +++ b/meilisearch/src/search.rs @@ -15,6 +15,7 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy}; +use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; use meilisearch_types::milli::vector::Embedder; use meilisearch_types::milli::{FacetValueHit, OrderBy, SearchForFacetValues, TimeBudget}; use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS; @@ -1066,18 +1067,13 @@ fn make_hits( if retrieve_vectors { let mut vectors = serde_json::Map::new(); - for (name, mut vector) in index.embeddings(rtxn, id)? { + for (name, vector) in index.embeddings(rtxn, id)? { let user_provided = embedding_configs .iter() .find(|conf| conf.name == name) .is_some_and(|conf| conf.user_provided.contains(id)); - let mut embedding = serde_json::Map::new(); - embedding.insert("userProvided".to_string(), user_provided.into()); - match vector.as_mut_slice() { - [one] => embedding.insert("embedding".to_string(), std::mem::take(one).into()), - _ => embedding.insert("embedding".to_string(), vector.into()), - }; - vectors.insert(name, embedding.into()); + let embeddings = ExplicitVectors { embeddings: vector.into(), user_provided }; + vectors.insert(name, serde_json::to_value(embeddings)?); } document.insert("_vectors".into(), vectors.into()); } diff --git a/meilisearch/tests/dumps/mod.rs b/meilisearch/tests/dumps/mod.rs index b657fc1ee..6f93d94a7 100644 --- a/meilisearch/tests/dumps/mod.rs +++ b/meilisearch/tests/dumps/mod.rs @@ -1940,8 +1940,9 @@ async fn import_dump_v6_containing_experimental_features() { } // In this test we must generate the dump ourselves to ensure the -// `user defined` vectors are well set +// `user provided` vectors are well set #[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] async fn generate_and_import_dump_containing_vectors() { let temp = tempfile::tempdir().unwrap(); let mut opt = default_settings(temp.path()); @@ -2087,15 +2088,15 @@ async fn generate_and_import_dump_containing_vectors() { index .search(json!({"retrieveVectors": true}), |response, code| { snapshot!(code, @"200 OK"); - snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embedding" => "[vector]" }), @r###" + snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }), @r###" [ { "id": 0, "doggo": "kefir", "_vectors": { "doggo_embedder": { - "userDefined": true, - "embedding": "[vector]" + "embeddings": "[vector]", + "userProvided": true } } }, @@ -2104,8 +2105,8 @@ async fn generate_and_import_dump_containing_vectors() { "doggo": "echo", "_vectors": { "doggo_embedder": { - "userDefined": true, - "embedding": "[vector]" + "embeddings": "[vector]", + "userProvided": true } } }, @@ -2114,8 +2115,8 @@ async fn generate_and_import_dump_containing_vectors() { "doggo": "intel", "_vectors": { "doggo_embedder": { - "userDefined": false, - "embedding": "[vector]" + "embeddings": "[vector]", + "userProvided": false } } }, @@ -2124,8 +2125,8 @@ async fn generate_and_import_dump_containing_vectors() { "doggo": "bill", "_vectors": { "doggo_embedder": { - "userDefined": false, - "embedding": "[vector]" + "embeddings": "[vector]", + "userProvided": false } } }, @@ -2134,8 +2135,8 @@ async fn generate_and_import_dump_containing_vectors() { "doggo": "max", "_vectors": { "doggo_embedder": { - "userDefined": false, - "embedding": "[vector]" + "embeddings": "[vector]", + "userProvided": false } } } diff --git a/meilisearch/tests/search/hybrid.rs b/meilisearch/tests/search/hybrid.rs index 713dbe3bb..b8a4110ad 100644 --- a/meilisearch/tests/search/hybrid.rs +++ b/meilisearch/tests/search/hybrid.rs @@ -128,7 +128,7 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}}}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}}}]"###); snapshot!(response["semanticHitCount"], @"0"); let (response, code) = index @@ -137,7 +137,7 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###); snapshot!(response["semanticHitCount"], @"2"); let (response, code) = index @@ -146,7 +146,7 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###); snapshot!(response["semanticHitCount"], @"3"); } @@ -207,7 +207,7 @@ async fn distribution_shift() { let search = json!({"q": "Captain", "vector": [1.0, 1.0], "showRankingScore": true, "hybrid": {"semanticRatio": 1.0}, "retrieveVectors": true}); let (response, code) = index.search_post(search.clone()).await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###); let (response, code) = index .update_settings(json!({ @@ -228,7 +228,7 @@ async fn distribution_shift() { let (response, code) = index.search_post(search).await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.1920928955078125e-7}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.1920928955078125e-7}]"###); } #[actix_rt::test] @@ -249,7 +249,7 @@ async fn highlighter() { })) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"}}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"}}]"###); snapshot!(response["semanticHitCount"], @"0"); let (response, code) = index @@ -265,7 +265,7 @@ async fn highlighter() { })) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###); snapshot!(response["semanticHitCount"], @"3"); // no highlighting on full semantic @@ -282,7 +282,7 @@ async fn highlighter() { })) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###); snapshot!(response["semanticHitCount"], @"3"); } @@ -370,7 +370,7 @@ async fn single_document() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0}"###); + snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0}"###); snapshot!(response["semanticHitCount"], @"1"); } @@ -385,7 +385,7 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###); + snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###); snapshot!(response["semanticHitCount"], @"null"); // same with a different semantic ratio @@ -394,7 +394,7 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###); + snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###); snapshot!(response["semanticHitCount"], @"null"); // wrong vector dimensions @@ -418,7 +418,7 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.6581138968467712}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.6581138968467712}]"###); snapshot!(response["semanticHitCount"], @"3"); // full keyword, without a query @@ -427,7 +427,7 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###); + snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###); snapshot!(response["semanticHitCount"], @"null"); // query + vector, full keyword => keyword @@ -436,7 +436,7 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9242424242424242}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9242424242424242}]"###); snapshot!(response["semanticHitCount"], @"null"); // query + vector, no hybrid keyword => @@ -479,6 +479,6 @@ async fn query_combination() { .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9242424242424242}]"###); + snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9242424242424242}]"###); snapshot!(response["semanticHitCount"], @"0"); } diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 2a2b23fd5..9e19fa4e8 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -1350,12 +1350,14 @@ async fn experimental_feature_vector_store() { "id": "287947", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 1.0, - 2.0, - 3.0 - ] + "embeddings": [ + [ + 1.0, + 2.0, + 3.0 + ] + ], + "userProvided": true } }, "_rankingScore": 1.0 @@ -1365,12 +1367,14 @@ async fn experimental_feature_vector_store() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 1.0, - 2.0, - 54.0 - ] + "embeddings": [ + [ + 1.0, + 2.0, + 54.0 + ] + ], + "userProvided": true } }, "_rankingScore": 0.9129111766815186 @@ -1380,12 +1384,14 @@ async fn experimental_feature_vector_store() { "id": "450465", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - -100.0, - 340.0, - 90.0 - ] + "embeddings": [ + [ + -100.0, + 340.0, + 90.0 + ] + ], + "userProvided": true } }, "_rankingScore": 0.8106412887573242 @@ -1395,12 +1401,14 @@ async fn experimental_feature_vector_store() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - -100.0, - 231.0, - 32.0 - ] + "embeddings": [ + [ + -100.0, + 231.0, + 32.0 + ] + ], + "userProvided": true } }, "_rankingScore": 0.7412010431289673 @@ -1410,12 +1418,14 @@ async fn experimental_feature_vector_store() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 10.0, - -23.0, - 32.0 - ] + "embeddings": [ + [ + 10.0, + -23.0, + 32.0 + ] + ], + "userProvided": true } }, "_rankingScore": 0.6972063183784485 diff --git a/meilisearch/tests/similar/mod.rs b/meilisearch/tests/similar/mod.rs index 2b70b3df5..0a568553c 100644 --- a/meilisearch/tests/similar/mod.rs +++ b/meilisearch/tests/similar/mod.rs @@ -88,12 +88,14 @@ async fn basic() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } } }, @@ -103,12 +105,14 @@ async fn basic() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } } }, @@ -118,12 +122,14 @@ async fn basic() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.699999988079071, - 0.699999988079071, - -0.4000000059604645 - ] + "embeddings": [ + [ + 0.699999988079071, + 0.699999988079071, + -0.4000000059604645 + ] + ], + "userProvided": true } } }, @@ -133,12 +139,14 @@ async fn basic() { "id": "287947", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.800000011920929, - 0.4000000059604645, - -0.5 - ] + "embeddings": [ + [ + 0.800000011920929, + 0.4000000059604645, + -0.5 + ] + ], + "userProvided": true } } } @@ -158,12 +166,14 @@ async fn basic() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.699999988079071, - 0.699999988079071, - -0.4000000059604645 - ] + "embeddings": [ + [ + 0.699999988079071, + 0.699999988079071, + -0.4000000059604645 + ] + ], + "userProvided": true } } }, @@ -173,12 +183,14 @@ async fn basic() { "id": "287947", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.800000011920929, - 0.4000000059604645, - -0.5 - ] + "embeddings": [ + [ + 0.800000011920929, + 0.4000000059604645, + -0.5 + ] + ], + "userProvided": true } } }, @@ -188,12 +200,14 @@ async fn basic() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } } }, @@ -203,12 +217,14 @@ async fn basic() { "id": "143", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - -0.5, - 0.30000001192092896, - 0.8500000238418579 - ] + "embeddings": [ + [ + -0.5, + 0.30000001192092896, + 0.8500000238418579 + ] + ], + "userProvided": true } } } @@ -264,12 +280,14 @@ async fn ranking_score_threshold() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } }, "_rankingScore": 0.890957772731781 @@ -280,12 +298,14 @@ async fn ranking_score_threshold() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } }, "_rankingScore": 0.39060014486312866 @@ -296,12 +316,14 @@ async fn ranking_score_threshold() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.699999988079071, - 0.699999988079071, - -0.4000000059604645 - ] + "embeddings": [ + [ + 0.699999988079071, + 0.699999988079071, + -0.4000000059604645 + ] + ], + "userProvided": true } }, "_rankingScore": 0.2819308042526245 @@ -312,12 +334,14 @@ async fn ranking_score_threshold() { "id": "287947", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.800000011920929, - 0.4000000059604645, - -0.5 - ] + "embeddings": [ + [ + 0.800000011920929, + 0.4000000059604645, + -0.5 + ] + ], + "userProvided": true } }, "_rankingScore": 0.1662663221359253 @@ -342,12 +366,14 @@ async fn ranking_score_threshold() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } }, "_rankingScore": 0.890957772731781 @@ -358,12 +384,14 @@ async fn ranking_score_threshold() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } }, "_rankingScore": 0.39060014486312866 @@ -374,12 +402,14 @@ async fn ranking_score_threshold() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.699999988079071, - 0.699999988079071, - -0.4000000059604645 - ] + "embeddings": [ + [ + 0.699999988079071, + 0.699999988079071, + -0.4000000059604645 + ] + ], + "userProvided": true } }, "_rankingScore": 0.2819308042526245 @@ -404,12 +434,14 @@ async fn ranking_score_threshold() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } }, "_rankingScore": 0.890957772731781 @@ -420,12 +452,14 @@ async fn ranking_score_threshold() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } }, "_rankingScore": 0.39060014486312866 @@ -450,12 +484,14 @@ async fn ranking_score_threshold() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } }, "_rankingScore": 0.890957772731781 @@ -522,12 +558,14 @@ async fn filter() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } } }, @@ -537,12 +575,14 @@ async fn filter() { "id": "166428", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.699999988079071, - 0.699999988079071, - -0.4000000059604645 - ] + "embeddings": [ + [ + 0.699999988079071, + 0.699999988079071, + -0.4000000059604645 + ] + ], + "userProvided": true } } }, @@ -552,12 +592,14 @@ async fn filter() { "id": "287947", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.800000011920929, - 0.4000000059604645, - -0.5 - ] + "embeddings": [ + [ + 0.800000011920929, + 0.4000000059604645, + -0.5 + ] + ], + "userProvided": true } } } @@ -580,12 +622,14 @@ async fn filter() { "id": "143", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - -0.5, - 0.30000001192092896, - 0.8500000238418579 - ] + "embeddings": [ + [ + -0.5, + 0.30000001192092896, + 0.8500000238418579 + ] + ], + "userProvided": true } } } @@ -639,12 +683,14 @@ async fn limit_and_offset() { "id": "522681", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.10000000149011612, - 0.6000000238418579, - 0.800000011920929 - ] + "embeddings": [ + [ + 0.10000000149011612, + 0.6000000238418579, + 0.800000011920929 + ] + ], + "userProvided": true } } } @@ -666,12 +712,14 @@ async fn limit_and_offset() { "id": "299537", "_vectors": { "manual": { - "userDefined": true, - "embedding": [ - 0.6000000238418579, - 0.800000011920929, - -0.20000000298023224 - ] + "embeddings": [ + [ + 0.6000000238418579, + 0.800000011920929, + -0.20000000298023224 + ] + ], + "userProvided": true } } } diff --git a/milli/src/vector/parsed_vectors.rs b/milli/src/vector/parsed_vectors.rs index 4e9e60520..501bd2ad2 100644 --- a/milli/src/vector/parsed_vectors.rs +++ b/milli/src/vector/parsed_vectors.rs @@ -160,6 +160,22 @@ impl VectorOrArrayOfVectors { pub fn from_array_of_vectors(array_of_vec: Vec) -> Self { Self { inner: Some(either::Either::Left(array_of_vec)) } } + + pub fn from_vector(vec: Embedding) -> Self { + Self { inner: Some(either::Either::Right(vec)) } + } +} + +impl From for VectorOrArrayOfVectors { + fn from(vec: Embedding) -> Self { + Self::from_vector(vec) + } +} + +impl From> for VectorOrArrayOfVectors { + fn from(vec: Vec) -> Self { + Self::from_array_of_vectors(vec) + } } #[cfg(test)]