searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

fix the regeneration of the embeddings in the search

Browse Source
This commit is contained in:
Tamo 2024-06-05 16:10:56 +02:00
parent d85ab23b82
commit 31a793d226
7 changed files with 281 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
index-scheduler/src/lib.rs
View File

@ -5366,7 +5366,7 @@ mod tests {
template: "{{doc.doggo}}",
},
},
user_defined: RoaringBitmap<[1, 2]>,
user_provided: RoaringBitmap<[1, 2]>,
},
]
"###);

12
meilisearch/src/search.rs
View File

@ -15,6 +15,7 @@ use meilisearch_types::error::{Code, ResponseError};
use meilisearch_types::heed::RoTxn;
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
use meilisearch_types::milli::vector::Embedder;
use meilisearch_types::milli::{FacetValueHit, OrderBy, SearchForFacetValues, TimeBudget};
use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
@ -1066,18 +1067,13 @@ fn make_hits(
if retrieve_vectors {
let mut vectors = serde_json::Map::new();
for (name, mut vector) in index.embeddings(rtxn, id)? {
for (name, vector) in index.embeddings(rtxn, id)? {
let user_provided = embedding_configs
.iter()
.find(|conf| conf.name == name)
.is_some_and(|conf| conf.user_provided.contains(id));
let mut embedding = serde_json::Map::new();
embedding.insert("userProvided".to_string(), user_provided.into());
match vector.as_mut_slice() {
[one] => embedding.insert("embedding".to_string(), std::mem::take(one).into()),
_ => embedding.insert("embedding".to_string(), vector.into()),
};
vectors.insert(name, embedding.into());
let embeddings = ExplicitVectors { embeddings: vector.into(), user_provided };
vectors.insert(name, serde_json::to_value(embeddings)?);
}
document.insert("_vectors".into(), vectors.into());
}

25
meilisearch/tests/dumps/mod.rs
View File

@ -1940,8 +1940,9 @@ async fn import_dump_v6_containing_experimental_features() {
}
// In this test we must generate the dump ourselves to ensure the
// `user defined` vectors are well set
// `user provided` vectors are well set
#[actix_rt::test]
#[cfg_attr(target_os = "windows", ignore)]
async fn generate_and_import_dump_containing_vectors() {
let temp = tempfile::tempdir().unwrap();
let mut opt = default_settings(temp.path());
@ -2087,15 +2088,15 @@ async fn generate_and_import_dump_containing_vectors() {
index
.search(json!({"retrieveVectors": true}), |response, code| {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embedding" => "[vector]" }), @r###"
snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }), @r###"
[
{
"id": 0,
"doggo": "kefir",
"_vectors": {
"doggo_embedder": {
"userDefined": true,
"embedding": "[vector]"
"embeddings": "[vector]",
"userProvided": true
}
}
},
@ -2104,8 +2105,8 @@ async fn generate_and_import_dump_containing_vectors() {
"doggo": "echo",
"_vectors": {
"doggo_embedder": {
"userDefined": true,
"embedding": "[vector]"
"embeddings": "[vector]",
"userProvided": true
}
}
},
@ -2114,8 +2115,8 @@ async fn generate_and_import_dump_containing_vectors() {
"doggo": "intel",
"_vectors": {
"doggo_embedder": {
"userDefined": false,
"embedding": "[vector]"
"embeddings": "[vector]",
"userProvided": false
}
}
},
@ -2124,8 +2125,8 @@ async fn generate_and_import_dump_containing_vectors() {
"doggo": "bill",
"_vectors": {
"doggo_embedder": {
"userDefined": false,
"embedding": "[vector]"
"embeddings": "[vector]",
"userProvided": false
}
}
},
@ -2134,8 +2135,8 @@ async fn generate_and_import_dump_containing_vectors() {
"doggo": "max",
"_vectors": {
"doggo_embedder": {
"userDefined": false,
"embedding": "[vector]"
"embeddings": "[vector]",
"userProvided": false
}
}
}

30
meilisearch/tests/search/hybrid.rs
View File

@ -128,7 +128,7 @@ async fn simple_search() {
)
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}}}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}}}]"###);
snapshot!(response["semanticHitCount"], @"0");
let (response, code) = index
@ -137,7 +137,7 @@ async fn simple_search() {
)
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["semanticHitCount"], @"2");
let (response, code) = index
@ -146,7 +146,7 @@ async fn simple_search() {
)
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["semanticHitCount"], @"3");
}
@ -207,7 +207,7 @@ async fn distribution_shift() {
let search = json!({"q": "Captain", "vector": [1.0, 1.0], "showRankingScore": true, "hybrid": {"semanticRatio": 1.0}, "retrieveVectors": true});
let (response, code) = index.search_post(search.clone()).await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9472135901451112}]"###);
let (response, code) = index
.update_settings(json!({
@ -228,7 +228,7 @@ async fn distribution_shift() {
let (response, code) = index.search_post(search).await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.1920928955078125e-7}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.1920928955078125e-7}]"###);
}
#[actix_rt::test]
@ -249,7 +249,7 @@ async fn highlighter() {
}))
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"}}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"}}]"###);
snapshot!(response["semanticHitCount"], @"0");
let (response, code) = index
@ -265,7 +265,7 @@ async fn highlighter() {
}))
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["semanticHitCount"], @"3");
// no highlighting on full semantic
@ -282,7 +282,7 @@ async fn highlighter() {
}))
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
snapshot!(response["semanticHitCount"], @"3");
}
@ -370,7 +370,7 @@ async fn single_document() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0}"###);
snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0}"###);
snapshot!(response["semanticHitCount"], @"1");
}
@ -385,7 +385,7 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###);
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###);
snapshot!(response["semanticHitCount"], @"null");
// same with a different semantic ratio
@ -394,7 +394,7 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###);
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###);
snapshot!(response["semanticHitCount"], @"null");
// wrong vector dimensions
@ -418,7 +418,7 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.6581138968467712}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.6581138968467712}]"###);
snapshot!(response["semanticHitCount"], @"3");
// full keyword, without a query
@ -427,7 +427,7 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":1.0}]"###);
snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":1.0}]"###);
snapshot!(response["semanticHitCount"], @"null");
// query + vector, full keyword => keyword
@ -436,7 +436,7 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"userDefined":true,"embedding":[2.0,3.0]}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"userDefined":true,"embedding":[1.0,3.0]}},"_rankingScore":0.9242424242424242}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"userProvided":true}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"userProvided":true}},"_rankingScore":0.9242424242424242}]"###);
snapshot!(response["semanticHitCount"], @"null");
// query + vector, no hybrid keyword =>
@ -479,6 +479,6 @@ async fn query_combination() {
.await;
snapshot!(code, @"200 OK");
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"userDefined":true,"embedding":[1.0,2.0]}},"_rankingScore":0.9242424242424242}]"###);
snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"userProvided":true}},"_rankingScore":0.9242424242424242}]"###);
snapshot!(response["semanticHitCount"], @"0");
}

30
meilisearch/tests/search/mod.rs
View File

@ -1350,12 +1350,14 @@ async fn experimental_feature_vector_store() {
"id": "287947",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
1.0,
2.0,
3.0
]
],
"userProvided": true
}
},
"_rankingScore": 1.0
@ -1365,12 +1367,14 @@ async fn experimental_feature_vector_store() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
1.0,
2.0,
54.0
]
],
"userProvided": true
}
},
"_rankingScore": 0.9129111766815186
@ -1380,12 +1384,14 @@ async fn experimental_feature_vector_store() {
"id": "450465",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
-100.0,
340.0,
90.0
]
],
"userProvided": true
}
},
"_rankingScore": 0.8106412887573242
@ -1395,12 +1401,14 @@ async fn experimental_feature_vector_store() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
-100.0,
231.0,
32.0
]
],
"userProvided": true
}
},
"_rankingScore": 0.7412010431289673
@ -1410,12 +1418,14 @@ async fn experimental_feature_vector_store() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
10.0,
-23.0,
32.0
]
],
"userProvided": true
}
},
"_rankingScore": 0.6972063183784485

144
meilisearch/tests/similar/mod.rs
View File

@ -88,12 +88,14 @@ async fn basic() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
}
},
@ -103,12 +105,14 @@ async fn basic() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
}
},
@ -118,12 +122,14 @@ async fn basic() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.699999988079071,
0.699999988079071,
-0.4000000059604645
]
],
"userProvided": true
}
}
},
@ -133,12 +139,14 @@ async fn basic() {
"id": "287947",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.800000011920929,
0.4000000059604645,
-0.5
]
],
"userProvided": true
}
}
}
@ -158,12 +166,14 @@ async fn basic() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.699999988079071,
0.699999988079071,
-0.4000000059604645
]
],
"userProvided": true
}
}
},
@ -173,12 +183,14 @@ async fn basic() {
"id": "287947",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.800000011920929,
0.4000000059604645,
-0.5
]
],
"userProvided": true
}
}
},
@ -188,12 +200,14 @@ async fn basic() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
}
},
@ -203,12 +217,14 @@ async fn basic() {
"id": "143",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
-0.5,
0.30000001192092896,
0.8500000238418579
]
],
"userProvided": true
}
}
}
@ -264,12 +280,14 @@ async fn ranking_score_threshold() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
},
"_rankingScore": 0.890957772731781
@ -280,12 +298,14 @@ async fn ranking_score_threshold() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
},
"_rankingScore": 0.39060014486312866
@ -296,12 +316,14 @@ async fn ranking_score_threshold() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.699999988079071,
0.699999988079071,
-0.4000000059604645
]
],
"userProvided": true
}
},
"_rankingScore": 0.2819308042526245
@ -312,12 +334,14 @@ async fn ranking_score_threshold() {
"id": "287947",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.800000011920929,
0.4000000059604645,
-0.5
]
],
"userProvided": true
}
},
"_rankingScore": 0.1662663221359253
@ -342,12 +366,14 @@ async fn ranking_score_threshold() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
},
"_rankingScore": 0.890957772731781
@ -358,12 +384,14 @@ async fn ranking_score_threshold() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
},
"_rankingScore": 0.39060014486312866
@ -374,12 +402,14 @@ async fn ranking_score_threshold() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.699999988079071,
0.699999988079071,
-0.4000000059604645
]
],
"userProvided": true
}
},
"_rankingScore": 0.2819308042526245
@ -404,12 +434,14 @@ async fn ranking_score_threshold() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
},
"_rankingScore": 0.890957772731781
@ -420,12 +452,14 @@ async fn ranking_score_threshold() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
},
"_rankingScore": 0.39060014486312866
@ -450,12 +484,14 @@ async fn ranking_score_threshold() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
},
"_rankingScore": 0.890957772731781
@ -522,12 +558,14 @@ async fn filter() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
}
},
@ -537,12 +575,14 @@ async fn filter() {
"id": "166428",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.699999988079071,
0.699999988079071,
-0.4000000059604645
]
],
"userProvided": true
}
}
},
@ -552,12 +592,14 @@ async fn filter() {
"id": "287947",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.800000011920929,
0.4000000059604645,
-0.5
]
],
"userProvided": true
}
}
}
@ -580,12 +622,14 @@ async fn filter() {
"id": "143",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
-0.5,
0.30000001192092896,
0.8500000238418579
]
],
"userProvided": true
}
}
}
@ -639,12 +683,14 @@ async fn limit_and_offset() {
"id": "522681",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.10000000149011612,
0.6000000238418579,
0.800000011920929
]
],
"userProvided": true
}
}
}
@ -666,12 +712,14 @@ async fn limit_and_offset() {
"id": "299537",
"_vectors": {
"manual": {
"userDefined": true,
"embedding": [
"embeddings": [
[
0.6000000238418579,
0.800000011920929,
-0.20000000298023224
]
],
"userProvided": true
}
}
}

16
milli/src/vector/parsed_vectors.rs
View File

@ -160,6 +160,22 @@ impl VectorOrArrayOfVectors {
pub fn from_array_of_vectors(array_of_vec: Vec<Embedding>) -> Self {
Self { inner: Some(either::Either::Left(array_of_vec)) }
}
pub fn from_vector(vec: Embedding) -> Self {
Self { inner: Some(either::Either::Right(vec)) }
}
}
impl From<Embedding> for VectorOrArrayOfVectors {
fn from(vec: Embedding) -> Self {
Self::from_vector(vec)
}
}
impl From<Vec<Embedding>> for VectorOrArrayOfVectors {
fn from(vec: Vec<Embedding>) -> Self {
Self::from_array_of_vectors(vec)
}
}
#[cfg(test)]