From 8af76a65bfe4430075bded3ff95e8a343c46b2f1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 2 Jul 2025 18:04:44 +0200 Subject: [PATCH 01/24] Add test_fragment_indexing --- crates/meilisearch/tests/vector/fragments.rs | 198 +++++++++++++++++++ crates/meilisearch/tests/vector/mod.rs | 1 + 2 files changed, 199 insertions(+) create mode 100644 crates/meilisearch/tests/vector/fragments.rs diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs new file mode 100644 index 000000000..5f6b1095e --- /dev/null +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -0,0 +1,198 @@ +use std::collections::BTreeMap; + +use meili_snap::{json_string, snapshot}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +use crate::common::Value; +use crate::json; +use crate::vector::{get_server_vector, GetAllDocumentsOptions}; + +async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { + let mock_server = MockServer::start().await; + + let text_to_embedding: BTreeMap<_, _> = vec![ + ("kefir", [0.5, -0.5, 2.0]), + ("intel", [1.0, 1.0, 1.0]), + ("bulldog", [1.5, -2.5, 0.0]), + ("dustin", [-0.5, 0.5, 2.5]), + ("labrador", [-3.5, 0.5, -1.0]), + ] + .into_iter() + .collect(); + + Mock::given(method("POST")) + .and(path("/")) + .respond_with(move |req: &Request| { + let text = String::from_utf8_lossy(&req.body).to_string(); + let mut data = [0.0, 0.0, 0.0]; + for (inner_text, inner_data) in &text_to_embedding { + if text.contains(inner_text) { + for (i, &value) in inner_data.iter().enumerate() { + data[i] += value; + } + } + } + ResponseTemplate::new(200).set_body_json( + json!({ "data": data }) + ) + }) + .mount(&mock_server) + .await; + let url = mock_server.uri(); + + let embedder_settings = json!({ + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": indexing_fragments, + "searchFragments": search_fragments, + "documentTemplate": "document template: {{dog.name}}", + }); + + (mock_server, embedder_settings) +} + + +#[actix_rt::test] +async fn test_fragment_indexing() { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }) + ).await; + let server = get_server_vector().await; + let index = server.index("doggo"); + + // Enable the experimental feature + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + println!("[task] {:?}", task); + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Make sure the documents have been indexed and their embeddings retrieved + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 2.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ], + [ + -2.5, + 1.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 2.5 + ], + [ + 1.0, + -2.0, + 2.5 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 98555dfac..837c34289 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -4,6 +4,7 @@ mod ollama; mod openai; mod rest; mod settings; +mod fragments; use std::str::FromStr; From 65ba7b47af77015d1baab22ef61e2693c13cc74c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 10:43:27 +0200 Subject: [PATCH 02/24] Test search fragments --- crates/meilisearch/tests/vector/fragments.rs | 169 ++++++++++++++++++- 1 file changed, 164 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 5f6b1095e..876e18ffe 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -12,11 +12,11 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc let mock_server = MockServer::start().await; let text_to_embedding: BTreeMap<_, _> = vec![ - ("kefir", [0.5, -0.5, 2.0]), - ("intel", [1.0, 1.0, 1.0]), - ("bulldog", [1.5, -2.5, 0.0]), - ("dustin", [-0.5, 0.5, 2.5]), - ("labrador", [-3.5, 0.5, -1.0]), + ("kefir", [0.5, -0.5, 0.0]), + ("intel", [1.0, 1.0, 0.0]), + ("dustin", [-0.5, 0.5, 0.0]), + ("bulldog", [0.0, 0.0, 1.0]), + ("labrador", [0.0, 0.0, -1.0]), ] .into_iter() .collect(); @@ -196,3 +196,162 @@ async fn test_fragment_indexing() { "#); } +#[actix_rt::test] +async fn test_search_fragments() { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + }) + ).await; + let server = get_server_vector().await; + let index = server.index("doggo"); + + // Enable the experimental feature + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Perform a search with a provided vector + let (value, code) = index.search_post( + json!({"vector": [1.0, 1.0, 1.0], "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 1, + "name": "echo" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); + + // Perform a search with some media + let (value, code) = index.search_post( + json!({ + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 2, + "name": "intel", + "breed": "labrador" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); + + // Perform a search that matches multiple media + let (value, code) = index.search_post( + json!({ + "media": { "name": "dustin", "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Error while generating embeddings: user error: Query matches multiple search fragments.\n - Note: First matched fragment `justBreed`.\n - Note: Second matched fragment `justName`.\n - Note: {\"q\":null,\"media\":{\"name\":\"dustin\",\"breed\":\"labrador\"}}", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); + + // Perform a search that matches no media + let (value, code) = index.search_post( + json!({ + "media": { "ticker": "GME", "section": "portfolio" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Error while generating embeddings: user error: Query matches no search fragment.\n - Note: {\"q\":null,\"media\":{\"ticker\":\"GME\",\"section\":\"portfolio\"}}", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); + + // Perform a search with a query media + let (value, code) = index.search_post( + json!({ + "q": "bulldog", + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 3, + "name": "dustin", + "breed": "bulldog" + } + ], + "query": "bulldog", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); +} + From 0b89ef1fd7c11c2909bc906911394bdc9ffd10fc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 11:24:09 +0200 Subject: [PATCH 03/24] Make tests use a shared index --- crates/meilisearch/tests/common/server.rs | 4 +- crates/meilisearch/tests/vector/fragments.rs | 261 +++++++++---------- 2 files changed, 128 insertions(+), 137 deletions(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 4367650c5..e3839855b 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -35,7 +35,7 @@ pub struct Server { pub static TEST_TEMP_DIR: Lazy = Lazy::new(|| TempDir::new().unwrap()); impl Server { - fn into_shared(self) -> Server { + pub fn into_shared(self) -> Server { Server { service: self.service, _dir: self._dir, _marker: PhantomData } } @@ -327,7 +327,7 @@ impl Server { self.service.get(url).await } - pub(super) fn _index(&self, uid: impl AsRef) -> Index<'_> { + pub fn _index(&self, uid: impl AsRef) -> Index<'_> { Index { uid: uid.as_ref().to_string(), service: &self.service, diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 876e18ffe..027a4069d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1,13 +1,73 @@ use std::collections::BTreeMap; use meili_snap::{json_string, snapshot}; +use tokio::sync::OnceCell; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, Request, ResponseTemplate}; +use crate::common::index::Index; +use crate::common::Shared; use crate::common::Value; use crate::json; +use crate::vector::Server; use crate::vector::{get_server_vector, GetAllDocumentsOptions}; +async fn shared_index_for_fragments() -> Index<'static, Shared> { + static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); + let (server, uid) = INDEX + .get_or_init(|| async { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + }), + ) + .await; + + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + let uid = index.uid.clone(); + (server.into_shared(), uid) + }) + .await; + server._index(uid).to_shared() +} + async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { let mock_server = MockServer::start().await; @@ -33,9 +93,7 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc } } } - ResponseTemplate::new(200).set_body_json( - json!({ "data": data }) - ) + ResponseTemplate::new(200).set_body_json(json!({ "data": data })) }) .mount(&mock_server) .await; @@ -57,52 +115,9 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc (mock_server, embedder_settings) } - #[actix_rt::test] -async fn test_fragment_indexing() { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }) - ).await; - let server = get_server_vector().await; - let index = server.index("doggo"); - - // Enable the experimental feature - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - println!("[task] {:?}", task); - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); +async fn indexing_fragments() { + let index = shared_index_for_fragments().await; // Make sure the documents have been indexed and their embeddings retrieved let (documents, code) = index @@ -121,7 +136,7 @@ async fn test_fragment_indexing() { [ 0.5, -0.5, - 2.0 + 0.0 ] ], "regenerate": true @@ -154,12 +169,12 @@ async fn test_fragment_indexing() { [ 1.0, 1.0, - 1.0 + 0.0 ], [ - -2.5, - 1.5, - 0.0 + 1.0, + 1.0, + -1.0 ] ], "regenerate": true @@ -176,12 +191,12 @@ async fn test_fragment_indexing() { [ -0.5, 0.5, - 2.5 + 0.0 ], [ - 1.0, - -2.0, - 2.5 + -0.5, + 0.5, + 1.0 ] ], "regenerate": true @@ -197,52 +212,9 @@ async fn test_fragment_indexing() { } #[actix_rt::test] -async fn test_search_fragments() { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "justBreed": {"value": "It's a {{ media.breed }}"}, - "justName": {"value": "{{ media.name }} is a dog"}, - "query": {"value": "Some pre-prompt for query {{ q }}"}, - }) - ).await; - let server = get_server_vector().await; - let index = server.index("doggo"); +async fn search_with_vector() { + let index = shared_index_for_fragments().await; - // Enable the experimental feature - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Perform a search with a provided vector let (value, code) = index.search_post( json!({"vector": [1.0, 1.0, 1.0], "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} )).await; @@ -263,15 +235,20 @@ async fn test_search_fragments() { "semanticHitCount": 1 } "#); +} - // Perform a search with some media - let (value, code) = index.search_post( - json!({ - "media": { "breed": "labrador" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"200 OK"); snapshot!(value, @r#" { @@ -290,15 +267,20 @@ async fn test_search_fragments() { "semanticHitCount": 1 } "#); +} - // Perform a search that matches multiple media - let (value, code) = index.search_post( - json!({ - "media": { "name": "dustin", "breed": "labrador" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media_matching_multiple_fragments() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "name": "dustin", "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -308,15 +290,20 @@ async fn test_search_fragments() { "link": "https://docs.meilisearch.com/errors#vector_embedding_error" } "#); +} - // Perform a search that matches no media - let (value, code) = index.search_post( - json!({ - "media": { "ticker": "GME", "section": "portfolio" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media_matching_no_fragment() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "ticker": "GME", "section": "portfolio" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -326,15 +313,20 @@ async fn test_search_fragments() { "link": "https://docs.meilisearch.com/errors#vector_embedding_error" } "#); +} - // Perform a search with a query media - let (value, code) = index.search_post( - json!({ - "q": "bulldog", - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_query() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "q": "bulldog", + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"200 OK"); snapshot!(value, @r#" { @@ -354,4 +346,3 @@ async fn test_search_fragments() { } "#); } - From b45eea0d3e101938ed7eaa58c839b754922504d4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 13:26:44 +0200 Subject: [PATCH 04/24] Add test for fragment deletion --- crates/meilisearch/tests/common/mod.rs | 2 +- crates/meilisearch/tests/vector/fragments.rs | 258 ++++++++++++++----- 2 files changed, 196 insertions(+), 64 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 1a73a7532..2fafbd11f 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,7 +3,7 @@ pub mod index; pub mod server; pub mod service; -use std::fmt::{self, Display}; +use std::{fmt::{self, Display}, future::Future}; #[allow(unused)] pub use index::GetAllDocumentsOptions; diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 027a4069d..c3fef8c79 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -6,69 +6,23 @@ use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::index::Index; -use crate::common::Shared; -use crate::common::Value; +use crate::common::{Owned, Shared}; use crate::json; use crate::vector::Server; -use crate::vector::{get_server_vector, GetAllDocumentsOptions}; +use crate::vector::GetAllDocumentsOptions; async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); let (server, uid) = INDEX .get_or_init(|| async { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "justBreed": {"value": "It's a {{ media.breed }}"}, - "justName": {"value": "{{ media.name }} is a dog"}, - "query": {"value": "Some pre-prompt for query {{ q }}"}, - }), - ) - .await; - - let server = Server::new().await; - let index = server.unique_index(); - - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - let uid = index.uid.clone(); + let (server, uid, _) = init_fragments_index().await; (server.into_shared(), uid) }) .await; server._index(uid).to_shared() } -async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { +pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { let mock_server = MockServer::start().await; let text_to_embedding: BTreeMap<_, _> = vec![ @@ -99,22 +53,62 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc .await; let url = mock_server.uri(); - let embedder_settings = json!({ - "source": "rest", - "url": url, - "dimensions": 3, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "indexingFragments": indexing_fragments, - "searchFragments": search_fragments, - "documentTemplate": "document template: {{dog.name}}", - }); + let server = Server::new().await; + let index = server.unique_index(); - (mock_server, embedder_settings) + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index + .update_settings(settings.clone()) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + let uid = index.uid.clone(); + (server, uid, settings) } +// TODO: Test cannot pass both fragments and document + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -346,3 +340,141 @@ async fn search_with_query() { } "#); } + +#[actix_rt::test] +async fn deleting_fragments_deletes_vectors() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + println!("Documents before update: {documents:?}"); + + let (response, code) = index + .update_settings(settings) + .await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": null, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From 5c792737486899d2af03aaf7cb9e0acba2fcc883 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 14:42:49 +0200 Subject: [PATCH 05/24] Add TODOs --- crates/meilisearch/tests/vector/fragments.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index c3fef8c79..338f9e7c5 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -109,6 +109,12 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: Test cannot pass both fragments and document +// TODO: test with 2 embedders + +// TODO: edit fragment + +// TODO: document fragment replaced + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; From 90683d0e4e77a1f8d9924a4a9e8065767b49c76d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 3 Jul 2025 14:33:45 +0200 Subject: [PATCH 06/24] add snapshot of get settings --- crates/meilisearch/tests/vector/fragments.rs | 99 +++++++++++++++++--- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 338f9e7c5..a21855d8d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -8,8 +8,7 @@ use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::index::Index; use crate::common::{Owned, Shared}; use crate::json; -use crate::vector::Server; -use crate::vector::GetAllDocumentsOptions; +use crate::vector::{GetAllDocumentsOptions, Server}; async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); @@ -82,9 +81,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va }, }, }); - let (response, code) = index - .update_settings(settings.clone()) - .await; + let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; @@ -359,9 +356,7 @@ async fn deleting_fragments_deletes_vectors() { .await; println!("Documents before update: {documents:?}"); - let (response, code) = index - .update_settings(settings) - .await; + let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value, @r#" @@ -410,11 +405,91 @@ async fn deleting_fragments_deletes_vectors() { } "#); + let (value, code) = index.settings().await; + snapshot!(value, @r###" + { + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "sortableAttributes": [], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": [], + "nonSeparatorTokens": [], + "separatorTokens": [], + "dictionary": [], + "synonyms": {}, + "distinctAttribute": null, + "proximityPrecision": "byWord", + "typoTolerance": { + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 5, + "twoTypos": 9 + }, + "disableOnWords": [], + "disableOnAttributes": [], + "disableOnNumbers": false + }, + "faceting": { + "maxValuesPerFacet": 100, + "sortFacetValuesBy": { + "*": "alpha" + } + }, + "pagination": { + "maxTotalHits": 1000 + }, + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "http://127.0.0.1:53832", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + }, + "searchCutoffMs": null, + "localizedAttributes": null, + "facetSearch": true, + "prefixSearch": "indexingTime" + } + "###); + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r#" + snapshot!(json_string!(documents), @r###" { "results": [ { @@ -453,7 +528,7 @@ async fn deleting_fragments_deletes_vectors() { [ 1.0, 1.0, - 0.0 + -1.0 ] ], "regenerate": true @@ -470,7 +545,7 @@ async fn deleting_fragments_deletes_vectors() { [ -0.5, 0.5, - 0.0 + 1.0 ] ], "regenerate": true @@ -482,5 +557,5 @@ async fn deleting_fragments_deletes_vectors() { "limit": 20, "total": 4 } - "#); + "###); } From a3af9fe0578903895c6c210e66760d0ab48205a9 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 3 Jul 2025 14:35:02 +0200 Subject: [PATCH 07/24] new extractor bugfixes: - fix old_has_fragments - new_is_user_provided is always false when generating fragments, even if no fragment ever matches --- .../src/update/new/extract/vectors/mod.rs | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/crates/milli/src/update/new/extract/vectors/mod.rs b/crates/milli/src/update/new/extract/vectors/mod.rs index 72a07dea6..4ca68027c 100644 --- a/crates/milli/src/update/new/extract/vectors/mod.rs +++ b/crates/milli/src/update/new/extract/vectors/mod.rs @@ -357,7 +357,7 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor> chunks.is_user_provided_must_regenerate(document.docid()); let old_has_fragments = old_embedders .get(embedder_name) - .map(|embedder| embedder.fragments().is_empty()) + .map(|embedder| !embedder.fragments().is_empty()) .unwrap_or_default(); let new_has_fragments = chunks.has_fragments(); @@ -628,9 +628,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { session.on_embed_mut().clear_vectors(docid); } - let mut extracted = false; - let extracted = &mut extracted; - settings_delta.try_for_each_fragment_diff( session.embedder_name(), |fragment_diff| { @@ -660,7 +657,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { ); } ExtractorDiff::Added(input) | ExtractorDiff::Updated(input) => { - *extracted = true; session.request_embedding( metadata, input, @@ -673,13 +669,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { Result::Ok(()) }, )?; - self.set_status( - docid, - old_is_user_provided, - true, - old_is_user_provided & !*extracted, - true, - ); + self.set_status(docid, old_is_user_provided, true, false, true); } ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); @@ -732,7 +722,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { where 'a: 'doc, { - let extracted = match &mut self.kind { + match &mut self.kind { ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); let ex = DocumentTemplateExtractor::new( @@ -785,7 +775,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { docid, old_is_user_provided, old_must_regenerate, - old_is_user_provided && !extracted, + false, new_must_regenerate, ); @@ -968,7 +958,7 @@ fn update_autogenerated<'doc, 'a: 'doc, 'b, E, OD, ND>( old_must_regenerate: bool, session: &mut EmbedSession<'a, OnEmbeddingDocumentUpdates<'a, 'b>, E::Input>, unused_vectors_distribution: &UnusedVectorsDistributionBump<'a>, -) -> Result +) -> Result<()> where OD: Document<'doc> + Debug, ND: Document<'doc> + Debug, @@ -976,7 +966,6 @@ where E::Input: Input, crate::Error: From, { - let mut extracted = false; for extractor in extractors { let new_rendered = extractor.extract(&new_document, meta)?; let must_regenerate = if !old_must_regenerate { @@ -995,7 +984,6 @@ where }; if must_regenerate { - extracted = true; let metadata = Metadata { docid, external_docid, extractor_id: extractor.extractor_id() }; @@ -1011,7 +999,7 @@ where } } - Ok(extracted) + Ok(()) } fn insert_autogenerated<'a, 'b, E, D: Document<'a> + Debug>( From de24e75be8a34da406d84656698c68185d7644a7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:00:11 +0200 Subject: [PATCH 08/24] Update test --- crates/meilisearch/tests/vector/fragments.rs | 101 +++++-------------- 1 file changed, 28 insertions(+), 73 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index a21855d8d..863d0127b 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -406,84 +406,39 @@ async fn deleting_fragments_deletes_vectors() { "#); let (value, code) = index.settings().await; - snapshot!(value, @r###" + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value["embedders"], { + ".rest.url" => "[url]", + }), @r#" { - "displayedAttributes": [ - "*" - ], - "searchableAttributes": [ - "*" - ], - "filterableAttributes": [], - "sortableAttributes": [], - "rankingRules": [ - "words", - "typo", - "proximity", - "attribute", - "sort", - "exactness" - ], - "stopWords": [], - "nonSeparatorTokens": [], - "separatorTokens": [], - "dictionary": [], - "synonyms": {}, - "distinctAttribute": null, - "proximityPrecision": "byWord", - "typoTolerance": { - "enabled": true, - "minWordSizeForTypos": { - "oneTypo": 5, - "twoTypos": 9 + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } }, - "disableOnWords": [], - "disableOnAttributes": [], - "disableOnNumbers": false - }, - "faceting": { - "maxValuesPerFacet": 100, - "sortFacetValuesBy": { - "*": "alpha" - } - }, - "pagination": { - "maxTotalHits": 1000 - }, - "embedders": { - "rest": { - "source": "rest", - "dimensions": 3, - "url": "http://127.0.0.1:53832", - "indexingFragments": { - "withBreed": { - "value": "{{ doc.name }} is a {{ doc.breed }}" - } + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" }, - "searchFragments": { - "justBreed": { - "value": "It's a {{ media.breed }}" - }, - "justName": { - "value": "{{ media.name }} is a dog" - }, - "query": { - "value": "Some pre-prompt for query {{ q }}" - } + "justName": { + "value": "{{ media.name }} is a dog" }, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "headers": {} - } - }, - "searchCutoffMs": null, - "localizedAttributes": null, - "facetSearch": true, - "prefixSearch": "indexingTime" + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } } - "###); + "#); let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) From 2bcd69750f61fb3f46f8e8759fac80bd3b2b170f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:08:27 +0200 Subject: [PATCH 09/24] Add fragment modification test --- crates/meilisearch/tests/vector/fragments.rs | 158 ++++++++++++++++++- 1 file changed, 153 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 863d0127b..2b7592316 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -112,6 +112,8 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced +// TODO: not setting to null but ommitting settings + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -351,11 +353,6 @@ async fn deleting_fragments_deletes_vectors() { settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; - let (documents, code) = index - .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) - .await; - println!("Documents before update: {documents:?}"); - let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); @@ -514,3 +511,154 @@ async fn deleting_fragments_deletes_vectors() { } "###); } + +#[actix_rt::test] +async fn modifying_fragments_modifies_vectors() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"]["value"] = + serde_json::Value::String("{{ doc.name }} is a dog (maybe bulldog?)".to_string()); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog (maybe bulldog?)" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + From 2faad504c6d7c2195bd99cd947cc5a4a3d1db38b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:12:47 +0200 Subject: [PATCH 10/24] Add test --- crates/meilisearch/tests/vector/fragments.rs | 92 +++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 2b7592316..00682299b 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -112,8 +112,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced -// TODO: not setting to null but ommitting settings - #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -662,3 +660,93 @@ async fn modifying_fragments_modifies_vectors() { "#); } +#[actix_rt::test] +async fn ommitted_fragment_isnt_removed() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; // basic is removed + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().remove("withBreed"); // withBreed isn't specified + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": null + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (value, code) = index.settings().await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value["embedders"], { + ".rest.url" => "[url]", + }), @r#" + { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + } + "#); +} + From 5690700601b4ef3970c8c9e4c7da57b6c3e6bbb0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:19:31 +0200 Subject: [PATCH 11/24] Add fragment addition test --- crates/meilisearch/tests/vector/fragments.rs | 168 +++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 00682299b..71bab6ea0 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -750,3 +750,171 @@ async fn ommitted_fragment_isnt_removed() { "#); } +#[actix_rt::test] +async fn fragment_insertion() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert(String::from("useless"), serde_json::json!({ + "value": "This fragment is useless" + })); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "useless": { + "value": "This fragment is useless" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From 7423243be0bc373a42b5518d6cc625e762187629 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:52:18 +0200 Subject: [PATCH 12/24] Add test with multiple embedders --- crates/meilisearch/tests/vector/fragments.rs | 447 ++++++++++++++++++- 1 file changed, 445 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 71bab6ea0..03d0ffc7a 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -106,12 +106,14 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: Test cannot pass both fragments and document -// TODO: test with 2 embedders - // TODO: edit fragment // TODO: document fragment replaced +// TODO: complex value + +// TODO: swapping fragments + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -918,3 +920,444 @@ async fn fragment_insertion() { } "#); } + +#[actix_rt::test] +async fn multiple_embedders() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + let url = settings["embedders"]["rest"]["url"].as_str().unwrap(); + + let settings2 = json!({ + "embedders": { + "rest2": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + "rest3": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index.update_settings(settings2).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest2": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + }, + "rest3": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + } + }, + "searchFragments": { + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest2": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); + + // Remove Rest2 + + settings["embedders"]["rest2"] = serde_json::Value::Null; + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value["status"], @r###""succeeded""###); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); + + // Remove rest's basic fragment + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value["status"], @r###""succeeded""###); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r""); +} From cf9b311f71d2316905a0d25487e953c68d6a345f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:53:09 +0200 Subject: [PATCH 13/24] Format --- crates/meilisearch/tests/common/mod.rs | 2 +- crates/meilisearch/tests/vector/fragments.rs | 13 ++++++++----- crates/meilisearch/tests/vector/mod.rs | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 2fafbd11f..1a73a7532 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,7 +3,7 @@ pub mod index; pub mod server; pub mod service; -use std::{fmt::{self, Display}, future::Future}; +use std::fmt::{self, Display}; #[allow(unused)] pub use index::GetAllDocumentsOptions; diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 03d0ffc7a..7cfa0c1af 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -757,9 +757,12 @@ async fn fragment_insertion() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert(String::from("useless"), serde_json::json!({ - "value": "This fragment is useless" - })); + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert( + String::from("useless"), + serde_json::json!({ + "value": "This fragment is useless" + }), + ); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1215,7 +1218,7 @@ async fn multiple_embedders() { snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value["status"], @r###""succeeded""###); - + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; @@ -1354,7 +1357,7 @@ async fn multiple_embedders() { snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value["status"], @r###""succeeded""###); - + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 837c34289..7f54489b6 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -1,10 +1,10 @@ mod binary_quantized; +mod fragments; #[cfg(feature = "test-ollama")] mod ollama; mod openai; mod rest; mod settings; -mod fragments; use std::str::FromStr; From caccb5181449fcde12bf967688608b6ef9fc7188 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 16:10:23 +0200 Subject: [PATCH 14/24] Add a complex value test --- crates/meilisearch/tests/vector/fragments.rs | 186 ++++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 7cfa0c1af..20bc4e7ce 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -30,6 +30,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va ("dustin", [-0.5, 0.5, 0.0]), ("bulldog", [0.0, 0.0, 1.0]), ("labrador", [0.0, 0.0, -1.0]), + ("{", [-9999.0, -9999.0, -9999.0]), // That wouldn't be nice ] .into_iter() .collect(); @@ -110,8 +111,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced -// TODO: complex value - // TODO: swapping fragments #[actix_rt::test] @@ -1364,3 +1363,186 @@ async fn multiple_embedders() { snapshot!(code, @"200 OK"); snapshot!(json_string!(documents), @r""); } + +#[actix_rt::test] +async fn complex_fragment() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert( + String::from("complex"), + serde_json::json!({ + "value": { + "breed": "{{ doc.breed }}", + "breeds": [ + "{{ doc.breed }}", + { + "breed": "{{ doc.breed }}", + } + ] + } + }), + ); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "complex": { + "value": { + "breed": "{{ doc.breed }}", + "breeds": [ + "{{ doc.breed }}", + { + "breed": "{{ doc.breed }}" + } + ] + } + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ], + [ + 0.0, + 0.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From d0cd3cacecbaa01fe3a273924232fe9d119e37f6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 18:18:04 +0200 Subject: [PATCH 15/24] Add a way to reproduce the bug --- crates/meilisearch/tests/vector/fragments.rs | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 20bc4e7ce..0135e2044 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1351,6 +1351,7 @@ async fn multiple_embedders() { // Remove rest's basic fragment settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + //settings["embedders"].as_object_mut().unwrap().remove("rest2"); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1364,6 +1365,56 @@ async fn multiple_embedders() { snapshot!(json_string!(documents), @r""); } +#[actix_rt::test] +async fn remove_non_existant_embedder() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"].as_object_mut().unwrap().insert(String::from("non-existant"), serde_json::Value::Null); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r""); +} + +#[actix_rt::test] +async fn double_remove_embedder() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"].as_object_mut().unwrap().insert(String::from("rest"), serde_json::Value::Null); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": null + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#""#); +} + #[actix_rt::test] async fn complex_fragment() { let (server, uid, mut settings) = init_fragments_index().await; From 3714f166967a37adc7f04a101bed4ca62e121762 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 10:40:50 +0200 Subject: [PATCH 16/24] Fix bug --- crates/milli/src/update/settings.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 911f51865..4124aa540 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -101,6 +101,10 @@ impl Setting { matches!(self, Self::NotSet) } + pub const fn is_reset(&self) -> bool { + matches!(self, Self::Reset) + } + /// If `Self` is `Reset`, then map self to `Set` with the provided `val`. pub fn or_reset(self, val: T) -> Self { match self { @@ -1213,6 +1217,10 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { // new config EitherOrBoth::Right((name, mut setting)) => { tracing::debug!(embedder = name, "new embedder"); + // if we are asked to reset an embedder that doesn't exist, just ignore it + if setting.is_reset() { + continue; + } // apply the default source in case the source was not set so that it gets validated crate::vector::settings::EmbeddingSettings::apply_default_source(&mut setting); crate::vector::settings::EmbeddingSettings::apply_default_openai_model( From 8dfded2993a13ea958593d8e0616660efd104758 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 10:49:03 +0200 Subject: [PATCH 17/24] Update tests --- crates/meilisearch/tests/vector/fragments.rs | 70 +++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 0135e2044..915f1c79d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1375,7 +1375,54 @@ async fn remove_non_existant_embedder() { let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; - snapshot!(task, @r""); + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "non-existant": null, + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); } #[actix_rt::test] @@ -1412,7 +1459,26 @@ async fn double_remove_embedder() { let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; - snapshot!(task, @r#""#); + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": null + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); } #[actix_rt::test] From 6792d048b8aafd13f799fe9254a7c67bb9f07495 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 11:47:38 +0200 Subject: [PATCH 18/24] Test both fragments and document template --- crates/meilisearch/tests/vector/fragments.rs | 155 ++++++++++++++++++- 1 file changed, 152 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 915f1c79d..25d7a7ffc 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -30,7 +30,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va ("dustin", [-0.5, 0.5, 0.0]), ("bulldog", [0.0, 0.0, 1.0]), ("labrador", [0.0, 0.0, -1.0]), - ("{", [-9999.0, -9999.0, -9999.0]), // That wouldn't be nice + ("{{ doc.", [-9999.0, -9999.0, -9999.0]), // If a template didn't render ] .into_iter() .collect(); @@ -68,7 +68,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va "dimensions": 3, "request": "{{fragment}}", "response": { - "data": "{{embedding}}" + "data": "{{embedding}}" }, "indexingFragments": { "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, @@ -1362,7 +1362,115 @@ async fn multiple_embedders() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r""); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); } #[actix_rt::test] @@ -1663,3 +1771,44 @@ async fn complex_fragment() { } "#); } + +#[actix_rt::test] +async fn both_fragments_and_document_template() { + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": "http://localhost:1337", + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "justBreed": {"value": "It's a {{ media.breed }}"}, + }, + "documentTemplate": "{{ doc.name }} is a dog", + }, + }, + }); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r#" + { + "message": "Error while generating embeddings: user error: cannot pass both fragments and a document template.\n - Note: 1 fragments declared in `indexingFragments` and 1 fragments declared in `search_fragments_len`.\n - Hint: remove the declared fragments or remove the `documentTemplate`", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); +} From 48527761e72bd04e87f28269430bccb31395e173 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 12:01:15 +0200 Subject: [PATCH 19/24] Add test --- crates/meilisearch/tests/vector/fragments.rs | 113 +++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 25d7a7ffc..3c0c154d5 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1812,3 +1812,116 @@ async fn both_fragments_and_document_template() { } "#); } + +#[actix_rt::test] +async fn set_fragments_then_document_template() { + let (server, uid, settings) = init_fragments_index().await; + let index = server.index(uid); + + let url = settings["embedders"]["rest"]["url"].as_str().unwrap(); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "documentTemplate": "{{ doc.name }} is a dog", + }, + }, + }); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r""); + + let (settings, code) = index.settings().await; + snapshot!(code, @"200 OK"); + snapshot!(settings, @r#" + { + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "sortableAttributes": [], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": [], + "nonSeparatorTokens": [], + "separatorTokens": [], + "dictionary": [], + "synonyms": {}, + "distinctAttribute": null, + "proximityPrecision": "byWord", + "typoTolerance": { + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 5, + "twoTypos": 9 + }, + "disableOnWords": [], + "disableOnAttributes": [], + "disableOnNumbers": false + }, + "faceting": { + "maxValuesPerFacet": 100, + "sortFacetValuesBy": { + "*": "alpha" + } + }, + "pagination": { + "maxTotalHits": 1000 + }, + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "http://127.0.0.1:55578", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + }, + "searchCutoffMs": null, + "localizedAttributes": null, + "facetSearch": true, + "prefixSearch": "indexingTime" + } + "#); +} + From b274106ad3adb3a3224d29114ecb2795a09676a7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:05:52 +0200 Subject: [PATCH 20/24] Add test --- crates/meilisearch/tests/vector/fragments.rs | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 3c0c154d5..cf4ed4ab4 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -267,6 +267,30 @@ async fn search_with_media() { "#); } +#[actix_rt::test] +async fn search_with_media_and_vector() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "vector": [1.0, 1.0, 1.0], + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid request: both `media` and `vector` parameters are present.", + "code": "invalid_search_media_and_vector", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_media_and_vector" + } + "#); +} + #[actix_rt::test] async fn search_with_media_matching_multiple_fragments() { let index = shared_index_for_fragments().await; From be9f4f96dfd295a90c92f286f2b5903af209abad Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:15:15 +0200 Subject: [PATCH 21/24] Add experimental feature test --- crates/meilisearch/tests/vector/fragments.rs | 40 +++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index cf4ed4ab4..0dde9dfc2 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,14 +105,50 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: Test cannot pass both fragments and document - // TODO: edit fragment // TODO: document fragment replaced // TODO: swapping fragments +// TODO: consistency + +#[actix_rt::test] +async fn experimental_feature_not_enabled() { + let server = Server::new().await; + let index = server.unique_index(); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": "http://localhost:1337", + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r#" + { + "message": "setting `indexingFragments` requires enabling the `multimodal` experimental feature. See https://github.com/orgs/meilisearch/discussions/846", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "#); +} + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; From 16234e1313c4d76b000c5c1ea1d229c6ff2c26c1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:25:42 +0200 Subject: [PATCH 22/24] Add fragment swapping test --- crates/meilisearch/tests/vector/fragments.rs | 160 ++++++++++++++++++- 1 file changed, 153 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 0dde9dfc2..f083e40d0 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,14 +105,8 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: edit fragment - // TODO: document fragment replaced -// TODO: swapping fragments - -// TODO: consistency - #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; @@ -158,7 +152,7 @@ async fn indexing_fragments() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r#" + snapshot!(documents, @r#" { "results": [ { @@ -721,6 +715,158 @@ async fn modifying_fragments_modifies_vectors() { "#); } +#[actix_rt::test] +async fn swapping_fragments() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + let basic = settings["embedders"]["rest"]["indexingFragments"]["basic"].clone(); + let with_breed = settings["embedders"]["rest"]["indexingFragments"]["withBreed"].clone(); + settings["embedders"]["rest"]["indexingFragments"]["basic"] = with_breed; + settings["embedders"]["rest"]["indexingFragments"]["withBreed"] = basic; + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + }, + "withBreed": { + "value": "{{ doc.name }} is a dog" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(documents, @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ], + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ], + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + #[actix_rt::test] async fn ommitted_fragment_isnt_removed() { let (server, uid, mut settings) = init_fragments_index().await; From c5993196b3c925dccd811afdbe3b9da5e112ed2b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:32:55 +0200 Subject: [PATCH 23/24] Add test --- crates/meilisearch/tests/vector/fragments.rs | 115 ++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index f083e40d0..70e4433ed 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,8 +105,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: document fragment replaced - #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; @@ -239,6 +237,119 @@ async fn indexing_fragments() { "#); } +#[actix_rt::test] +async fn replace_document() { + let (server, uid, _settings) = init_fragments_index().await; + let index = server.index(uid); + + let documents = json!([ + { "id": 0, "name": "kefir", "breed": "sorry-I-forgot" }, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Make sure kefir now has 2 vectors + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(documents, @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "breed": "sorry-I-forgot", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ], + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + + #[actix_rt::test] async fn search_with_vector() { let index = shared_index_for_fragments().await; From fa3990daf90920bec99cf4051afb48ce46e79a66 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:33:49 +0200 Subject: [PATCH 24/24] Format --- crates/meilisearch/tests/vector/fragments.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 70e4433ed..337f01ca6 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -349,7 +349,6 @@ async fn replace_document() { "#); } - #[actix_rt::test] async fn search_with_vector() { let index = shared_index_for_fragments().await; @@ -891,7 +890,7 @@ async fn swapping_fragments() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(documents, @r#" + snapshot!(documents, @r#" { "results": [ { @@ -1795,7 +1794,10 @@ async fn remove_non_existant_embedder() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"].as_object_mut().unwrap().insert(String::from("non-existant"), serde_json::Value::Null); + settings["embedders"] + .as_object_mut() + .unwrap() + .insert(String::from("non-existant"), serde_json::Value::Null); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1855,7 +1857,10 @@ async fn double_remove_embedder() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"].as_object_mut().unwrap().insert(String::from("rest"), serde_json::Value::Null); + settings["embedders"] + .as_object_mut() + .unwrap() + .insert(String::from("rest"), serde_json::Value::Null); let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); @@ -2241,4 +2246,3 @@ async fn set_fragments_then_document_template() { } "#); } -