From 633537ccd71811a1ee0cd6bd2c2d5b41b0f710a4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 19 Sep 2024 11:41:55 +0200 Subject: [PATCH] fix updating documents without updating the settings --- meilisearch/tests/vector/binary_quantized.rs | 54 +++++++++++++------ milli/src/update/index_documents/mod.rs | 6 ++- .../src/update/index_documents/typed_chunk.rs | 5 +- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/meilisearch/tests/vector/binary_quantized.rs b/meilisearch/tests/vector/binary_quantized.rs index 469ec878b..0f3d01c2d 100644 --- a/meilisearch/tests/vector/binary_quantized.rs +++ b/meilisearch/tests/vector/binary_quantized.rs @@ -112,27 +112,49 @@ async fn binary_quantize_before_sending_documents() { snapshot!(code, @"202 Accepted"); index.wait_task(value.uid()).await.succeeded(); - // Make sure the documents DB has been cleared + // Make sure the documents are binary quantized let (documents, _code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(json_string!(documents), @r###" { - "message": "internal: Invalid distance provided. Got binary quantized angular but expected angular.", - "code": "internal", - "type": "internal", - "link": "https://docs.meilisearch.com/errors#internal" - } - "###); - - // Make sure the arroy DB has been cleared - let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1] })).await; - snapshot!(documents, @r###" - { - "message": "internal: Invalid distance provided. Got binary quantized angular but expected angular.", - "code": "internal", - "type": "internal", - "link": "https://docs.meilisearch.com/errors#internal" + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "manual": { + "embeddings": [ + [ + -1.0, + -1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "manual": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": false + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 2 } "###); } diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index d8566582c..326dd842d 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -705,7 +705,11 @@ where InternalError::DatabaseMissingEntry { db_name: "embedder_category_id", key: None }, )?; let embedder_config = settings_diff.embedding_config_updates.get(&embedder_name); - let was_quantized = embedder_config.map_or(false, |action| action.was_quantized); + let was_quantized = settings_diff + .old + .embedding_configs + .get(&embedder_name) + .map_or(false, |conf| conf.2); let is_quantizing = embedder_config.map_or(false, |action| action.is_being_quantized); pool.install(|| { diff --git a/milli/src/update/index_documents/typed_chunk.rs b/milli/src/update/index_documents/typed_chunk.rs index 90e49d23b..97a4bf712 100644 --- a/milli/src/update/index_documents/typed_chunk.rs +++ b/milli/src/update/index_documents/typed_chunk.rs @@ -668,9 +668,10 @@ pub(crate) fn write_typed_chunk_into_index( InternalError::DatabaseMissingEntry { db_name: "embedder_category_id", key: None }, )?; let binary_quantized = settings_diff - .embedding_config_updates + .old + .embedding_configs .get(&embedder_name) - .map_or(false, |conf| conf.was_quantized); + .map_or(false, |conf| conf.2); // FIXME: allow customizing distance let writers: Vec<_> = crate::vector::arroy_db_range_for_embedder(embedder_index) .map(|k| ArroyWrapper::new(index.vector_arroy, k, binary_quantized))