Merge #4649

4649: Don't store the vectors in the documents database r=dureuill a=irevoire

# Pull Request

## Related issue
Fixes https://github.com/meilisearch/meilisearch/issues/4607

## What does this PR do?
- Ensure that anything falling under `_vectors` is NOT searchable, filterable or sortable
- [x] per embedder, add a roaring bitmap of documents that provide "userProvided" embeddings
- [x] in the indexing process in extract_vector_points, set the bit corresponding to the document depending on the "userProvided" subfield in the _vectors field.
- [x] in the document DB in typed chunks, when writing the _vectors field, remove all keys corresponding to an embedder

Co-authored-by: Tamo <tamo@meilisearch.com>
Co-authored-by: Louis Dureuil <louis@meilisearch.com>

meilisearch/tests/common/index.rs

@@ -182,14 +182,10 @@ impl Index<'_> {
         self.service.get(url).await
     }
 
-    pub async fn get_document(
-        &self,
-        id: u64,
-        options: Option<GetDocumentOptions>,
-    ) -> (Value, StatusCode) {
+    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
         let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
-        if let Some(fields) = options.and_then(|o| o.fields) {
-            let _ = write!(url, "?fields={}", fields.join(","));
+        if let Some(options) = options {
+            write!(url, "?{}", yaup::to_string(&options).unwrap()).unwrap();
         }
         self.service.get(url).await
     }
@@ -205,18 +201,11 @@ impl Index<'_> {
     }
 
     pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref()));
-        if let Some(limit) = options.limit {
-            let _ = write!(url, "limit={}&", limit);
-        }
-
-        if let Some(offset) = options.offset {
-            let _ = write!(url, "offset={}&", offset);
-        }
-
-        if let Some(attributes_to_retrieve) = options.attributes_to_retrieve {
-            let _ = write!(url, "fields={}&", attributes_to_retrieve.join(","));
-        }
+        let url = format!(
+            "/indexes/{}/documents?{}",
+            urlencode(self.uid.as_ref()),
+            yaup::to_string(&options).unwrap()
+        );
 
         self.service.get(url).await
     }
@@ -435,13 +424,11 @@ impl Index<'_> {
     }
 }
 
-pub struct GetDocumentOptions {
-    pub fields: Option<Vec<&'static str>>,
-}
-
-#[derive(Debug, Default)]
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
 pub struct GetAllDocumentsOptions {
     pub limit: Option<usize>,
     pub offset: Option<usize>,
-    pub attributes_to_retrieve: Option<Vec<&'static str>>,
+    pub retrieve_vectors: bool,
+    pub fields: Option<Vec<&'static str>>,
 }

meilisearch/tests/common/mod.rs

@@ -6,7 +6,7 @@ pub mod service;
 use std::fmt::{self, Display};
 
 #[allow(unused)]
-pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
+pub use index::GetAllDocumentsOptions;
 use meili_snap::json_string;
 use serde::{Deserialize, Serialize};
 #[allow(unused)]

meilisearch/tests/documents/errors.rs

@@ -795,3 +795,70 @@ async fn fetch_document_by_filter() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn retrieve_vectors() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+
+    // GET ALL DOCUMENTS BY QUERY
+    let (response, _code) = index.get_all_documents_raw("?retrieveVectors=tamo").await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `tamo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_all_documents_raw("?retrieveVectors=true").await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    // FETCH ALL DOCUMENTS BY POST
+    let (response, _code) =
+        index.get_document_by_filter(json!({ "retrieveVectors": "tamo" })).await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found a string: `\"tamo\"`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_document_by_filter(json!({ "retrieveVectors": true })).await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    // GET A SINGLE DOCUMENT
+    let (response, _code) = index.get_document(0, Some(json!({"retrieveVectors": "tamo"}))).await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `tamo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_document(0, Some(json!({"retrieveVectors": true}))).await;
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+}

meilisearch/tests/documents/get_documents.rs

@@ -4,7 +4,7 @@ use meili_snap::*;
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
+use crate::common::{GetAllDocumentsOptions, Server, Value};
 use crate::json;
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
@@ -59,8 +59,7 @@ async fn get_document() {
         })
     );
 
-    let (response, code) =
-        index.get_document(0, Some(GetDocumentOptions { fields: Some(vec!["id"]) })).await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["id"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -69,9 +68,8 @@ async fn get_document() {
         })
     );
 
-    let (response, code) = index
-        .get_document(0, Some(GetDocumentOptions { fields: Some(vec!["nested.content"]) }))
-        .await;
+    let (response, code) =
+        index.get_document(0, Some(json!({ "fields": ["nested.content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -211,7 +209,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["name"]),
+            fields: Some(vec!["name"]),
             ..Default::default()
         })
         .await;
@@ -225,9 +223,19 @@ async fn test_get_all_documents_attributes_to_retrieve() {
     assert_eq!(response["limit"], json!(20));
     assert_eq!(response["total"], json!(77));
 
+    let (response, code) = index.get_all_documents_raw("?fields=").await;
+    assert_eq!(code, 200);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
+    for results in response["results"].as_array().unwrap() {
+        assert_eq!(results.as_object().unwrap().keys().count(), 0);
+    }
+    assert_eq!(response["offset"], json!(0));
+    assert_eq!(response["limit"], json!(20));
+    assert_eq!(response["total"], json!(77));
+
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec![]),
+            fields: Some(vec!["wrong"]),
             ..Default::default()
         })
         .await;
@@ -242,22 +250,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["wrong"]),
-            ..Default::default()
-        })
-        .await;
-    assert_eq!(code, 200);
-    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    for results in response["results"].as_array().unwrap() {
-        assert_eq!(results.as_object().unwrap().keys().count(), 0);
-    }
-    assert_eq!(response["offset"], json!(0));
-    assert_eq!(response["limit"], json!(20));
-    assert_eq!(response["total"], json!(77));
-
-    let (response, code) = index
-        .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["name", "tags"]),
+            fields: Some(vec!["name", "tags"]),
             ..Default::default()
         })
         .await;
@@ -270,10 +263,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
     }
 
     let (response, code) = index
-        .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["*"]),
-            ..Default::default()
-        })
+        .get_all_documents(GetAllDocumentsOptions { fields: Some(vec!["*"]), ..Default::default() })
         .await;
     assert_eq!(code, 200);
     assert_eq!(response["results"].as_array().unwrap().len(), 20);
@@ -283,7 +273,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["*", "wrong"]),
+            fields: Some(vec!["*", "wrong"]),
             ..Default::default()
         })
         .await;
@@ -316,12 +306,10 @@ async fn get_document_s_nested_attributes_to_retrieve() {
     assert_eq!(code, 202);
     index.wait_task(1).await;
 
-    let (response, code) =
-        index.get_document(0, Some(GetDocumentOptions { fields: Some(vec!["content"]) })).await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(response, json!({}));
-    let (response, code) =
-        index.get_document(1, Some(GetDocumentOptions { fields: Some(vec!["content"]) })).await;
+    let (response, code) = index.get_document(1, Some(json!({ "fields": ["content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -333,9 +321,7 @@ async fn get_document_s_nested_attributes_to_retrieve() {
         })
     );
 
-    let (response, code) = index
-        .get_document(0, Some(GetDocumentOptions { fields: Some(vec!["content.truc"]) }))
-        .await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["content.truc"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -343,9 +329,7 @@ async fn get_document_s_nested_attributes_to_retrieve() {
             "content.truc": "foobar",
         })
     );
-    let (response, code) = index
-        .get_document(1, Some(GetDocumentOptions { fields: Some(vec!["content.truc"]) }))
-        .await;
+    let (response, code) = index.get_document(1, Some(json!({ "fields": ["content.truc"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -540,3 +524,207 @@ async fn get_document_by_filter() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn get_document_with_vectors() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+    let (value, code) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(value, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "manual": {
+                  "source": "userProvided",
+                  "dimensions": 3,
+              }
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    let documents = json!([
+      {"id": 0, "name": "kefir", "_vectors": { "manual": [0, 0, 0] }},
+      {"id": 1, "name": "echo", "_vectors": { "manual": null }},
+    ]);
+    let (value, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    // by default you shouldn't see the `_vectors` object
+    let (documents, _code) = index.get_all_documents(Default::default()).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir"
+        },
+        {
+          "id": 1,
+          "name": "echo"
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) = index.get_document(0, None).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "id": 0,
+      "name": "kefir"
+    }
+    "###);
+
+    // if we try to retrieve the vectors with the `fields` parameter they
+    // still shouldn't be displayed
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            fields: Some(vec!["name", "_vectors"]),
+            ..Default::default()
+        })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "name": "kefir"
+        },
+        {
+          "name": "echo"
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) =
+        index.get_document(0, Some(json!({"fields": ["name", "_vectors"]}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "name": "kefir"
+    }
+    "###);
+
+    // If we specify the retrieve vectors boolean and nothing else we should get the vectors
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) = index.get_document(0, Some(json!({"retrieveVectors": true}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "id": 0,
+      "name": "kefir",
+      "_vectors": {
+        "manual": {
+          "embeddings": [
+            [
+              0.0,
+              0.0,
+              0.0
+            ]
+          ],
+          "regenerate": false
+        }
+      }
+    }
+    "###);
+
+    // If we specify the retrieve vectors boolean and exclude vectors form the `fields` we should still get the vectors
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            retrieve_vectors: true,
+            fields: Some(vec!["name"]),
+            ..Default::default()
+        })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) =
+        index.get_document(0, Some(json!({"retrieveVectors": true, "fields": ["name"]}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "name": "kefir",
+      "_vectors": {
+        "manual": {
+          "embeddings": [
+            [
+              0.0,
+              0.0,
+              0.0
+            ]
+          ],
+          "regenerate": false
+        }
+      }
+    }
+    "###);
+}

meilisearch/tests/dumps/mod.rs

@@ -1938,3 +1938,210 @@ async fn import_dump_v6_containing_experimental_features() {
         })
         .await;
 }
+
+// In this test we must generate the dump ourselves to ensure the
+// `user provided` vectors are well set
+#[actix_rt::test]
+#[cfg_attr(target_os = "windows", ignore)]
+async fn generate_and_import_dump_containing_vectors() {
+    let temp = tempfile::tempdir().unwrap();
+    let mut opt = default_settings(temp.path());
+    let server = Server::new_with_options(opt.clone()).await.unwrap();
+    let (code, _) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+    let index = server.index("pets");
+    let (response, code) = index
+        .update_settings(json!(
+        {
+            "embedders": {
+                "doggo_embedder": {
+                    "source": "huggingFace",
+                    "model": "sentence-transformers/all-MiniLM-L6-v2",
+                    "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+                    "documentTemplate": "{{doc.doggo}}",
+                }
+            }
+        }
+        ))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response);
+    let (response, code) = index
+        .add_documents(
+            json!([
+                {"id": 0, "doggo": "kefir", "_vectors": { "doggo_embedder": vec![0; 384] }},
+                {"id": 1, "doggo": "echo", "_vectors": { "doggo_embedder": { "regenerate": false, "embeddings": vec![1; 384] }}},
+                {"id": 2, "doggo": "intel", "_vectors": { "doggo_embedder": { "regenerate": true, "embeddings": vec![2; 384] }}},
+                {"id": 3, "doggo": "bill", "_vectors": { "doggo_embedder": { "regenerate": true }}},
+                {"id": 4, "doggo": "max" },
+            ]),
+            None,
+        )
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response);
+
+    let (response, code) = server.create_dump().await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response["status"], @r###""succeeded""###);
+
+    // ========= We made a dump, now we should clear the DB and try to import our dump
+    drop(server);
+    tokio::fs::remove_dir_all(&opt.db_path).await.unwrap();
+    let dump_name = format!("{}.dump", response["details"]["dumpUid"].as_str().unwrap());
+    let dump_path = opt.dump_dir.join(dump_name);
+    assert!(dump_path.exists(), "path: `{}`", dump_path.display());
+
+    opt.import_dump = Some(dump_path);
+    // NOTE: We shouldn't have to change the database path but I lost one hour
+    // because of a « bad path » error and that fixed it.
+    opt.db_path = temp.path().join("data.ms");
+
+    let mut server = Server::new_auth_with_options(opt, temp).await;
+    server.use_api_key("MASTER_KEY");
+
+    let (indexes, code) = server.list_indexes(None, None).await;
+    assert_eq!(code, 200, "{indexes}");
+
+    snapshot!(indexes["results"].as_array().unwrap().len(), @"1");
+    snapshot!(indexes["results"][0]["uid"], @r###""pets""###);
+    snapshot!(indexes["results"][0]["primaryKey"], @r###""id""###);
+
+    let (response, code) = server.get_features().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+
+    let index = server.index("pets");
+
+    let (response, code) = index.settings().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "displayedAttributes": [
+        "*"
+      ],
+      "searchableAttributes": [
+        "*"
+      ],
+      "filterableAttributes": [],
+      "sortableAttributes": [],
+      "rankingRules": [
+        "words",
+        "typo",
+        "proximity",
+        "attribute",
+        "sort",
+        "exactness"
+      ],
+      "stopWords": [],
+      "nonSeparatorTokens": [],
+      "separatorTokens": [],
+      "dictionary": [],
+      "synonyms": {},
+      "distinctAttribute": null,
+      "proximityPrecision": "byWord",
+      "typoTolerance": {
+        "enabled": true,
+        "minWordSizeForTypos": {
+          "oneTypo": 5,
+          "twoTypos": 9
+        },
+        "disableOnWords": [],
+        "disableOnAttributes": []
+      },
+      "faceting": {
+        "maxValuesPerFacet": 100,
+        "sortFacetValuesBy": {
+          "*": "alpha"
+        }
+      },
+      "pagination": {
+        "maxTotalHits": 1000
+      },
+      "embedders": {
+        "doggo_embedder": {
+          "source": "huggingFace",
+          "model": "sentence-transformers/all-MiniLM-L6-v2",
+          "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+          "documentTemplate": "{{doc.doggo}}"
+        }
+      },
+      "searchCutoffMs": null
+    }
+    "###);
+
+    index
+        .search(json!({"retrieveVectors": true}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }), @r###"
+            [
+              {
+                "id": 0,
+                "doggo": "kefir",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": false
+                  }
+                }
+              },
+              {
+                "id": 1,
+                "doggo": "echo",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": false
+                  }
+                }
+              },
+              {
+                "id": 2,
+                "doggo": "intel",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              },
+              {
+                "id": 3,
+                "doggo": "bill",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              },
+              {
+                "id": 4,
+                "doggo": "max",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              }
+            ]
+            "###);
+        })
+        .await;
+}

meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap

@@ -0,0 +1,25 @@
+---
+source: meilisearch/tests/dumps/mod.rs
+---
+{
+  "uid": 0,
+  "indexUid": "pets",
+  "status": "succeeded",
+  "type": "settingsUpdate",
+  "canceledBy": null,
+  "details": {
+    "embedders": {
+      "doggo_embedder": {
+        "source": "huggingFace",
+        "model": "sentence-transformers/all-MiniLM-L6-v2",
+        "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+        "documentTemplate": "{{doc.doggo}}"
+      }
+    }
+  },
+  "error": null,
+  "duration": "[duration]",
+  "enqueuedAt": "[date]",
+  "startedAt": "[date]",
+  "finishedAt": "[date]"
+}

meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap

@@ -0,0 +1,19 @@
+---
+source: meilisearch/tests/dumps/mod.rs
+---
+{
+  "uid": 1,
+  "indexUid": "pets",
+  "status": "succeeded",
+  "type": "documentAdditionOrUpdate",
+  "canceledBy": null,
+  "details": {
+    "receivedDocuments": 5,
+    "indexedDocuments": 5
+  },
+  "error": null,
+  "duration": "[duration]",
+  "enqueuedAt": "[date]",
+  "startedAt": "[date]",
+  "finishedAt": "[date]"
+}

meilisearch/tests/integration.rs

@@ -13,6 +13,7 @@ mod snapshot;
 mod stats;
 mod swap_indexes;
 mod tasks;
+mod vector;
 
 // Tests are isolated by features in different modules to allow better readability, test
 // targetability, and improved incremental compilation times.

meilisearch/tests/search/errors.rs

@@ -167,6 +167,74 @@ async fn search_bad_hits_per_page() {
     "###);
 }
 
+#[actix_rt::test]
+async fn search_bad_attributes_to_retrieve() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (response, code) = index.search_post(json!({"attributesToRetrieve": "doggo"})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.attributesToRetrieve`: expected an array, but found a string: `\"doggo\"`",
+      "code": "invalid_search_attributes_to_retrieve",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_retrieve"
+    }
+    "###);
+    // Can't make the `attributes_to_retrieve` fail with a get search since it'll accept anything as an array of strings.
+}
+
+#[actix_rt::test]
+async fn search_bad_retrieve_vectors() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (response, code) = index.search_post(json!({"retrieveVectors": "doggo"})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found a string: `\"doggo\"`",
+      "code": "invalid_search_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_search_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.search_post(json!({"retrieveVectors": [true]})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found an array: `[true]`",
+      "code": "invalid_search_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_search_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.search_get("retrieveVectors=").await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `` as a boolean, expected either `true` or `false`",
+      "code": "invalid_search_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_search_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.search_get("retrieveVectors=doggo").await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `doggo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_search_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_search_retrieve_vectors"
+    }
+    "###);
+}
+
 #[actix_rt::test]
 async fn search_bad_attributes_to_crop() {
     let server = Server::new().await;

meilisearch/tests/search/hybrid.rs

@@ -124,29 +124,29 @@ async fn simple_search() {
 
     let (response, code) = index
         .search_post(
-            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.2}}),
+            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.2}, "retrieveVectors": true}),
         )
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]}}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}}}]"###);
     snapshot!(response["semanticHitCount"], @"0");
 
     let (response, code) = index
         .search_post(
-            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.5}, "showRankingScore": true}),
+            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.5}, "showRankingScore": true, "retrieveVectors": true}),
         )
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":0.9472135901451112}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###);
     snapshot!(response["semanticHitCount"], @"2");
 
     let (response, code) = index
         .search_post(
-            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.8}, "showRankingScore": true}),
+            json!({"q": "Captain", "vector": [1.0, 1.0], "hybrid": {"semanticRatio": 0.8}, "showRankingScore": true, "retrieveVectors": true}),
         )
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":0.9472135901451112}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###);
     snapshot!(response["semanticHitCount"], @"3");
 }
 
@@ -204,10 +204,10 @@ async fn distribution_shift() {
     let server = Server::new().await;
     let index = index_with_documents_user_provided(&server, &SIMPLE_SEARCH_DOCUMENTS_VEC).await;
 
-    let search = json!({"q": "Captain", "vector": [1.0, 1.0], "showRankingScore": true, "hybrid": {"semanticRatio": 1.0}});
+    let search = json!({"q": "Captain", "vector": [1.0, 1.0], "showRankingScore": true, "hybrid": {"semanticRatio": 1.0}, "retrieveVectors": true});
     let (response, code) = index.search_post(search.clone()).await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":0.9472135901451112}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###);
 
     let (response, code) = index
         .update_settings(json!({
@@ -228,7 +228,7 @@ async fn distribution_shift() {
 
     let (response, code) = index.search_post(search).await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":1.1920928955078125e-7}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.19161224365234375},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":1.1920928955078125e-7},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":1.1920928955078125e-7}]"###);
 }
 
 #[actix_rt::test]
@@ -239,20 +239,23 @@ async fn highlighter() {
     let (response, code) = index
         .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
             "hybrid": {"semanticRatio": 0.2},
-            "attributesToHighlight": [
-                     "desc"
+           "retrieveVectors": true,
+           "attributesToHighlight": [
+                     "desc",
+                     "_vectors",
                    ],
-                   "highlightPreTag": "**BEGIN**",
-                   "highlightPostTag": "**END**"
+           "highlightPreTag": "**BEGIN**",
+           "highlightPostTag": "**END**",
         }))
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}}}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"}}]"###);
     snapshot!(response["semanticHitCount"], @"0");
 
     let (response, code) = index
         .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
             "hybrid": {"semanticRatio": 0.8},
+            "retrieveVectors": true,
             "showRankingScore": true,
             "attributesToHighlight": [
                      "desc"
@@ -262,13 +265,14 @@ async fn highlighter() {
         }))
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}},"_rankingScore":0.9472135901451112}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
     snapshot!(response["semanticHitCount"], @"3");
 
     // no highlighting on full semantic
     let (response, code) = index
         .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
             "hybrid": {"semanticRatio": 1.0},
+            "retrieveVectors": true,
             "showRankingScore": true,
             "attributesToHighlight": [
                      "desc"
@@ -278,7 +282,7 @@ async fn highlighter() {
         }))
         .await;
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}},"_rankingScore":0.9472135901451112}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3"},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2"},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1"},"_rankingScore":0.9472135901451112}]"###);
     snapshot!(response["semanticHitCount"], @"3");
 }
 
@@ -361,12 +365,12 @@ async fn single_document() {
 
     let (response, code) = index
     .search_post(
-        json!({"vector": [1.0, 3.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true}),
+        json!({"vector": [1.0, 3.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true, "retrieveVectors": true}),
     )
     .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":1.0}"###);
+    snapshot!(response["hits"][0], @r###"{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":1.0}"###);
     snapshot!(response["semanticHitCount"], @"1");
 }
 
@@ -377,25 +381,25 @@ async fn query_combination() {
 
     // search without query and vector, but with hybrid => still placeholder
     let (response, code) = index
-        .search_post(json!({"hybrid": {"semanticRatio": 1.0}, "showRankingScore": true}))
+        .search_post(json!({"hybrid": {"semanticRatio": 1.0}, "showRankingScore": true, "retrieveVectors": true}))
         .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":1.0}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":1.0}]"###);
     snapshot!(response["semanticHitCount"], @"null");
 
     // same with a different semantic ratio
     let (response, code) = index
-        .search_post(json!({"hybrid": {"semanticRatio": 0.76}, "showRankingScore": true}))
+        .search_post(json!({"hybrid": {"semanticRatio": 0.76}, "showRankingScore": true, "retrieveVectors": true}))
         .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":1.0}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":1.0}]"###);
     snapshot!(response["semanticHitCount"], @"null");
 
     // wrong vector dimensions
     let (response, code) = index
-    .search_post(json!({"vector": [1.0, 0.0, 1.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true}))
+    .search_post(json!({"vector": [1.0, 0.0, 1.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true, "retrieveVectors": true}))
     .await;
 
     snapshot!(code, @"400 Bad Request");
@@ -410,34 +414,34 @@ async fn query_combination() {
 
     // full vector
     let (response, code) = index
-    .search_post(json!({"vector": [1.0, 0.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true}))
+    .search_post(json!({"vector": [1.0, 0.0], "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true, "retrieveVectors": true}))
     .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":0.6581138968467712}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.7773500680923462},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.7236068248748779},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.6581138968467712}]"###);
     snapshot!(response["semanticHitCount"], @"3");
 
     // full keyword, without a query
     let (response, code) = index
-    .search_post(json!({"vector": [1.0, 0.0], "hybrid": {"semanticRatio": 0.0}, "showRankingScore": true}))
+    .search_post(json!({"vector": [1.0, 0.0], "hybrid": {"semanticRatio": 0.0}, "showRankingScore": true, "retrieveVectors": true}))
     .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":1.0}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":1.0},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":1.0}]"###);
     snapshot!(response["semanticHitCount"], @"null");
 
     // query + vector, full keyword => keyword
     let (response, code) = index
-    .search_post(json!({"q": "Captain", "vector": [1.0, 0.0], "hybrid": {"semanticRatio": 0.0}, "showRankingScore": true}))
+    .search_post(json!({"q": "Captain", "vector": [1.0, 0.0], "hybrid": {"semanticRatio": 0.0}, "showRankingScore": true, "retrieveVectors": true}))
     .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_rankingScore":0.9242424242424242}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.9848484848484848},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9242424242424242}]"###);
     snapshot!(response["semanticHitCount"], @"null");
 
     // query + vector, no hybrid keyword =>
     let (response, code) = index
-        .search_post(json!({"q": "Captain", "vector": [1.0, 0.0], "showRankingScore": true}))
+        .search_post(json!({"q": "Captain", "vector": [1.0, 0.0], "showRankingScore": true, "retrieveVectors": true}))
         .await;
 
     snapshot!(code, @"400 Bad Request");
@@ -453,7 +457,7 @@ async fn query_combination() {
     // full vector, without a vector => error
     let (response, code) = index
         .search_post(
-            json!({"q": "Captain", "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true}),
+            json!({"q": "Captain", "hybrid": {"semanticRatio": 1.0}, "showRankingScore": true, "retrieveVectors": true}),
         )
         .await;
 
@@ -470,11 +474,93 @@ async fn query_combination() {
     // hybrid without a vector => full keyword
     let (response, code) = index
         .search_post(
-            json!({"q": "Planet", "hybrid": {"semanticRatio": 0.99}, "showRankingScore": true}),
+            json!({"q": "Planet", "hybrid": {"semanticRatio": 0.99}, "showRankingScore": true, "retrieveVectors": true}),
         )
         .await;
 
     snapshot!(code, @"200 OK");
-    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_rankingScore":0.9242424242424242}]"###);
+    snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.9242424242424242}]"###);
     snapshot!(response["semanticHitCount"], @"0");
 }
+
+#[actix_rt::test]
+async fn retrieve_vectors() {
+    let server = Server::new().await;
+    let index = index_with_documents_hf(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
+
+    let (response, code) = index
+        .search_post(
+            json!({"q": "Captain", "hybrid": {"semanticRatio": 0.2}, "retrieveVectors": true}),
+        )
+        .await;
+    snapshot!(code, @"200 OK");
+    insta::assert_json_snapshot!(response["hits"], {"[]._vectors.default.embeddings" => "[vectors]"},  @r###"
+    [
+      {
+        "title": "Captain Planet",
+        "desc": "He's not part of the Marvel Cinematic Universe",
+        "id": "2",
+        "_vectors": {
+          "default": {
+            "embeddings": "[vectors]",
+            "regenerate": true
+          }
+        }
+      },
+      {
+        "title": "Captain Marvel",
+        "desc": "a Shazam ersatz",
+        "id": "3",
+        "_vectors": {
+          "default": {
+            "embeddings": "[vectors]",
+            "regenerate": true
+          }
+        }
+      },
+      {
+        "title": "Shazam!",
+        "desc": "a Captain Marvel ersatz",
+        "id": "1",
+        "_vectors": {
+          "default": {
+            "embeddings": "[vectors]",
+            "regenerate": true
+          }
+        }
+      }
+    ]
+    "###);
+
+    // remove `_vectors` from displayed attributes
+    let (response, code) =
+        index.update_settings(json!({ "displayedAttributes": ["id", "title", "desc"]} )).await;
+    assert_eq!(202, code, "{:?}", response);
+    index.wait_task(response.uid()).await;
+
+    let (response, code) = index
+        .search_post(
+            json!({"q": "Captain", "hybrid": {"semanticRatio": 0.2}, "retrieveVectors": true}),
+        )
+        .await;
+    snapshot!(code, @"200 OK");
+    insta::assert_json_snapshot!(response["hits"], {"[]._vectors.default.embeddings" => "[vectors]"},  @r###"
+    [
+      {
+        "title": "Captain Planet",
+        "desc": "He's not part of the Marvel Cinematic Universe",
+        "id": "2"
+      },
+      {
+        "title": "Captain Marvel",
+        "desc": "a Shazam ersatz",
+        "id": "3"
+      },
+      {
+        "title": "Shazam!",
+        "desc": "a Captain Marvel ersatz",
+        "id": "1"
+      }
+    ]
+    "###);
+}

meilisearch/tests/search/mod.rs

@@ -1290,21 +1290,38 @@ async fn experimental_feature_vector_store() {
     index.add_documents(json!(documents), None).await;
     index.wait_task(0).await;
 
-    let (response, code) = index
-        .search_post(json!({
+    index
+        .search(json!({
             "vector": [1.0, 2.0, 3.0],
             "showRankingScore": true
-        }))
+        }), |response, code|{
+            meili_snap::snapshot!(code, @"400 Bad Request");
+            meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+            {
+              "message": "Passing `vector` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+              "code": "feature_not_enabled",
+              "type": "invalid_request",
+              "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+            }
+            "###);
+        })
+        .await;
+    index
+        .search(json!({
+            "retrieveVectors": true,
+            "showRankingScore": true
+        }), |response, code|{
+            meili_snap::snapshot!(code, @"400 Bad Request");
+            meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+            {
+              "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+              "code": "feature_not_enabled",
+              "type": "invalid_request",
+              "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+            }
+            "###);
+        })
         .await;
-    meili_snap::snapshot!(code, @"400 Bad Request");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "message": "Passing `vector` as a query parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
-      "code": "feature_not_enabled",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
-    }
-    "###);
 
     let (response, code) = server.set_features(json!({"vectorStore": true})).await;
     meili_snap::snapshot!(code, @"200 OK");
@@ -1337,6 +1354,7 @@ async fn experimental_feature_vector_store() {
         .search_post(json!({
             "vector": [1.0, 2.0, 3.0],
             "showRankingScore": true,
+            "retrieveVectors": true,
         }))
         .await;
 
@@ -1348,11 +1366,16 @@ async fn experimental_feature_vector_store() {
         "title": "Shazam!",
         "id": "287947",
         "_vectors": {
-          "manual": [
-            1.0,
-            2.0,
-            3.0
-          ]
+          "manual": {
+            "embeddings": [
+              [
+                1.0,
+                2.0,
+                3.0
+              ]
+            ],
+            "regenerate": false
+          }
         },
         "_rankingScore": 1.0
       },
@@ -1360,11 +1383,16 @@ async fn experimental_feature_vector_store() {
         "title": "Captain Marvel",
         "id": "299537",
         "_vectors": {
-          "manual": [
-            1.0,
-            2.0,
-            54.0
-          ]
+          "manual": {
+            "embeddings": [
+              [
+                1.0,
+                2.0,
+                54.0
+              ]
+            ],
+            "regenerate": false
+          }
         },
         "_rankingScore": 0.9129111766815186
       },
@@ -1372,11 +1400,16 @@ async fn experimental_feature_vector_store() {
         "title": "Gläss",
         "id": "450465",
         "_vectors": {
-          "manual": [
-            -100.0,
-            340.0,
-            90.0
-          ]
+          "manual": {
+            "embeddings": [
+              [
+                -100.0,
+                340.0,
+                90.0
+              ]
+            ],
+            "regenerate": false
+          }
         },
         "_rankingScore": 0.8106412887573242
       },
@@ -1384,11 +1417,16 @@ async fn experimental_feature_vector_store() {
         "title": "How to Train Your Dragon: The Hidden World",
         "id": "166428",
         "_vectors": {
-          "manual": [
-            -100.0,
-            231.0,
-            32.0
-          ]
+          "manual": {
+            "embeddings": [
+              [
+                -100.0,
+                231.0,
+                32.0
+              ]
+            ],
+            "regenerate": false
+          }
         },
         "_rankingScore": 0.7412010431289673
       },
@@ -1396,11 +1434,16 @@ async fn experimental_feature_vector_store() {
         "title": "Escape Room",
         "id": "522681",
         "_vectors": {
-          "manual": [
-            10.0,
-            -23.0,
-            32.0
-          ]
+          "manual": {
+            "embeddings": [
+              [
+                10.0,
+                -23.0,
+                32.0
+              ]
+            ],
+            "regenerate": false
+          }
         },
         "_rankingScore": 0.6972063183784485
       }

meilisearch/tests/similar/errors.rs

@@ -756,3 +756,54 @@ async fn filter_reserved_geo_point_string() {
         })
         .await;
 }
+
+#[actix_rt::test]
+async fn similar_bad_retrieve_vectors() {
+    let server = Server::new().await;
+    server.set_features(json!({"vectorStore": true})).await;
+    let index = server.index("test");
+
+    let (response, code) = index.similar_post(json!({"retrieveVectors": "doggo"})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found a string: `\"doggo\"`",
+      "code": "invalid_similar_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_similar_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.similar_post(json!({"retrieveVectors": [true]})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found an array: `[true]`",
+      "code": "invalid_similar_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_similar_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.similar_get("retrieveVectors=").await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `` as a boolean, expected either `true` or `false`",
+      "code": "invalid_similar_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_similar_retrieve_vectors"
+    }
+    "###);
+
+    let (response, code) = index.similar_get("retrieveVectors=doggo").await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `doggo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_similar_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_similar_retrieve_vectors"
+    }
+    "###);
+}

meilisearch/tests/similar/mod.rs

@@ -78,7 +78,7 @@ async fn basic() {
     index.wait_task(value.uid()).await;
 
     index
-        .similar(json!({"id": 143}), |response, code| {
+        .similar(json!({"id": 143, "retrieveVectors": true}), |response, code| {
             snapshot!(code, @"200 OK");
             snapshot!(json_string!(response["hits"]), @r###"
             [
@@ -87,11 +87,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "522681",
                 "_vectors": {
-                  "manual": [
-                    0.1,
-                    0.6,
-                    0.8
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.10000000149011612,
+                        0.6000000238418579,
+                        0.800000011920929
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -99,11 +104,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "299537",
                 "_vectors": {
-                  "manual": [
-                    0.6,
-                    0.8,
-                    -0.2
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.6000000238418579,
+                        0.800000011920929,
+                        -0.20000000298023224
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -111,11 +121,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "166428",
                 "_vectors": {
-                  "manual": [
-                    0.7,
-                    0.7,
-                    -0.4
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.699999988079071,
+                        0.699999988079071,
+                        -0.4000000059604645
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -123,11 +138,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "287947",
                 "_vectors": {
-                  "manual": [
-                    0.8,
-                    0.4,
-                    -0.5
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.800000011920929,
+                        0.4000000059604645,
+                        -0.5
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               }
             ]
@@ -136,7 +156,7 @@ async fn basic() {
         .await;
 
     index
-        .similar(json!({"id": "299537"}), |response, code| {
+        .similar(json!({"id": "299537", "retrieveVectors": true}), |response, code| {
             snapshot!(code, @"200 OK");
             snapshot!(json_string!(response["hits"]), @r###"
             [
@@ -145,11 +165,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "166428",
                 "_vectors": {
-                  "manual": [
-                    0.7,
-                    0.7,
-                    -0.4
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.699999988079071,
+                        0.699999988079071,
+                        -0.4000000059604645
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -157,11 +182,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "287947",
                 "_vectors": {
-                  "manual": [
-                    0.8,
-                    0.4,
-                    -0.5
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.800000011920929,
+                        0.4000000059604645,
+                        -0.5
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -169,11 +199,16 @@ async fn basic() {
                 "release_year": 2019,
                 "id": "522681",
                 "_vectors": {
-                  "manual": [
-                    0.1,
-                    0.6,
-                    0.8
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.10000000149011612,
+                        0.6000000238418579,
+                        0.800000011920929
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               },
               {
@@ -181,11 +216,16 @@ async fn basic() {
                 "release_year": 1930,
                 "id": "143",
                 "_vectors": {
-                  "manual": [
-                    -0.5,
-                    0.3,
-                    0.85
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        -0.5,
+                        0.30000001192092896,
+                        0.8500000238418579
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               }
             ]
@@ -228,7 +268,7 @@ async fn ranking_score_threshold() {
 
     index
         .similar(
-            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0}),
+            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0, "retrieveVectors": true}),
             |response, code| {
                 snapshot!(code, @"200 OK");
                 meili_snap::snapshot!(meili_snap::json_string!(response["estimatedTotalHits"]), @"4");
@@ -239,11 +279,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "522681",
                     "_vectors": {
-                      "manual": [
-                        0.1,
-                        0.6,
-                        0.8
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.10000000149011612,
+                            0.6000000238418579,
+                            0.800000011920929
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.890957772731781
                   },
@@ -252,11 +297,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "299537",
                     "_vectors": {
-                      "manual": [
-                        0.6,
-                        0.8,
-                        -0.2
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.6000000238418579,
+                            0.800000011920929,
+                            -0.20000000298023224
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.39060014486312866
                   },
@@ -265,11 +315,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "166428",
                     "_vectors": {
-                      "manual": [
-                        0.7,
-                        0.7,
-                        -0.4
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.699999988079071,
+                            0.699999988079071,
+                            -0.4000000059604645
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.2819308042526245
                   },
@@ -278,11 +333,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "287947",
                     "_vectors": {
-                      "manual": [
-                        0.8,
-                        0.4,
-                        -0.5
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.800000011920929,
+                            0.4000000059604645,
+                            -0.5
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.1662663221359253
                   }
@@ -294,7 +354,7 @@ async fn ranking_score_threshold() {
 
     index
         .similar(
-            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.2}),
+            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.2, "retrieveVectors": true}),
             |response, code| {
                 snapshot!(code, @"200 OK");
                 meili_snap::snapshot!(meili_snap::json_string!(response["estimatedTotalHits"]), @"3");
@@ -305,11 +365,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "522681",
                     "_vectors": {
-                      "manual": [
-                        0.1,
-                        0.6,
-                        0.8
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.10000000149011612,
+                            0.6000000238418579,
+                            0.800000011920929
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.890957772731781
                   },
@@ -318,11 +383,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "299537",
                     "_vectors": {
-                      "manual": [
-                        0.6,
-                        0.8,
-                        -0.2
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.6000000238418579,
+                            0.800000011920929,
+                            -0.20000000298023224
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.39060014486312866
                   },
@@ -331,11 +401,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "166428",
                     "_vectors": {
-                      "manual": [
-                        0.7,
-                        0.7,
-                        -0.4
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.699999988079071,
+                            0.699999988079071,
+                            -0.4000000059604645
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.2819308042526245
                   }
@@ -347,7 +422,7 @@ async fn ranking_score_threshold() {
 
     index
         .similar(
-            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.3}),
+            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.3, "retrieveVectors": true}),
             |response, code| {
                 snapshot!(code, @"200 OK");
                 meili_snap::snapshot!(meili_snap::json_string!(response["estimatedTotalHits"]), @"2");
@@ -358,11 +433,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "522681",
                     "_vectors": {
-                      "manual": [
-                        0.1,
-                        0.6,
-                        0.8
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.10000000149011612,
+                            0.6000000238418579,
+                            0.800000011920929
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.890957772731781
                   },
@@ -371,11 +451,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "299537",
                     "_vectors": {
-                      "manual": [
-                        0.6,
-                        0.8,
-                        -0.2
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.6000000238418579,
+                            0.800000011920929,
+                            -0.20000000298023224
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.39060014486312866
                   }
@@ -387,7 +472,7 @@ async fn ranking_score_threshold() {
 
     index
         .similar(
-            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.6}),
+            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.6, "retrieveVectors": true}),
             |response, code| {
                 snapshot!(code, @"200 OK");
                 meili_snap::snapshot!(meili_snap::json_string!(response["estimatedTotalHits"]), @"1");
@@ -398,11 +483,16 @@ async fn ranking_score_threshold() {
                     "release_year": 2019,
                     "id": "522681",
                     "_vectors": {
-                      "manual": [
-                        0.1,
-                        0.6,
-                        0.8
-                      ]
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.10000000149011612,
+                            0.6000000238418579,
+                            0.800000011920929
+                          ]
+                        ],
+                        "regenerate": false
+                      }
                     },
                     "_rankingScore": 0.890957772731781
                   }
@@ -414,7 +504,7 @@ async fn ranking_score_threshold() {
 
     index
         .similar(
-            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.9}),
+            json!({"id": 143, "showRankingScore": true, "rankingScoreThreshold": 0.9, "retrieveVectors": true}),
             |response, code| {
                 snapshot!(code, @"200 OK");
                 snapshot!(json_string!(response["hits"]), @"[]");
@@ -456,71 +546,97 @@ async fn filter() {
     index.wait_task(value.uid()).await;
 
     index
-        .similar(json!({"id": 522681, "filter": "release_year = 2019"}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "Captain Marvel",
-                "release_year": 2019,
-                "id": "299537",
-                "_vectors": {
-                  "manual": [
-                    0.6,
-                    0.8,
-                    -0.2
-                  ]
-                }
-              },
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "release_year": 2019,
-                "id": "166428",
-                "_vectors": {
-                  "manual": [
-                    0.7,
-                    0.7,
-                    -0.4
-                  ]
-                }
-              },
-              {
-                "title": "Shazam!",
-                "release_year": 2019,
-                "id": "287947",
-                "_vectors": {
-                  "manual": [
-                    0.8,
-                    0.4,
-                    -0.5
-                  ]
-                }
-              }
-            ]
-            "###);
-        })
+        .similar(
+            json!({"id": 522681, "filter": "release_year = 2019", "retrieveVectors": true}),
+            |response, code| {
+                snapshot!(code, @"200 OK");
+                snapshot!(json_string!(response["hits"]), @r###"
+                [
+                  {
+                    "title": "Captain Marvel",
+                    "release_year": 2019,
+                    "id": "299537",
+                    "_vectors": {
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.6000000238418579,
+                            0.800000011920929,
+                            -0.20000000298023224
+                          ]
+                        ],
+                        "regenerate": false
+                      }
+                    }
+                  },
+                  {
+                    "title": "How to Train Your Dragon: The Hidden World",
+                    "release_year": 2019,
+                    "id": "166428",
+                    "_vectors": {
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.699999988079071,
+                            0.699999988079071,
+                            -0.4000000059604645
+                          ]
+                        ],
+                        "regenerate": false
+                      }
+                    }
+                  },
+                  {
+                    "title": "Shazam!",
+                    "release_year": 2019,
+                    "id": "287947",
+                    "_vectors": {
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.800000011920929,
+                            0.4000000059604645,
+                            -0.5
+                          ]
+                        ],
+                        "regenerate": false
+                      }
+                    }
+                  }
+                ]
+                "###);
+            },
+        )
         .await;
 
     index
-        .similar(json!({"id": 522681, "filter": "release_year < 2000"}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "All Quiet on the Western Front",
-                "release_year": 1930,
-                "id": "143",
-                "_vectors": {
-                  "manual": [
-                    -0.5,
-                    0.3,
-                    0.85
-                  ]
-                }
-              }
-            ]
-            "###);
-        })
+        .similar(
+            json!({"id": 522681, "filter": "release_year < 2000", "retrieveVectors": true}),
+            |response, code| {
+                snapshot!(code, @"200 OK");
+                snapshot!(json_string!(response["hits"]), @r###"
+                [
+                  {
+                    "title": "All Quiet on the Western Front",
+                    "release_year": 1930,
+                    "id": "143",
+                    "_vectors": {
+                      "manual": {
+                        "embeddings": [
+                          [
+                            -0.5,
+                            0.30000001192092896,
+                            0.8500000238418579
+                          ]
+                        ],
+                        "regenerate": false
+                      }
+                    }
+                  }
+                ]
+                "###);
+            },
+        )
         .await;
 }
 
@@ -557,7 +673,7 @@ async fn limit_and_offset() {
     index.wait_task(value.uid()).await;
 
     index
-        .similar(json!({"id": 143, "limit": 1}), |response, code| {
+        .similar(json!({"id": 143, "limit": 1, "retrieveVectors": true}), |response, code| {
             snapshot!(code, @"200 OK");
             snapshot!(json_string!(response["hits"]), @r###"
             [
@@ -566,11 +682,16 @@ async fn limit_and_offset() {
                 "release_year": 2019,
                 "id": "522681",
                 "_vectors": {
-                  "manual": [
-                    0.1,
-                    0.6,
-                    0.8
-                  ]
+                  "manual": {
+                    "embeddings": [
+                      [
+                        0.10000000149011612,
+                        0.6000000238418579,
+                        0.800000011920929
+                      ]
+                    ],
+                    "regenerate": false
+                  }
                 }
               }
             ]
@@ -579,24 +700,32 @@ async fn limit_and_offset() {
         .await;
 
     index
-        .similar(json!({"id": 143, "limit": 1, "offset": 1}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "Captain Marvel",
-                "release_year": 2019,
-                "id": "299537",
-                "_vectors": {
-                  "manual": [
-                    0.6,
-                    0.8,
-                    -0.2
-                  ]
-                }
-              }
-            ]
-            "###);
-        })
+        .similar(
+            json!({"id": 143, "limit": 1, "offset": 1, "retrieveVectors": true}),
+            |response, code| {
+                snapshot!(code, @"200 OK");
+                snapshot!(json_string!(response["hits"]), @r###"
+                [
+                  {
+                    "title": "Captain Marvel",
+                    "release_year": 2019,
+                    "id": "299537",
+                    "_vectors": {
+                      "manual": {
+                        "embeddings": [
+                          [
+                            0.6000000238418579,
+                            0.800000011920929,
+                            -0.20000000298023224
+                          ]
+                        ],
+                        "regenerate": false
+                      }
+                    }
+                  }
+                ]
+                "###);
+            },
+        )
         .await;
 }

meilisearch/tests/vector/mod.rs

@@ -0,0 +1,227 @@
+mod settings;
+
+use meili_snap::{json_string, snapshot};
+
+use crate::common::index::Index;
+use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
+
+#[actix_rt::test]
+async fn add_remove_user_provided() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+    let (value, code) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(value, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "manual": {
+                  "source": "userProvided",
+                  "dimensions": 3,
+              }
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    let documents = json!([
+      {"id": 0, "name": "kefir", "_vectors": { "manual": [0, 0, 0] }},
+      {"id": 1, "name": "echo", "_vectors": { "manual": [1, 1, 1] }},
+    ]);
+    let (value, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  1.0,
+                  1.0,
+                  1.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+
+    let documents = json!([
+      {"id": 0, "name": "kefir", "_vectors": { "manual": [10, 10, 10] }},
+      {"id": 1, "name": "echo", "_vectors": { "manual": null }},
+    ]);
+    let (value, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  10.0,
+                  10.0,
+                  10.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+
+    let (value, code) = index.delete_document(0).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 1
+    }
+    "###);
+}
+
+async fn generate_default_user_provided_documents(server: &Server) -> Index {
+    let index = server.index("doggo");
+    let (value, code) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(value, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "manual": {
+                  "source": "userProvided",
+                  "dimensions": 3,
+              }
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    let documents = json!([
+      {"id": 0, "name": "kefir", "_vectors": { "manual": [0, 0, 0] }},
+      {"id": 1, "name": "echo", "_vectors": { "manual": [1, 1, 1] }},
+      {"id": 2, "name": "billou", "_vectors": { "manual": [[2, 2, 2], [2, 2, 3]] }},
+      {"id": 3, "name": "intel", "_vectors": { "manual": { "regenerate": false, "embeddings": [3, 3, 3] }}},
+      {"id": 4, "name": "max", "_vectors": { "manual": { "regenerate": false, "embeddings": [[4, 4, 4], [4, 4, 5]] }}},
+    ]);
+    let (value, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    index
+}
+
+#[actix_rt::test]
+async fn clear_documents() {
+    let server = Server::new().await;
+    let index = generate_default_user_provided_documents(&server).await;
+
+    let (value, _code) = index.clear_all_documents().await;
+    index.wait_task(value.uid()).await;
+
+    // Make sure the documents DB has been cleared
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [],
+      "offset": 0,
+      "limit": 20,
+      "total": 0
+    }
+    "###);
+
+    // Make sure the arroy DB has been cleared
+    let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1] })).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "hits": [],
+      "query": "",
+      "processingTimeMs": 0,
+      "limit": 20,
+      "offset": 0,
+      "estimatedTotalHits": 0,
+      "semanticHitCount": 0
+    }
+    "###);
+}

meilisearch/tests/vector/settings.rs

@@ -0,0 +1,228 @@
+use meili_snap::{json_string, snapshot};
+
+use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
+use crate::vector::generate_default_user_provided_documents;
+
+#[actix_rt::test]
+async fn update_embedder() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+    let (value, code) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(value, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false
+    }
+    "###);
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": { "manual": {}},
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "manual": {
+                  "source": "userProvided",
+                  "dimensions": 2,
+              }
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+
+    let ret = server.wait_task(response.uid()).await;
+    snapshot!(ret, @r###"
+    {
+      "uid": 1,
+      "indexUid": "doggo",
+      "status": "succeeded",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "embedders": {
+          "manual": {
+            "source": "userProvided",
+            "dimensions": 2
+          }
+        }
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn reset_embedder_documents() {
+    let server = Server::new().await;
+    let index = generate_default_user_provided_documents(&server).await;
+
+    let (response, code) = index.delete_settings().await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    // Make sure the documents are still present
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            limit: None,
+            offset: None,
+            retrieve_vectors: false,
+            fields: None,
+        })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir"
+        },
+        {
+          "id": 1,
+          "name": "echo"
+        },
+        {
+          "id": 2,
+          "name": "billou"
+        },
+        {
+          "id": 3,
+          "name": "intel"
+        },
+        {
+          "id": 4,
+          "name": "max"
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 5
+    }
+    "###);
+
+    // Make sure we are still able to retrieve their vectors
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  1.0,
+                  1.0,
+                  1.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 2,
+          "name": "billou",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  2.0,
+                  2.0,
+                  2.0
+                ],
+                [
+                  2.0,
+                  2.0,
+                  3.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 3,
+          "name": "intel",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  3.0,
+                  3.0,
+                  3.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 4,
+          "name": "max",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  4.0,
+                  4.0,
+                  4.0
+                ],
+                [
+                  4.0,
+                  4.0,
+                  5.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 5
+    }
+    "###);
+
+    // Make sure the arroy DB has been cleared
+    let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1] })).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "message": "Cannot find embedder with name `default`.",
+      "code": "invalid_embedder",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_embedder"
+    }
+    "###);
+}