Merge #4937

4937: Support iso 639 1 r=ManyTheFish a=ManyTheFish

# Pull Request

## Related issue
Fixes #4827

## What does this PR do?
- Add iso-639-1 variants to the Locales enum
- Convert iso-639-1 into iso-639-3


Co-authored-by: ManyTheFish <many@meilisearch.com>

meilisearch-types/src/locales.rs

@@ -1,134 +1,6 @@
 use deserr::Deserr;
 use milli::LocalizedAttributesRule;
 use serde::{Deserialize, Serialize};
-use serde_json::json;
-
-/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
-///
-/// this enum implements `Deserr` in order to be used in the API.
-macro_rules! make_locale {
-
-    ($($language:tt), +) => {
-        #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
-        #[deserr(rename_all = camelCase)]
-        #[serde(rename_all = "camelCase")]
-        pub enum Locale {
-            $($language),+,
-        }
-
-        impl From<milli::tokenizer::Language> for Locale {
-            fn from(other: milli::tokenizer::Language) -> Locale {
-                match other {
-                    $(milli::tokenizer::Language::$language => Locale::$language), +
-                }
-            }
-        }
-
-        impl From<Locale> for milli::tokenizer::Language {
-            fn from(other: Locale) -> milli::tokenizer::Language {
-                match other {
-                    $(Locale::$language => milli::tokenizer::Language::$language), +,
-                }
-            }
-        }
-
-        #[derive(Debug)]
-        pub struct LocaleFormatError {
-            pub invalid_locale: String,
-        }
-
-        impl std::fmt::Display for LocaleFormatError {
-            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                let valid_locales = [$(Locale::$language),+].iter().map(|l| format!("`{}`", json!(l).as_str().unwrap())).collect::<Vec<_>>().join(", ");
-                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
-            }
-        }
-    };
-}
-
-make_locale! {
-    Epo,
-    Eng,
-    Rus,
-    Cmn,
-    Spa,
-    Por,
-    Ita,
-    Ben,
-    Fra,
-    Deu,
-    Ukr,
-    Kat,
-    Ara,
-    Hin,
-    Jpn,
-    Heb,
-    Yid,
-    Pol,
-    Amh,
-    Jav,
-    Kor,
-    Nob,
-    Dan,
-    Swe,
-    Fin,
-    Tur,
-    Nld,
-    Hun,
-    Ces,
-    Ell,
-    Bul,
-    Bel,
-    Mar,
-    Kan,
-    Ron,
-    Slv,
-    Hrv,
-    Srp,
-    Mkd,
-    Lit,
-    Lav,
-    Est,
-    Tam,
-    Vie,
-    Urd,
-    Tha,
-    Guj,
-    Uzb,
-    Pan,
-    Aze,
-    Ind,
-    Tel,
-    Pes,
-    Mal,
-    Ori,
-    Mya,
-    Nep,
-    Sin,
-    Khm,
-    Tuk,
-    Aka,
-    Zul,
-    Sna,
-    Afr,
-    Lat,
-    Slk,
-    Cat,
-    Tgl,
-    Hye
-}
-
-impl std::error::Error for LocaleFormatError {}
-
-impl std::str::FromStr for Locale {
-    type Err = LocaleFormatError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        milli::tokenizer::Language::from_code(s)
-            .map(Self::from)
-            .ok_or(LocaleFormatError { invalid_locale: s.to_string() })
-    }
-}
 
 #[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)]
 #[deserr(rename_all = camelCase)]
@@ -155,3 +27,135 @@ impl From<LocalizedAttributesRuleView> for LocalizedAttributesRule {
         }
     }
 }
+
+/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
+///
+/// this enum implements `Deserr` in order to be used in the API.
+macro_rules! make_locale {
+    ($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => {
+        #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
+        #[deserr(rename_all = camelCase)]
+        #[serde(rename_all = "camelCase")]
+        pub enum Locale {
+            $($iso_639_1,)+
+            $($iso_639_3,)+
+        }
+
+        impl From<milli::tokenizer::Language> for Locale {
+            fn from(other: milli::tokenizer::Language) -> Locale {
+                match other {
+                    $(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
+                }
+            }
+        }
+
+        impl From<Locale> for milli::tokenizer::Language {
+            fn from(other: Locale) -> milli::tokenizer::Language {
+                match other {
+                    $(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
+                    $(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
+                }
+            }
+        }
+
+        impl std::str::FromStr for Locale {
+            type Err = LocaleFormatError;
+
+            fn from_str(s: &str) -> Result<Self, Self::Err> {
+                let locale = match s {
+                    $($iso_639_1_str => Locale::$iso_639_1,)+
+                    $($iso_639_3_str => Locale::$iso_639_3,)+
+                    _ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
+                };
+
+                Ok(locale)
+            }
+        }
+
+        #[derive(Debug)]
+        pub struct LocaleFormatError {
+            pub invalid_locale: String,
+        }
+
+        impl std::fmt::Display for LocaleFormatError {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                let valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+].join(", ");
+                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
+            }
+        }
+
+        impl std::error::Error for LocaleFormatError {}
+    };
+}
+
+make_locale!(
+    (Af, "af") => (Afr, "afr"),
+    (Ak, "ak") => (Aka, "aka"),
+    (Am, "am") => (Amh, "amh"),
+    (Ar, "ar") => (Ara, "ara"),
+    (Az, "az") => (Aze, "aze"),
+    (Be, "be") => (Bel, "bel"),
+    (Bn, "bn") => (Ben, "ben"),
+    (Bg, "bg") => (Bul, "bul"),
+    (Ca, "ca") => (Cat, "cat"),
+    (Cs, "cs") => (Ces, "ces"),
+    (Zh, "zh") => (Cmn, "cmn"),
+    (Da, "da") => (Dan, "dan"),
+    (De, "de") => (Deu, "deu"),
+    (El, "el") => (Ell, "ell"),
+    (En, "en") => (Eng, "eng"),
+    (Eo, "eo") => (Epo, "epo"),
+    (Et, "et") => (Est, "est"),
+    (Fi, "fi") => (Fin, "fin"),
+    (Fr, "fr") => (Fra, "fra"),
+    (Gu, "gu") => (Guj, "guj"),
+    (He, "he") => (Heb, "heb"),
+    (Hi, "hi") => (Hin, "hin"),
+    (Hr, "hr") => (Hrv, "hrv"),
+    (Hu, "hu") => (Hun, "hun"),
+    (Hy, "hy") => (Hye, "hye"),
+    (Id, "id") => (Ind, "ind"),
+    (It, "it") => (Ita, "ita"),
+    (Jv, "jv") => (Jav, "jav"),
+    (Ja, "ja") => (Jpn, "jpn"),
+    (Kn, "kn") => (Kan, "kan"),
+    (Ka, "ka") => (Kat, "kat"),
+    (Km, "km") => (Khm, "khm"),
+    (Ko, "ko") => (Kor, "kor"),
+    (La, "la") => (Lat, "lat"),
+    (Lv, "lv") => (Lav, "lav"),
+    (Lt, "lt") => (Lit, "lit"),
+    (Ml, "ml") => (Mal, "mal"),
+    (Mr, "mr") => (Mar, "mar"),
+    (Mk, "mk") => (Mkd, "mkd"),
+    (My, "my") => (Mya, "mya"),
+    (Ne, "ne") => (Nep, "nep"),
+    (Nl, "nl") => (Nld, "nld"),
+    (Nb, "nb") => (Nob, "nob"),
+    (Or, "or") => (Ori, "ori"),
+    (Pa, "pa") => (Pan, "pan"),
+    (Fa, "fa") => (Pes, "pes"),
+    (Pl, "pl") => (Pol, "pol"),
+    (Pt, "pt") => (Por, "por"),
+    (Ro, "ro") => (Ron, "ron"),
+    (Ru, "ru") => (Rus, "rus"),
+    (Si, "si") => (Sin, "sin"),
+    (Sk, "sk") => (Slk, "slk"),
+    (Sl, "sl") => (Slv, "slv"),
+    (Sn, "sn") => (Sna, "sna"),
+    (Es, "es") => (Spa, "spa"),
+    (Sr, "sr") => (Srp, "srp"),
+    (Sv, "sv") => (Swe, "swe"),
+    (Ta, "ta") => (Tam, "tam"),
+    (Te, "te") => (Tel, "tel"),
+    (Tl, "tl") => (Tgl, "tgl"),
+    (Th, "th") => (Tha, "tha"),
+    (Tk, "tk") => (Tuk, "tuk"),
+    (Tr, "tr") => (Tur, "tur"),
+    (Uk, "uk") => (Ukr, "ukr"),
+    (Ur, "ur") => (Urd, "urd"),
+    (Uz, "uz") => (Uzb, "uzb"),
+    (Vi, "vi") => (Vie, "vie"),
+    (Yi, "yi") => (Yid, "yid"),
+    (Zu, "zu") => (Zul, "zul"),
+);

meilisearch/tests/search/locales.rs

@@ -103,41 +103,12 @@ async fn simple_search() {
 
     // english
     index
-        .search(json!({"q": "Atta", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "Atta", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_en": "Attack on Titan",
+                  "id": 852
-                  "name_ja": "進撃の巨人",
-                  "author_en": "Hajime Isayama",
-                  "author_ja": "諫山 創",
-                  "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                  "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                  "id": 852,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_en": "<em>Atta</em>ck on Titan",
-                    "name_ja": "進撃の巨人",
-                    "author_en": "Hajime Isayama",
-                    "author_ja": "諫山 創",
-                    "description_en": "<em>Atta</em>ck on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                    "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                    "id": "852",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
                 }
               ],
               "query": "Atta",
@@ -153,35 +124,12 @@ async fn simple_search() {
 
     // japanese
     index
-        .search(json!({"q": "進撃", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "進撃", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_zh": "进击的巨人",
+                  "id": 853
-                  "author_zh": "諫山創",
-                  "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                  "id": 853,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_zh": "<em>进击</em>的巨人",
-                    "author_zh": "諫山創",
-                    "description_zh": "<em>进击</em>的巨人是日本的漫画系列，由諫山 創作画。",
-                    "id": "853",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
                 }
               ],
               "query": "進撃",
@@ -197,68 +145,16 @@ async fn simple_search() {
 
     index
         .search(
-            json!({"q": "進撃", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "進撃", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_en": "Attack on Titan",
+                      "id": 852
-                      "name_ja": "進撃の巨人",
-                      "author_en": "Hajime Isayama",
-                      "author_ja": "諫山 創",
-                      "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                      "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_en": "Attack on Titan",
-                        "name_ja": "<em>進撃</em>の巨人",
-                        "author_en": "Hajime Isayama",
-                        "author_ja": "諫山 創",
-                        "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     },
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "进击的巨人",
-                        "author_zh": "諫山創",
-                        "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "進撃",
@@ -275,67 +171,15 @@ async fn simple_search() {
 
     // chinese
     index
-        .search(json!({"q": "进击", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "进击", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_zh": "进击的巨人",
+                  "id": 853
-                  "author_zh": "諫山創",
-                  "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                  "id": 853,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_zh": "<em>进击</em>的巨人",
-                    "author_zh": "諫山創",
-                    "description_zh": "<em>进击</em>的巨人是日本的漫画系列，由諫山 創作画。",
-                    "id": "853",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
                 },
                 {
-                  "name_en": "Attack on Titan",
+                  "id": 852
-                  "name_ja": "進撃の巨人",
-                  "author_en": "Hajime Isayama",
-                  "author_ja": "諫山 創",
-                  "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                  "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                  "id": 852,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_en": "Attack on Titan",
-                    "name_ja": "<em>進撃</em>の巨人",
-                    "author_en": "Hajime Isayama",
-                    "author_ja": "諫山 創",
-                    "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                    "description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                    "id": "852",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
                 }
               ],
               "query": "进击",
@@ -382,36 +226,13 @@ async fn force_locales() {
     // chinese detection
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -429,36 +250,13 @@ async fn force_locales() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -506,36 +304,13 @@ async fn force_locales_with_pattern() {
     // chinese detection
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -553,36 +328,13 @@ async fn force_locales_with_pattern() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -628,7 +380,7 @@ async fn force_locales_with_pattern_nested() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -648,60 +400,13 @@ async fn force_locales_with_pattern_nested() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "document_en": {
+                      "id": 852
-                        "name": "Attack on Titan",
-                        "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "author": "Hajime Isayama"
-                      },
-                      "document_ja": {
-                        "name": "進撃の巨人",
-                        "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "author": "諫山 創"
-                      },
-                      "document_zh": {
-                        "name": "进击的巨人",
-                        "description": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "author": "諫山創"
-                      },
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "document_en": {
-                          "name": "Attack on Titan",
-                          "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                          "author": "Hajime Isayama"
-                        },
-                        "document_ja": {
-                          "name": "進撃の巨人",
-                          "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                          "author": "諫山 創"
-                        },
-                        "document_zh": {
-                          "name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                          "description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                          "author": "諫山創"
-                        },
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -750,7 +455,7 @@ async fn force_different_locales_with_pattern() {
     // force chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -770,36 +475,13 @@ async fn force_different_locales_with_pattern() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -851,7 +533,7 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
     // auto infer any language
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -871,36 +553,13 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
     // should infer chinese
     index
             .search(
-                json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"], "attributesToSearchOn": ["name_zh", "description_zh"]}),
+                json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"], "attributesToSearchOn": ["name_zh", "description_zh"]}),
                 |response, code| {
                     snapshot!(response, @r###"
                     {
                       "hits": [
                         {
-                          "name_zh": "进击的巨人",
+                          "id": 853
-                          "author_zh": "諫山創",
-                          "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                          "id": 853,
-                          "_vectors": {
-                            "manual": [
-                              1.0,
-                              2.0,
-                              3.0
-                            ]
-                          },
-                          "_formatted": {
-                            "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                            "author_zh": "諫山創",
-                            "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                            "id": "853",
-                            "_vectors": {
-                              "manual": [
-                                "1.0",
-                                "2.0",
-                                "3.0"
-                              ]
-                            }
-                          }
                         }
                       ],
                       "query": "\"进击的巨人\"",
@@ -947,36 +606,13 @@ async fn auto_infer_locales_at_search() {
 
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -993,36 +629,13 @@ async fn auto_infer_locales_at_search() {
 
     index
         .search(
-                json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                     {
                       "hits": [
                         {
-                          "name_zh": "进击的巨人",
+                          "id": 853
-                          "author_zh": "諫山創",
-                          "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                          "id": 853,
-                          "_vectors": {
-                            "manual": [
-                              1.0,
-                              2.0,
-                              3.0
-                            ]
-                          },
-                          "_formatted": {
-                            "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                            "author_zh": "諫山創",
-                            "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                            "id": "853",
-                            "_vectors": {
-                              "manual": [
-                                "1.0",
-                                "2.0",
-                                "3.0"
-                              ]
-                            }
-                          }
                         }
                       ],
                       "query": "\"进击的巨人\"",
@@ -1039,36 +652,13 @@ async fn auto_infer_locales_at_search() {
 
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
+                      "id": 853
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -1116,7 +706,7 @@ async fn force_different_locales_with_pattern_nested() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1136,60 +726,37 @@ async fn force_different_locales_with_pattern_nested() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                     {
                       "hits": [
                         {
-                      "document_en": {
+                          "id": 852
-                        "name": "Attack on Titan",
-                        "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "author": "Hajime Isayama"
-                      },
-                      "document_ja": {
-                        "name": "進撃の巨人",
-                        "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "author": "諫山 創"
-                      },
-                      "document_zh": {
-                        "name": "进击的巨人",
-                        "description": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "author": "諫山創"
-                      },
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "document_en": {
-                          "name": "Attack on Titan",
-                          "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                          "author": "Hajime Isayama"
-                        },
-                        "document_ja": {
-                          "name": "進撃の巨人",
-                          "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                          "author": "諫山 創"
-                        },
-                        "document_zh": {
-                          "name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                          "description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                          "author": "諫山創"
-                        },
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
                         }
+                      ],
+                      "query": "\"进击的巨人\"",
+                      "processingTimeMs": "[duration]",
+                      "limit": 20,
+                      "offset": 0,
+                      "estimatedTotalHits": 1
                     }
+                    "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+
+    // force japanese
+    index
+        .search(
+            json!({"q": "\"进击的巨人\"", "locales": ["ja"], "attributesToRetrieve": ["id"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+                {
+                  "hits": [
+                    {
+                      "id": 852
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -1236,7 +803,7 @@ async fn settings_change() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1256,7 +823,7 @@ async fn settings_change() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1299,7 +866,7 @@ async fn settings_change() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1319,7 +886,7 @@ async fn settings_change() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1355,7 +922,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -1368,7 +935,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bn, bg, ca, cs, zh, da, de, el, en, eo, et, fi, fr, gu, he, hi, hr, hu, hy, id, it, jv, ja, kn, ka, km, ko, la, lv, lt, ml, mr, mk, my, ne, nl, nb, or, pa, fa, pl, pt, ro, ru, si, sk, sl, sn, es, sr, sv, ta, te, tl, th, tk, tr, uk, ur, uz, vi, yi, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zul",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -1390,7 +957,7 @@ async fn invalid_localized_attributes_rules() {
         .await;
     snapshot!(response, @r###"
     {
-      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
       "code": "invalid_settings_localized_attributes",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"