4937: Support iso 639 1 r=ManyTheFish a=ManyTheFish

# Pull Request

## Related issue
Fixes #4827

## What does this PR do?
- Add iso-639-1 variants to the Locales enum
- Convert iso-639-1 into iso-639-3


Co-authored-by: ManyTheFish <many@meilisearch.com>
This commit is contained in:
meili-bors[bot] 2024-09-18 05:29:32 +00:00 committed by GitHub
commit 4c355bede7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 206 additions and 635 deletions

View File

@ -1,134 +1,6 @@
use deserr::Deserr; use deserr::Deserr;
use milli::LocalizedAttributesRule; use milli::LocalizedAttributesRule;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::json;
/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
///
/// this enum implements `Deserr` in order to be used in the API.
macro_rules! make_locale {
($($language:tt), +) => {
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
#[deserr(rename_all = camelCase)]
#[serde(rename_all = "camelCase")]
pub enum Locale {
$($language),+,
}
impl From<milli::tokenizer::Language> for Locale {
fn from(other: milli::tokenizer::Language) -> Locale {
match other {
$(milli::tokenizer::Language::$language => Locale::$language), +
}
}
}
impl From<Locale> for milli::tokenizer::Language {
fn from(other: Locale) -> milli::tokenizer::Language {
match other {
$(Locale::$language => milli::tokenizer::Language::$language), +,
}
}
}
#[derive(Debug)]
pub struct LocaleFormatError {
pub invalid_locale: String,
}
impl std::fmt::Display for LocaleFormatError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let valid_locales = [$(Locale::$language),+].iter().map(|l| format!("`{}`", json!(l).as_str().unwrap())).collect::<Vec<_>>().join(", ");
write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
}
}
};
}
make_locale! {
Epo,
Eng,
Rus,
Cmn,
Spa,
Por,
Ita,
Ben,
Fra,
Deu,
Ukr,
Kat,
Ara,
Hin,
Jpn,
Heb,
Yid,
Pol,
Amh,
Jav,
Kor,
Nob,
Dan,
Swe,
Fin,
Tur,
Nld,
Hun,
Ces,
Ell,
Bul,
Bel,
Mar,
Kan,
Ron,
Slv,
Hrv,
Srp,
Mkd,
Lit,
Lav,
Est,
Tam,
Vie,
Urd,
Tha,
Guj,
Uzb,
Pan,
Aze,
Ind,
Tel,
Pes,
Mal,
Ori,
Mya,
Nep,
Sin,
Khm,
Tuk,
Aka,
Zul,
Sna,
Afr,
Lat,
Slk,
Cat,
Tgl,
Hye
}
impl std::error::Error for LocaleFormatError {}
impl std::str::FromStr for Locale {
type Err = LocaleFormatError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
milli::tokenizer::Language::from_code(s)
.map(Self::from)
.ok_or(LocaleFormatError { invalid_locale: s.to_string() })
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)]
#[deserr(rename_all = camelCase)] #[deserr(rename_all = camelCase)]
@ -155,3 +27,135 @@ impl From<LocalizedAttributesRuleView> for LocalizedAttributesRule {
} }
} }
} }
/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
///
/// this enum implements `Deserr` in order to be used in the API.
macro_rules! make_locale {
($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => {
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
#[deserr(rename_all = camelCase)]
#[serde(rename_all = "camelCase")]
pub enum Locale {
$($iso_639_1,)+
$($iso_639_3,)+
}
impl From<milli::tokenizer::Language> for Locale {
fn from(other: milli::tokenizer::Language) -> Locale {
match other {
$(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
}
}
}
impl From<Locale> for milli::tokenizer::Language {
fn from(other: Locale) -> milli::tokenizer::Language {
match other {
$(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
$(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
}
}
}
impl std::str::FromStr for Locale {
type Err = LocaleFormatError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let locale = match s {
$($iso_639_1_str => Locale::$iso_639_1,)+
$($iso_639_3_str => Locale::$iso_639_3,)+
_ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
};
Ok(locale)
}
}
#[derive(Debug)]
pub struct LocaleFormatError {
pub invalid_locale: String,
}
impl std::fmt::Display for LocaleFormatError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+].join(", ");
write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
}
}
impl std::error::Error for LocaleFormatError {}
};
}
make_locale!(
(Af, "af") => (Afr, "afr"),
(Ak, "ak") => (Aka, "aka"),
(Am, "am") => (Amh, "amh"),
(Ar, "ar") => (Ara, "ara"),
(Az, "az") => (Aze, "aze"),
(Be, "be") => (Bel, "bel"),
(Bn, "bn") => (Ben, "ben"),
(Bg, "bg") => (Bul, "bul"),
(Ca, "ca") => (Cat, "cat"),
(Cs, "cs") => (Ces, "ces"),
(Zh, "zh") => (Cmn, "cmn"),
(Da, "da") => (Dan, "dan"),
(De, "de") => (Deu, "deu"),
(El, "el") => (Ell, "ell"),
(En, "en") => (Eng, "eng"),
(Eo, "eo") => (Epo, "epo"),
(Et, "et") => (Est, "est"),
(Fi, "fi") => (Fin, "fin"),
(Fr, "fr") => (Fra, "fra"),
(Gu, "gu") => (Guj, "guj"),
(He, "he") => (Heb, "heb"),
(Hi, "hi") => (Hin, "hin"),
(Hr, "hr") => (Hrv, "hrv"),
(Hu, "hu") => (Hun, "hun"),
(Hy, "hy") => (Hye, "hye"),
(Id, "id") => (Ind, "ind"),
(It, "it") => (Ita, "ita"),
(Jv, "jv") => (Jav, "jav"),
(Ja, "ja") => (Jpn, "jpn"),
(Kn, "kn") => (Kan, "kan"),
(Ka, "ka") => (Kat, "kat"),
(Km, "km") => (Khm, "khm"),
(Ko, "ko") => (Kor, "kor"),
(La, "la") => (Lat, "lat"),
(Lv, "lv") => (Lav, "lav"),
(Lt, "lt") => (Lit, "lit"),
(Ml, "ml") => (Mal, "mal"),
(Mr, "mr") => (Mar, "mar"),
(Mk, "mk") => (Mkd, "mkd"),
(My, "my") => (Mya, "mya"),
(Ne, "ne") => (Nep, "nep"),
(Nl, "nl") => (Nld, "nld"),
(Nb, "nb") => (Nob, "nob"),
(Or, "or") => (Ori, "ori"),
(Pa, "pa") => (Pan, "pan"),
(Fa, "fa") => (Pes, "pes"),
(Pl, "pl") => (Pol, "pol"),
(Pt, "pt") => (Por, "por"),
(Ro, "ro") => (Ron, "ron"),
(Ru, "ru") => (Rus, "rus"),
(Si, "si") => (Sin, "sin"),
(Sk, "sk") => (Slk, "slk"),
(Sl, "sl") => (Slv, "slv"),
(Sn, "sn") => (Sna, "sna"),
(Es, "es") => (Spa, "spa"),
(Sr, "sr") => (Srp, "srp"),
(Sv, "sv") => (Swe, "swe"),
(Ta, "ta") => (Tam, "tam"),
(Te, "te") => (Tel, "tel"),
(Tl, "tl") => (Tgl, "tgl"),
(Th, "th") => (Tha, "tha"),
(Tk, "tk") => (Tuk, "tuk"),
(Tr, "tr") => (Tur, "tur"),
(Uk, "uk") => (Ukr, "ukr"),
(Ur, "ur") => (Urd, "urd"),
(Uz, "uz") => (Uzb, "uzb"),
(Vi, "vi") => (Vie, "vie"),
(Yi, "yi") => (Yid, "yid"),
(Zu, "zu") => (Zul, "zul"),
);

View File

@ -103,41 +103,12 @@ async fn simple_search() {
// english // english
index index
.search(json!({"q": "Atta", "attributesToHighlight": ["*"]}), |response, code| { .search(json!({"q": "Atta", "attributesToRetrieve": ["id"]}), |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_en": "Attack on Titan", "id": 852
"name_ja": "進撃の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": 852,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_en": "<em>Atta</em>ck on Titan",
"name_ja": "進撃の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "<em>Atta</em>ck on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": "852",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "Atta", "query": "Atta",
@ -153,35 +124,12 @@ async fn simple_search() {
// japanese // japanese
index index
.search(json!({"q": "進撃", "attributesToHighlight": ["*"]}), |response, code| { .search(json!({"q": "進撃", "attributesToRetrieve": ["id"]}), |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进击</em>的巨人",
"author_zh": "諫山創",
"description_zh": "<em>进击</em>的巨人是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "進撃", "query": "進撃",
@ -197,68 +145,16 @@ async fn simple_search() {
index index
.search( .search(
json!({"q": "進撃", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "進撃", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_en": "Attack on Titan", "id": 852
"name_ja": "進撃の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": 852,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_en": "Attack on Titan",
"name_ja": "<em>進撃</em>の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": "852",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
}, },
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "进击的巨人",
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "進撃", "query": "進撃",
@ -275,67 +171,15 @@ async fn simple_search() {
// chinese // chinese
index index
.search(json!({"q": "进击", "attributesToHighlight": ["*"]}), |response, code| { .search(json!({"q": "进击", "attributesToRetrieve": ["id"]}), |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进击</em>的巨人",
"author_zh": "諫山創",
"description_zh": "<em>进击</em>的巨人是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
}, },
{ {
"name_en": "Attack on Titan", "id": 852
"name_ja": "進撃の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": 852,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_en": "Attack on Titan",
"name_ja": "<em>進撃</em>の巨人",
"author_en": "Hajime Isayama",
"author_ja": "諫山 創",
"description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"id": "852",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "进击", "query": "进击",
@ -382,36 +226,13 @@ async fn force_locales() {
// chinese detection // chinese detection
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -429,36 +250,13 @@ async fn force_locales() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -506,36 +304,13 @@ async fn force_locales_with_pattern() {
// chinese detection // chinese detection
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -553,36 +328,13 @@ async fn force_locales_with_pattern() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -628,7 +380,7 @@ async fn force_locales_with_pattern_nested() {
// chinese // chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -648,60 +400,13 @@ async fn force_locales_with_pattern_nested() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"document_en": { "id": 852
"name": "Attack on Titan",
"description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"author": "Hajime Isayama"
},
"document_ja": {
"name": "進撃の巨人",
"description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"author": "諫山 創"
},
"document_zh": {
"name": "进击的巨人",
"description": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"author": "諫山創"
},
"id": 852,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"document_en": {
"name": "Attack on Titan",
"description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"author": "Hajime Isayama"
},
"document_ja": {
"name": "進撃の巨人",
"description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"author": "諫山 創"
},
"document_zh": {
"name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"author": "諫山創"
},
"id": "852",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -750,7 +455,7 @@ async fn force_different_locales_with_pattern() {
// force chinese // force chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -770,36 +475,13 @@ async fn force_different_locales_with_pattern() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -851,7 +533,7 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
// auto infer any language // auto infer any language
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -871,36 +553,13 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
// should infer chinese // should infer chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"], "attributesToSearchOn": ["name_zh", "description_zh"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"], "attributesToSearchOn": ["name_zh", "description_zh"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -947,36 +606,13 @@ async fn auto_infer_locales_at_search() {
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -992,37 +628,14 @@ async fn auto_infer_locales_at_search() {
.await; .await;
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -1032,43 +645,20 @@ async fn auto_infer_locales_at_search() {
"estimatedTotalHits": 1 "estimatedTotalHits": 1
} }
"###); "###);
snapshot!(code, @"200 OK"); snapshot!(code, @"200 OK");
}, },
) )
.await; .await;
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"name_zh": "进击的巨人", "id": 853
"author_zh": "諫山創",
"description_zh": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"id": 853,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"author_zh": "諫山創",
"description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"id": "853",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -1116,7 +706,7 @@ async fn force_different_locales_with_pattern_nested() {
// chinese // chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -1136,60 +726,37 @@ async fn force_different_locales_with_pattern_nested() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| {
snapshot!(response, @r###"
{
"hits": [
{
"id": 852
}
],
"query": "\"进击的巨人\"",
"processingTimeMs": "[duration]",
"limit": 20,
"offset": 0,
"estimatedTotalHits": 1
}
"###);
snapshot!(code, @"200 OK");
},
)
.await;
// force japanese
index
.search(
json!({"q": "\"进击的巨人\"", "locales": ["ja"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"hits": [ "hits": [
{ {
"document_en": { "id": 852
"name": "Attack on Titan",
"description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"author": "Hajime Isayama"
},
"document_ja": {
"name": "進撃の巨人",
"description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"author": "諫山 創"
},
"document_zh": {
"name": "进击的巨人",
"description": "进击的巨人是日本的漫画系列,由諫山 創作画。",
"author": "諫山創"
},
"id": 852,
"_vectors": {
"manual": [
1.0,
2.0,
3.0
]
},
"_formatted": {
"document_en": {
"name": "Attack on Titan",
"description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
"author": "Hajime Isayama"
},
"document_ja": {
"name": "進撃の巨人",
"description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
"author": "諫山 創"
},
"document_zh": {
"name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
"description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列,由諫山 創作画。",
"author": "諫山創"
},
"id": "852",
"_vectors": {
"manual": [
"1.0",
"2.0",
"3.0"
]
}
}
} }
], ],
"query": "\"进击的巨人\"", "query": "\"进击的巨人\"",
@ -1236,7 +803,7 @@ async fn settings_change() {
// chinese // chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -1256,7 +823,7 @@ async fn settings_change() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -1299,7 +866,7 @@ async fn settings_change() {
// chinese // chinese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -1319,7 +886,7 @@ async fn settings_change() {
// force japanese // force japanese
index index
.search( .search(
json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}), json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
|response, code| { |response, code| {
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
@ -1355,7 +922,7 @@ async fn invalid_locales() {
snapshot!(code, @"400 Bad Request"); snapshot!(code, @"400 Bad Request");
snapshot!(json_string!(response), @r###" snapshot!(json_string!(response), @r###"
{ {
"message": "Unknown value `invalid` at `.locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`", "message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
"code": "invalid_search_locales", "code": "invalid_search_locales",
"type": "invalid_request", "type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_search_locales" "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@ -1368,7 +935,7 @@ async fn invalid_locales() {
snapshot!(code, @"400 Bad Request"); snapshot!(code, @"400 Bad Request");
snapshot!(json_string!(response), @r###" snapshot!(json_string!(response), @r###"
{ {
"message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`", "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bn, bg, ca, cs, zh, da, de, el, en, eo, et, fi, fr, gu, he, hi, hr, hu, hy, id, it, jv, ja, kn, ka, km, ko, la, lv, lt, ml, mr, mk, my, ne, nl, nb, or, pa, fa, pl, pt, ro, ru, si, sk, sl, sn, es, sr, sv, ta, te, tl, th, tk, tr, uk, ur, uz, vi, yi, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zul",
"code": "invalid_search_locales", "code": "invalid_search_locales",
"type": "invalid_request", "type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_search_locales" "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@ -1390,7 +957,7 @@ async fn invalid_localized_attributes_rules() {
.await; .await;
snapshot!(response, @r###" snapshot!(response, @r###"
{ {
"message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`", "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
"code": "invalid_settings_localized_attributes", "code": "invalid_settings_localized_attributes",
"type": "invalid_request", "type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes" "link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"