mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-10 23:18:55 +01:00
Merge #4945
4945: Add swedish in default pipelines r=dureuill a=ManyTheFish # Summary ## Fix Swedish support In Swedish the characters `å`/`ä`/`ö` are completely different than `a` or `o` and should not be normalized as the same character. because the Swedish specialized pipeline was not activated by default, these characters were normalized even with the settings: ```json { "localizedAttributes": [ { "locales": ["swe"], "attributePatterns": ["*"] } ] } ``` ## Update Charabia adding German support German segmentation will now be activated using the setting: ```json { "localizedAttributes": [ { "locales": ["deu"], "attributePatterns": ["*"] } ] } ``` # TODO - [x] Activate Swedish Pipeline - [x] Add a test to avoid future regressions - [x] Update Charabia Co-authored-by: ManyTheFish <many@meilisearch.com>
This commit is contained in:
commit
bd34ed01d9
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -933,9 +933,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "charabia"
|
name = "charabia"
|
||||||
version = "0.9.0"
|
version = "0.9.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "03cd8f290cae94934cdd0103c14c2de9faf2d7d85be0d24d511af2bf1b14119d"
|
checksum = "55ff52497324e7d168505a16949ae836c14595606fab94687238d2f6c8d4c798"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aho-corasick",
|
"aho-corasick",
|
||||||
"csv",
|
"csv",
|
||||||
@ -2838,7 +2838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
|
checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"windows-targets 0.48.1",
|
"windows-targets 0.52.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -66,3 +66,5 @@ khmer = ["milli/khmer"]
|
|||||||
vietnamese = ["milli/vietnamese"]
|
vietnamese = ["milli/vietnamese"]
|
||||||
# force swedish character recomposition
|
# force swedish character recomposition
|
||||||
swedish-recomposition = ["milli/swedish-recomposition"]
|
swedish-recomposition = ["milli/swedish-recomposition"]
|
||||||
|
# force german character recomposition
|
||||||
|
german = ["milli/german"]
|
||||||
|
@ -39,12 +39,14 @@ macro_rules! make_locale {
|
|||||||
pub enum Locale {
|
pub enum Locale {
|
||||||
$($iso_639_1,)+
|
$($iso_639_1,)+
|
||||||
$($iso_639_3,)+
|
$($iso_639_3,)+
|
||||||
|
Cmn,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<milli::tokenizer::Language> for Locale {
|
impl From<milli::tokenizer::Language> for Locale {
|
||||||
fn from(other: milli::tokenizer::Language) -> Locale {
|
fn from(other: milli::tokenizer::Language) -> Locale {
|
||||||
match other {
|
match other {
|
||||||
$(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
|
$(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
|
||||||
|
milli::tokenizer::Language::Cmn => Locale::Cmn,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -54,6 +56,7 @@ macro_rules! make_locale {
|
|||||||
match other {
|
match other {
|
||||||
$(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
|
$(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
|
||||||
$(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
|
$(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
|
||||||
|
Locale::Cmn => milli::tokenizer::Language::Cmn,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -65,6 +68,7 @@ macro_rules! make_locale {
|
|||||||
let locale = match s {
|
let locale = match s {
|
||||||
$($iso_639_1_str => Locale::$iso_639_1,)+
|
$($iso_639_1_str => Locale::$iso_639_1,)+
|
||||||
$($iso_639_3_str => Locale::$iso_639_3,)+
|
$($iso_639_3_str => Locale::$iso_639_3,)+
|
||||||
|
"cmn" => Locale::Cmn,
|
||||||
_ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
|
_ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -79,8 +83,9 @@ macro_rules! make_locale {
|
|||||||
|
|
||||||
impl std::fmt::Display for LocaleFormatError {
|
impl std::fmt::Display for LocaleFormatError {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
let valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+].join(", ");
|
let mut valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+,"cmn"];
|
||||||
write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
|
valid_locales.sort_by(|left, right| left.len().cmp(&right.len()).then(left.cmp(right)));
|
||||||
|
write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales.join(", "))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -99,7 +104,6 @@ make_locale!(
|
|||||||
(Bg, "bg") => (Bul, "bul"),
|
(Bg, "bg") => (Bul, "bul"),
|
||||||
(Ca, "ca") => (Cat, "cat"),
|
(Ca, "ca") => (Cat, "cat"),
|
||||||
(Cs, "cs") => (Ces, "ces"),
|
(Cs, "cs") => (Ces, "ces"),
|
||||||
(Zh, "zh") => (Cmn, "cmn"),
|
|
||||||
(Da, "da") => (Dan, "dan"),
|
(Da, "da") => (Dan, "dan"),
|
||||||
(De, "de") => (Deu, "deu"),
|
(De, "de") => (Deu, "deu"),
|
||||||
(El, "el") => (Ell, "ell"),
|
(El, "el") => (Ell, "ell"),
|
||||||
@ -157,5 +161,6 @@ make_locale!(
|
|||||||
(Uz, "uz") => (Uzb, "uzb"),
|
(Uz, "uz") => (Uzb, "uzb"),
|
||||||
(Vi, "vi") => (Vie, "vie"),
|
(Vi, "vi") => (Vie, "vie"),
|
||||||
(Yi, "yi") => (Yid, "yid"),
|
(Yi, "yi") => (Yid, "yid"),
|
||||||
|
(Zh, "zh") => (Zho, "zho"),
|
||||||
(Zu, "zu") => (Zul, "zul"),
|
(Zu, "zu") => (Zul, "zul"),
|
||||||
);
|
);
|
||||||
|
@ -153,6 +153,7 @@ greek = ["meilisearch-types/greek"]
|
|||||||
khmer = ["meilisearch-types/khmer"]
|
khmer = ["meilisearch-types/khmer"]
|
||||||
vietnamese = ["meilisearch-types/vietnamese"]
|
vietnamese = ["meilisearch-types/vietnamese"]
|
||||||
swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
|
swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
|
||||||
|
german = ["meilisearch-types/german"]
|
||||||
|
|
||||||
[package.metadata.mini-dashboard]
|
[package.metadata.mini-dashboard]
|
||||||
assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"
|
assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"
|
||||||
|
@ -922,7 +922,7 @@ async fn invalid_locales() {
|
|||||||
snapshot!(code, @"400 Bad Request");
|
snapshot!(code, @"400 Bad Request");
|
||||||
snapshot!(json_string!(response), @r###"
|
snapshot!(json_string!(response), @r###"
|
||||||
{
|
{
|
||||||
"message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
|
"message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zh`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zho`, `zul`, `cmn`",
|
||||||
"code": "invalid_search_locales",
|
"code": "invalid_search_locales",
|
||||||
"type": "invalid_request",
|
"type": "invalid_request",
|
||||||
"link": "https://docs.meilisearch.com/errors#invalid_search_locales"
|
"link": "https://docs.meilisearch.com/errors#invalid_search_locales"
|
||||||
@ -935,7 +935,7 @@ async fn invalid_locales() {
|
|||||||
snapshot!(code, @"400 Bad Request");
|
snapshot!(code, @"400 Bad Request");
|
||||||
snapshot!(json_string!(response), @r###"
|
snapshot!(json_string!(response), @r###"
|
||||||
{
|
{
|
||||||
"message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bn, bg, ca, cs, zh, da, de, el, en, eo, et, fi, fr, gu, he, hi, hr, hu, hy, id, it, jv, ja, kn, ka, km, ko, la, lv, lt, ml, mr, mk, my, ne, nl, nb, or, pa, fa, pl, pt, ro, ru, si, sk, sl, sn, es, sr, sv, ta, te, tl, th, tk, tr, uk, ur, uz, vi, yi, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zul",
|
"message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bg, bn, ca, cs, da, de, el, en, eo, es, et, fa, fi, fr, gu, he, hi, hr, hu, hy, id, it, ja, jv, ka, km, kn, ko, la, lt, lv, mk, ml, mr, my, nb, ne, nl, or, pa, pl, pt, ro, ru, si, sk, sl, sn, sr, sv, ta, te, th, tk, tl, tr, uk, ur, uz, vi, yi, zh, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zho, zul",
|
||||||
"code": "invalid_search_locales",
|
"code": "invalid_search_locales",
|
||||||
"type": "invalid_request",
|
"type": "invalid_request",
|
||||||
"link": "https://docs.meilisearch.com/errors#invalid_search_locales"
|
"link": "https://docs.meilisearch.com/errors#invalid_search_locales"
|
||||||
@ -957,7 +957,7 @@ async fn invalid_localized_attributes_rules() {
|
|||||||
.await;
|
.await;
|
||||||
snapshot!(response, @r###"
|
snapshot!(response, @r###"
|
||||||
{
|
{
|
||||||
"message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
|
"message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zh`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zho`, `zul`, `cmn`",
|
||||||
"code": "invalid_settings_localized_attributes",
|
"code": "invalid_settings_localized_attributes",
|
||||||
"type": "invalid_request",
|
"type": "invalid_request",
|
||||||
"link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"
|
"link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"
|
||||||
@ -1143,3 +1143,195 @@ async fn facet_search_with_localized_attributes() {
|
|||||||
}
|
}
|
||||||
"###);
|
"###);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[actix_rt::test]
|
||||||
|
async fn swedish_search() {
|
||||||
|
let server = Server::new().await;
|
||||||
|
|
||||||
|
let index = server.index("test");
|
||||||
|
let documents = json!([
|
||||||
|
{"id": "tra1-1", "product": "trä"},
|
||||||
|
{"id": "tra2-1", "product": "traktor"},
|
||||||
|
{"id": "tra1-2", "product": "träbjälke"},
|
||||||
|
{"id": "tra2-2", "product": "trafiksignal"},
|
||||||
|
]);
|
||||||
|
index.add_documents(documents, None).await;
|
||||||
|
let (_response, _) = index
|
||||||
|
.update_settings(json!({
|
||||||
|
"searchableAttributes": ["product"],
|
||||||
|
"localizedAttributes": [
|
||||||
|
// force swedish
|
||||||
|
{"attributePatterns": ["product"], "locales": ["swe"]}
|
||||||
|
]
|
||||||
|
}))
|
||||||
|
.await;
|
||||||
|
index.wait_task(1).await;
|
||||||
|
|
||||||
|
// infer swedish
|
||||||
|
index
|
||||||
|
.search(json!({"q": "trä", "attributesToRetrieve": ["product"]}), |response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "trä"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"product": "träbjälke"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "trä",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 2
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
index
|
||||||
|
.search(json!({"q": "tra", "attributesToRetrieve": ["product"]}), |response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "traktor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"product": "trafiksignal"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "tra",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 2
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// force swedish
|
||||||
|
index
|
||||||
|
.search(
|
||||||
|
json!({"q": "trä", "locales": ["swe"], "attributesToRetrieve": ["product"]}),
|
||||||
|
|response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "trä"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"product": "träbjälke"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "trä",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 2
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
index
|
||||||
|
.search(
|
||||||
|
json!({"q": "tra", "locales": ["swe"], "attributesToRetrieve": ["product"]}),
|
||||||
|
|response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "traktor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"product": "trafiksignal"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "tra",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 2
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[actix_rt::test]
|
||||||
|
async fn german_search() {
|
||||||
|
let server = Server::new().await;
|
||||||
|
|
||||||
|
let index = server.index("test");
|
||||||
|
let documents = json!([
|
||||||
|
{"id": 1, "product": "Interkulturalität"},
|
||||||
|
{"id": 2, "product": "Wissensorganisation"},
|
||||||
|
]);
|
||||||
|
index.add_documents(documents, None).await;
|
||||||
|
let (_response, _) = index
|
||||||
|
.update_settings(json!({
|
||||||
|
"searchableAttributes": ["product"],
|
||||||
|
"localizedAttributes": [
|
||||||
|
// force swedish
|
||||||
|
{"attributePatterns": ["product"], "locales": ["deu"]}
|
||||||
|
]
|
||||||
|
}))
|
||||||
|
.await;
|
||||||
|
index.wait_task(1).await;
|
||||||
|
|
||||||
|
// infer swedish
|
||||||
|
index
|
||||||
|
.search(
|
||||||
|
json!({"q": "kulturalität", "attributesToRetrieve": ["product"]}),
|
||||||
|
|response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "Interkulturalität"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "kulturalität",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 1
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
index
|
||||||
|
.search(
|
||||||
|
json!({"q": "organisation", "attributesToRetrieve": ["product"]}),
|
||||||
|
|response, code| {
|
||||||
|
snapshot!(response, @r###"
|
||||||
|
{
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"product": "Wissensorganisation"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": "organisation",
|
||||||
|
"processingTimeMs": "[duration]",
|
||||||
|
"limit": 20,
|
||||||
|
"offset": 0,
|
||||||
|
"estimatedTotalHits": 1
|
||||||
|
}
|
||||||
|
"###);
|
||||||
|
snapshot!(code, @"200 OK");
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
@ -17,7 +17,7 @@ bincode = "1.3.3"
|
|||||||
bstr = "1.9.1"
|
bstr = "1.9.1"
|
||||||
bytemuck = { version = "1.16.1", features = ["extern_crate_alloc"] }
|
bytemuck = { version = "1.16.1", features = ["extern_crate_alloc"] }
|
||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
charabia = { version = "0.9.0", default-features = false }
|
charabia = { version = "0.9.1", default-features = false }
|
||||||
concat-arrays = "0.1.2"
|
concat-arrays = "0.1.2"
|
||||||
crossbeam-channel = "0.5.13"
|
crossbeam-channel = "0.5.13"
|
||||||
deserr = "0.6.2"
|
deserr = "0.6.2"
|
||||||
@ -106,6 +106,8 @@ all-tokenizations = [
|
|||||||
"charabia/greek",
|
"charabia/greek",
|
||||||
"charabia/khmer",
|
"charabia/khmer",
|
||||||
"charabia/vietnamese",
|
"charabia/vietnamese",
|
||||||
|
"charabia/swedish-recomposition",
|
||||||
|
"charabia/german-segmentation",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Use POSIX semaphores instead of SysV semaphores in LMDB
|
# Use POSIX semaphores instead of SysV semaphores in LMDB
|
||||||
@ -138,6 +140,9 @@ khmer = ["charabia/khmer"]
|
|||||||
# allow vietnamese specialized tokenization
|
# allow vietnamese specialized tokenization
|
||||||
vietnamese = ["charabia/vietnamese"]
|
vietnamese = ["charabia/vietnamese"]
|
||||||
|
|
||||||
|
# allow german specialized tokenization
|
||||||
|
german = ["charabia/german-segmentation"]
|
||||||
|
|
||||||
# force swedish character recomposition
|
# force swedish character recomposition
|
||||||
swedish-recomposition = ["charabia/swedish-recomposition"]
|
swedish-recomposition = ["charabia/swedish-recomposition"]
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user