Merge #4945

4945: Add swedish in default pipelines r=dureuill a=ManyTheFish

# Summary
## Fix Swedish support

In Swedish the characters `å`/`ä`/`ö` are completely different than `a` or `o`  and should not be normalized as the same character.
because the Swedish specialized pipeline was not activated by default, these characters were normalized even with the settings:
```json
{
  "localizedAttributes": [ { "locales": ["swe"], "attributePatterns": ["*"] } ]
}
```

## Update Charabia adding German support

German segmentation will now be activated using the setting:
```json
{
  "localizedAttributes": [ { "locales": ["deu"], "attributePatterns": ["*"] } ]
}
```

# TODO

- [x] Activate Swedish Pipeline
- [x] Add a test to avoid future regressions
- [x] Update Charabia


Co-authored-by: ManyTheFish <many@meilisearch.com>

Cargo.lock

@@ -933,9 +933,9 @@ dependencies = [
 
 [[package]]
 name = "charabia"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03cd8f290cae94934cdd0103c14c2de9faf2d7d85be0d24d511af2bf1b14119d"
+checksum = "55ff52497324e7d168505a16949ae836c14595606fab94687238d2f6c8d4c798"
 dependencies = [
  "aho-corasick",
  "csv",
@@ -2838,7 +2838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
 dependencies = [
  "cfg-if",
- "windows-targets 0.48.1",
+ "windows-targets 0.52.4",
 ]
 
 [[package]]

meilisearch-types/Cargo.toml

@@ -66,3 +66,5 @@ khmer = ["milli/khmer"]
 vietnamese = ["milli/vietnamese"]
 # force swedish character recomposition
 swedish-recomposition = ["milli/swedish-recomposition"]
+# force german character recomposition
+german = ["milli/german"]

meilisearch-types/src/locales.rs

@@ -39,12 +39,14 @@ macro_rules! make_locale {
         pub enum Locale {
             $($iso_639_1,)+
             $($iso_639_3,)+
+            Cmn,
         }
 
         impl From<milli::tokenizer::Language> for Locale {
             fn from(other: milli::tokenizer::Language) -> Locale {
                 match other {
                     $(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
+                    milli::tokenizer::Language::Cmn => Locale::Cmn,
                 }
             }
         }
@@ -54,6 +56,7 @@ macro_rules! make_locale {
                 match other {
                     $(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
                     $(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
+                    Locale::Cmn => milli::tokenizer::Language::Cmn,
                 }
             }
         }
@@ -65,6 +68,7 @@ macro_rules! make_locale {
                 let locale = match s {
                     $($iso_639_1_str => Locale::$iso_639_1,)+
                     $($iso_639_3_str => Locale::$iso_639_3,)+
+                    "cmn" => Locale::Cmn,
                     _ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
                 };
 
@@ -79,8 +83,9 @@ macro_rules! make_locale {
 
         impl std::fmt::Display for LocaleFormatError {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                let valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+].join(", ");
-                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
+                let mut valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+,"cmn"];
+                valid_locales.sort_by(|left, right| left.len().cmp(&right.len()).then(left.cmp(right)));
+                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales.join(", "))
             }
         }
 
@@ -99,7 +104,6 @@ make_locale!(
     (Bg, "bg") => (Bul, "bul"),
     (Ca, "ca") => (Cat, "cat"),
     (Cs, "cs") => (Ces, "ces"),
-    (Zh, "zh") => (Cmn, "cmn"),
     (Da, "da") => (Dan, "dan"),
     (De, "de") => (Deu, "deu"),
     (El, "el") => (Ell, "ell"),
@@ -157,5 +161,6 @@ make_locale!(
     (Uz, "uz") => (Uzb, "uzb"),
     (Vi, "vi") => (Vie, "vie"),
     (Yi, "yi") => (Yid, "yid"),
+    (Zh, "zh") => (Zho, "zho"),
     (Zu, "zu") => (Zul, "zul"),
 );

meilisearch/Cargo.toml

@@ -153,6 +153,7 @@ greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
 vietnamese = ["meilisearch-types/vietnamese"]
 swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
+german = ["meilisearch-types/german"]
 
 [package.metadata.mini-dashboard]
 assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"

meilisearch/tests/search/locales.rs

@@ -922,7 +922,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
+      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zh`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zho`, `zul`, `cmn`",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -935,7 +935,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bn, bg, ca, cs, zh, da, de, el, en, eo, et, fi, fr, gu, he, hi, hr, hu, hy, id, it, jv, ja, kn, ka, km, ko, la, lv, lt, ml, mr, mk, my, ne, nl, nb, or, pa, fa, pl, pt, ro, ru, si, sk, sl, sn, es, sr, sv, ta, te, tl, th, tk, tr, uk, ur, uz, vi, yi, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zul",
+      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bg, bn, ca, cs, da, de, el, en, eo, es, et, fa, fi, fr, gu, he, hi, hr, hu, hy, id, it, ja, jv, ka, km, kn, ko, la, lt, lv, mk, ml, mr, my, nb, ne, nl, or, pa, pl, pt, ro, ru, si, sk, sl, sn, sr, sv, ta, te, th, tk, tl, tr, uk, ur, uz, vi, yi, zh, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zho, zul",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -957,7 +957,7 @@ async fn invalid_localized_attributes_rules() {
         .await;
     snapshot!(response, @r###"
     {
-      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
+      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zh`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zho`, `zul`, `cmn`",
       "code": "invalid_settings_localized_attributes",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"
@@ -1143,3 +1143,195 @@ async fn facet_search_with_localized_attributes() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn swedish_search() {
+    let server = Server::new().await;
+
+    let index = server.index("test");
+    let documents = json!([
+      {"id": "tra1-1", "product": "trä"},
+      {"id": "tra2-1", "product": "traktor"},
+      {"id": "tra1-2", "product": "träbjälke"},
+      {"id": "tra2-2", "product": "trafiksignal"},
+    ]);
+    index.add_documents(documents, None).await;
+    let (_response, _) = index
+        .update_settings(json!({
+            "searchableAttributes": ["product"],
+            "localizedAttributes": [
+                // force swedish
+                {"attributePatterns": ["product"], "locales": ["swe"]}
+            ]
+        }))
+        .await;
+    index.wait_task(1).await;
+
+    // infer swedish
+    index
+        .search(json!({"q": "trä", "attributesToRetrieve": ["product"]}), |response, code| {
+            snapshot!(response, @r###"
+            {
+              "hits": [
+                {
+                  "product": "trä"
+                },
+                {
+                  "product": "träbjälke"
+                }
+              ],
+              "query": "trä",
+              "processingTimeMs": "[duration]",
+              "limit": 20,
+              "offset": 0,
+              "estimatedTotalHits": 2
+            }
+            "###);
+            snapshot!(code, @"200 OK");
+        })
+        .await;
+
+    index
+        .search(json!({"q": "tra", "attributesToRetrieve": ["product"]}), |response, code| {
+            snapshot!(response, @r###"
+            {
+              "hits": [
+                {
+                  "product": "traktor"
+                },
+                {
+                  "product": "trafiksignal"
+                }
+              ],
+              "query": "tra",
+              "processingTimeMs": "[duration]",
+              "limit": 20,
+              "offset": 0,
+              "estimatedTotalHits": 2
+            }
+            "###);
+            snapshot!(code, @"200 OK");
+        })
+        .await;
+
+    // force swedish
+    index
+        .search(
+            json!({"q": "trä", "locales": ["swe"], "attributesToRetrieve": ["product"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+                {
+                  "hits": [
+                    {
+                      "product": "trä"
+                    },
+                    {
+                      "product": "träbjälke"
+                    }
+                  ],
+                  "query": "trä",
+                  "processingTimeMs": "[duration]",
+                  "limit": 20,
+                  "offset": 0,
+                  "estimatedTotalHits": 2
+                }
+                "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+    index
+        .search(
+            json!({"q": "tra", "locales": ["swe"], "attributesToRetrieve": ["product"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+                {
+                  "hits": [
+                    {
+                      "product": "traktor"
+                    },
+                    {
+                      "product": "trafiksignal"
+                    }
+                  ],
+                  "query": "tra",
+                  "processingTimeMs": "[duration]",
+                  "limit": 20,
+                  "offset": 0,
+                  "estimatedTotalHits": 2
+                }
+                "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn german_search() {
+    let server = Server::new().await;
+
+    let index = server.index("test");
+    let documents = json!([
+      {"id": 1, "product": "Interkulturalität"},
+      {"id": 2, "product": "Wissensorganisation"},
+    ]);
+    index.add_documents(documents, None).await;
+    let (_response, _) = index
+        .update_settings(json!({
+            "searchableAttributes": ["product"],
+            "localizedAttributes": [
+                // force swedish
+                {"attributePatterns": ["product"], "locales": ["deu"]}
+            ]
+        }))
+        .await;
+    index.wait_task(1).await;
+
+    // infer swedish
+    index
+        .search(
+            json!({"q": "kulturalität", "attributesToRetrieve": ["product"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+            {
+              "hits": [
+                {
+                  "product": "Interkulturalität"
+                }
+              ],
+              "query": "kulturalität",
+              "processingTimeMs": "[duration]",
+              "limit": 20,
+              "offset": 0,
+              "estimatedTotalHits": 1
+            }
+            "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+
+    index
+        .search(
+            json!({"q": "organisation", "attributesToRetrieve": ["product"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+            {
+              "hits": [
+                {
+                  "product": "Wissensorganisation"
+                }
+              ],
+              "query": "organisation",
+              "processingTimeMs": "[duration]",
+              "limit": 20,
+              "offset": 0,
+              "estimatedTotalHits": 1
+            }
+            "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+}

milli/Cargo.toml

@@ -17,7 +17,7 @@ bincode = "1.3.3"
 bstr = "1.9.1"
 bytemuck = { version = "1.16.1", features = ["extern_crate_alloc"] }
 byteorder = "1.5.0"
-charabia = { version = "0.9.0", default-features = false }
+charabia = { version = "0.9.1", default-features = false }
 concat-arrays = "0.1.2"
 crossbeam-channel = "0.5.13"
 deserr = "0.6.2"
@@ -106,6 +106,8 @@ all-tokenizations = [
     "charabia/greek",
     "charabia/khmer",
     "charabia/vietnamese",
+    "charabia/swedish-recomposition",
+    "charabia/german-segmentation",
 ]
 
 # Use POSIX semaphores instead of SysV semaphores in LMDB
@@ -138,6 +140,9 @@ khmer = ["charabia/khmer"]
 # allow vietnamese specialized tokenization
 vietnamese = ["charabia/vietnamese"]
 
+# allow german specialized tokenization
+german = ["charabia/german-segmentation"]
+
 # force swedish character recomposition
 swedish-recomposition = ["charabia/swedish-recomposition"]