Merge #4957

4957: Update charabia feature flags r=dureuill a=ManyTheFish

# Pull Request

Add charabia's `turkish` feature flag into Meilisearch default tokenization flag



[All tests pipeline](https://github.com/meilisearch/meilisearch/actions/runs/11030036031)

Co-authored-by: ManyTheFish <many@meilisearch.com>

meilisearch-types/Cargo.toml

@@ -66,5 +66,8 @@ khmer = ["milli/khmer"]
 vietnamese = ["milli/vietnamese"]
 # force swedish character recomposition
 swedish-recomposition = ["milli/swedish-recomposition"]
-# force german character recomposition
+# allow german tokenization
 german = ["milli/german"]
+# allow turkish normalization
+turkish = ["milli/turkish"]
+

meilisearch/Cargo.toml

@@ -154,6 +154,7 @@ khmer = ["meilisearch-types/khmer"]
 vietnamese = ["meilisearch-types/vietnamese"]
 swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
 german = ["meilisearch-types/german"]
+turkish = ["meilisearch-types/turkish"]
 
 [package.metadata.mini-dashboard]
 assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"

milli/Cargo.toml

@@ -98,16 +98,7 @@ rand = { version = "0.8.5", features = ["small_rng"] }
 
 [features]
 all-tokenizations = [
-    "charabia/chinese",
+    "charabia/default",
-    "charabia/hebrew",
-    "charabia/japanese",
-    "charabia/thai",
-    "charabia/korean",
-    "charabia/greek",
-    "charabia/khmer",
-    "charabia/vietnamese",
-    "charabia/swedish-recomposition",
-    "charabia/german-segmentation",
 ]
 
 # Use POSIX semaphores instead of SysV semaphores in LMDB
@@ -146,5 +137,8 @@ german = ["charabia/german-segmentation"]
 # force swedish character recomposition
 swedish-recomposition = ["charabia/swedish-recomposition"]
 
+# allow turkish specialized tokenization
+turkish = ["charabia/turkish"]
+
 # allow CUDA support, see <https://github.com/meilisearch/meilisearch/issues/4306>
 cuda = ["candle-core/cuda"]