Merge #273

273: Update tokenizer version to v0.2.3 r=Kerollmops a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -1018,7 +1018,7 @@ dependencies = [
  "jemallocator",
  "log",
  "maplit",
- "meilisearch-tokenizer 0.2.3",
+ "meilisearch-tokenizer",
  "memmap",
  "milli",
  "once_cell",
@@ -1379,22 +1379,6 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 
-[[package]]
-name = "meilisearch-tokenizer"
-version = "0.2.2"
-source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f"
-dependencies = [
- "character_converter",
- "cow-utils",
- "deunicode",
- "fst",
- "jieba-rs",
- "once_cell",
- "slice-group-by",
- "unicode-segmentation",
- "whatlang",
-]
-
 [[package]]
 name = "meilisearch-tokenizer"
 version = "0.2.3"
@@ -1458,7 +1442,7 @@ dependencies = [
  "log",
  "logging_timer",
  "maplit",
- "meilisearch-tokenizer 0.2.2",
+ "meilisearch-tokenizer",
  "memmap",
  "obkv",
  "once_cell",

milli/Cargo.toml

@@ -18,7 +18,7 @@ heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-fe
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
 obkv = "0.1.1"
 once_cell = "1.5.2"