From 7d5395c12b9d8e664789340c0bfdd60e4d50fe62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 10 Jun 2021 17:00:04 +0200 Subject: [PATCH] Update Tokenizer version to v0.2.3 --- Cargo.lock | 20 ++++++++++++++++++-- http-ui/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba4a4c60c..c719f6f03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,7 +971,7 @@ dependencies = [ "heed", "log", "maplit", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.3", "memmap", "milli", "once_cell", @@ -1332,6 +1332,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.3" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.3.4" @@ -1380,7 +1396,7 @@ dependencies = [ "log", "logging_timer", "maplit", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.2", "memmap", "obkv", "once_cell", diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index 146404772..a11307fbe 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -11,7 +11,7 @@ byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } crossbeam-channel = "0.5.0" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" milli = { path = "../milli" } once_cell = "1.5.2"