From b0717b75d94f6f10769edafdb35b81fe8dd7b5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 14 Apr 2021 19:06:18 +0200 Subject: [PATCH] Update tokenizer to v0.2.1 --- Cargo.lock | 20 ++++++++++++++++++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ef514ffe..e2a98faed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1847,7 +1847,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.1", "memmap", "milli", "mime", @@ -1894,6 +1894,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.3.4" @@ -1942,7 +1958,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.1.1", "memmap", "num-traits", "obkv", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9ab386882..efaafd077 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -40,7 +40,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" } mime = "0.3.16"