Update tokenizer version to v0.2.3

This commit is contained in:
Clémentine Urquizar 2021-06-21 17:00:09 +02:00 committed by Tamo
parent 2e3d85c31a
commit e2844f3a92
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
2 changed files with 19 additions and 3 deletions

20
Cargo.lock generated
View File

@ -1626,7 +1626,7 @@ dependencies = [
"log", "log",
"main_error", "main_error",
"meilisearch-error", "meilisearch-error",
"meilisearch-tokenizer", "meilisearch-tokenizer 0.2.3",
"memmap", "memmap",
"milli", "milli",
"mime", "mime",
@ -1680,6 +1680,22 @@ dependencies = [
"whatlang", "whatlang",
] ]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.3"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.4.0" version = "2.4.0"
@ -1726,7 +1742,7 @@ dependencies = [
"linked-hash-map", "linked-hash-map",
"log", "log",
"logging_timer", "logging_timer",
"meilisearch-tokenizer", "meilisearch-tokenizer 0.2.2",
"memmap", "memmap",
"obkv", "obkv",
"once_cell", "once_cell",

View File

@ -49,7 +49,7 @@ itertools = "0.10.0"
log = "0.4.8" log = "0.4.8"
main_error = "0.1.0" main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" } meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
memmap = "0.7.0" memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" }
mime = "0.3.16" mime = "0.3.16"