1578: Update tokenizer version to v0.2.4 r=ManyTheFish a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
This commit is contained in:
bors[bot] 2021-08-12 15:27:12 +00:00 committed by GitHub
commit 535aff8f7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 29 deletions

31
Cargo.lock generated
View File

@ -1658,7 +1658,7 @@ dependencies = [
"log",
"main_error",
"meilisearch-error",
"meilisearch-tokenizer 0.2.3",
"meilisearch-tokenizer",
"memmap",
"milli",
"mime",
@ -1697,22 +1697,6 @@ dependencies = [
"zip",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.3"
source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang 0.9.0",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.4"
@ -1726,7 +1710,7 @@ dependencies = [
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang 0.12.0",
"whatlang",
]
[[package]]
@ -1776,7 +1760,7 @@ dependencies = [
"linked-hash-map",
"log",
"logging_timer",
"meilisearch-tokenizer 0.2.4",
"meilisearch-tokenizer",
"memmap",
"obkv",
"once_cell",
@ -3554,15 +3538,6 @@ dependencies = [
"webpki",
]
[[package]]
name = "whatlang"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
dependencies = [
"hashbrown 0.7.2",
]
[[package]]
name = "whatlang"
version = "0.12.0"

View File

@ -49,7 +49,7 @@ itertools = "0.10.0"
log = "0.4.8"
main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.4" }
memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.9.0" }
mime = "0.3.16"