remove meilisearch tokenizer

This commit is contained in:
mpostma 2020-11-19 16:00:14 +01:00 committed by many
parent 061832af7f
commit 8a4d05b7bb
No known key found for this signature in database
GPG Key ID: 2CEF23B75189EACA
6 changed files with 44 additions and 41 deletions

80
Cargo.lock generated
View File

@ -210,7 +210,7 @@ dependencies = [
"rustls 0.18.1", "rustls 0.18.1",
"tokio-rustls", "tokio-rustls",
"webpki", "webpki",
"webpki-roots 0.20.0", "webpki-roots",
] ]
[[package]] [[package]]
@ -332,7 +332,7 @@ checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
[[package]] [[package]]
name = "assert-json-diff" name = "assert-json-diff"
version = "1.0.1" version = "1.0.1"
source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4"
dependencies = [ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
@ -383,7 +383,7 @@ dependencies = [
"actix-rt", "actix-rt",
"actix-service", "actix-service",
"base64 0.13.0", "base64 0.13.0",
"bytes 0.5.6", "bytes",
"cfg-if 1.0.0", "cfg-if 1.0.0",
"derive_more", "derive_more",
"futures-core", "futures-core",
@ -790,6 +790,12 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "data-encoding"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993a608597367c6377b258c25d7120740f00ed23a2252b729b1932dd7866f908"
[[package]] [[package]]
name = "debugid" name = "debugid"
version = "0.7.2" version = "0.7.2"
@ -962,16 +968,6 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "fs_extra" name = "fs_extra"
version = "1.2.0" version = "1.2.0"
@ -1229,9 +1225,9 @@ checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c"
[[package]] [[package]]
name = "heed-types" name = "heed-types"
version = "0.7.2" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" checksum = "72fc61caee13e85ea330eabf0c6c7098c511ff173bcb57a760b1eda3bba9f6eb"
dependencies = [ dependencies = [
"bincode", "bincode",
"heed-traits", "heed-traits",
@ -1516,9 +1512,9 @@ dependencies = [
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.81" version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
@ -1616,7 +1612,6 @@ dependencies = [
"log", "log",
"meilisearch-error", "meilisearch-error",
"meilisearch-schema", "meilisearch-schema",
"meilisearch-tokenizer",
"meilisearch-types", "meilisearch-types",
"once_cell", "once_cell",
"ordered-float", "ordered-float",
@ -1666,7 +1661,6 @@ dependencies = [
"meilisearch-core", "meilisearch-core",
"meilisearch-error", "meilisearch-error",
"meilisearch-schema", "meilisearch-schema",
"meilisearch-tokenizer",
"mime", "mime",
"once_cell", "once_cell",
"rand 0.8.1", "rand 0.8.1",
@ -1702,14 +1696,6 @@ dependencies = [
"zerocopy", "zerocopy",
] ]
[[package]]
name = "meilisearch-tokenizer"
version = "0.17.0"
dependencies = [
"deunicode",
"slice-group-by",
]
[[package]] [[package]]
name = "meilisearch-types" name = "meilisearch-types"
version = "0.17.0" version = "0.17.0"
@ -2353,6 +2339,7 @@ dependencies = [
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-bindgen-test",
"web-sys", "web-sys",
"webpki-roots 0.20.0", "webpki-roots 0.20.0",
"winreg 0.7.0", "winreg 0.7.0",
@ -2459,6 +2446,12 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "scoped-tls"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.1.0" version = "1.1.0"
@ -3388,6 +3381,30 @@ name = "web-sys"
version = "0.3.46" version = "0.3.46"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3"
dependencies = [
"console_error_panic_hook",
"js-sys",
"scoped-tls",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-bindgen-test-macro",
]
[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "webpki-roots"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@ -3403,15 +3420,6 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "webpki-roots"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
dependencies = [
"webpki",
]
[[package]] [[package]]
name = "webpki-roots" name = "webpki-roots"
version = "0.21.0" version = "0.21.0"

View File

@ -3,7 +3,6 @@ members = [
"meilisearch-core", "meilisearch-core",
"meilisearch-http", "meilisearch-http",
"meilisearch-schema", "meilisearch-schema",
"meilisearch-tokenizer",
"meilisearch-types", "meilisearch-types",
] ]

View File

@ -26,7 +26,6 @@ levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
log = "0.4.11" log = "0.4.11"
meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" } meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" }
meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" } meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" }
meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.17.0" }
meilisearch-types = { path = "../meilisearch-types", version = "0.17.0" } meilisearch-types = { path = "../meilisearch-types", version = "0.17.0" }
once_cell = "1.5.2" once_cell = "1.5.2"
ordered-float = { version = "2.0.1", features = ["serde"] } ordered-float = { version = "2.0.1", features = ["serde"] }

View File

@ -1,6 +1,5 @@
mod dfa; mod dfa;
use meilisearch_tokenizer::is_cjk;
pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa}; pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};

View File

@ -4,7 +4,6 @@ use std::convert::TryFrom;
use deunicode::deunicode_with_tofu; use deunicode::deunicode_with_tofu;
use meilisearch_schema::IndexedPos; use meilisearch_schema::IndexedPos;
use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
use sdset::SetBuf; use sdset::SetBuf;
use crate::{DocIndex, DocumentId}; use crate::{DocIndex, DocumentId};

View File

@ -35,7 +35,6 @@ main_error = "0.1.1"
meilisearch-core = { path = "../meilisearch-core", version = "0.17.0" } meilisearch-core = { path = "../meilisearch-core", version = "0.17.0" }
meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" } meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" }
meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" } meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" }
meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.17.0"}
mime = "0.3.16" mime = "0.3.16"
once_cell = "1.5.2" once_cell = "1.5.2"
rand = "0.8.1" rand = "0.8.1"