From eb03a3ccb16ccba703e9847f9d292a599ff93d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 3 May 2021 15:39:53 +0200 Subject: [PATCH] Upgrade Milli and Tokenizer --- Cargo.lock | 185 ++++++++++++++++-------------------- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 85 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 778652bca..f1c109a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "actix-codec" version = "0.4.0" @@ -54,7 +52,7 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "h2 0.3.2", + "h2 0.3.3", "http", "httparse", "itoa", @@ -83,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -215,7 +213,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -266,9 +264,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -321,7 +319,7 @@ checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -332,7 +330,7 @@ checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -354,11 +352,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.57" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ed203b9ba68b242c62b3fb7480f589dd49829be1edb3fe8fc8b4ffda2dcb8d" +checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82" dependencies = [ "addr2line", + "cc", "cfg-if 1.0.0", "libc", "miniz_oxide", @@ -460,9 +459,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" dependencies = [ "lazy_static", "memchr", @@ -691,7 +690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", ] [[package]] @@ -702,17 +701,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", ] [[package]] name = "crossbeam-epoch" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", "lazy_static", "memoffset", "scopeguard", @@ -739,9 +738,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -789,7 +788,7 @@ dependencies = [ "convert_case", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -893,7 +892,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "synstructure", ] @@ -1049,7 +1048,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -1182,9 +1181,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ "bytes 1.0.1", "fnv", @@ -1391,15 +1390,15 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f006b8784cfb01fe7aa9c46f5f5cd4cf5c85a8c612a0653ec97642979062665" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" dependencies = [ "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", - "h2 0.3.2", + "h2 0.3.3", "http", "http-body 0.4.1", "httparse", @@ -1436,7 +1435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "futures-util", - "hyper 0.14.6", + "hyper 0.14.7", "log", "rustls 0.19.1", "tokio 1.5.0", @@ -1615,9 +1614,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" [[package]] name = "linked-hash-map" @@ -1656,9 +1655,9 @@ checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" [[package]] name = "lock_api" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -1761,7 +1760,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.1", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1798,22 +1797,6 @@ dependencies = [ "zip", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "meilisearch-tokenizer" version = "0.2.2" @@ -1832,9 +1815,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memmap" @@ -1878,7 +1861,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.2.2", + "meilisearch-tokenizer", "memmap", "obkv", "once_cell", @@ -2001,7 +1984,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2103,9 +2086,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +checksum = "b50b8919aecb97e5ee9aceef27e24f39c46b11831130f4a6b7b091ec5de0de12" dependencies = [ "num-traits", ] @@ -2215,7 +2198,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2284,7 +2267,7 @@ checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -2319,9 +2302,9 @@ checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "predicates" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" dependencies = [ "difference", "float-cmp", @@ -2355,7 +2338,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "version_check", ] @@ -2397,7 +2380,7 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -2572,7 +2555,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.3", + "crossbeam-utils 0.8.4", "lazy_static", "num_cpus", ] @@ -2588,18 +2571,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" dependencies = [ "aho-corasick", "memchr", @@ -2617,9 +2600,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -2680,7 +2663,7 @@ dependencies = [ "futures-util", "http", "http-body 0.4.1", - "hyper 0.14.6", + "hyper 0.14.7", "hyper-rustls 0.22.1", "ipnet", "js-sys", @@ -2725,9 +2708,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4b2e7ab0bbb2d144558ae3f4761a0db06d21463b45756fc64c3393cdba3d447" +checksum = "536cfa885fc388b8ae69edf96d7970849b7d9c1395da1b8330f17715babf8a09" dependencies = [ "bytemuck", "byteorder", @@ -2736,9 +2719,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" [[package]] name = "rustc_version" @@ -2904,7 +2887,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3040,9 +3023,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" +checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" [[package]] name = "socket2" @@ -3104,7 +3087,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3120,7 +3103,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3156,7 +3139,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3172,13 +3155,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.70" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -3198,8 +3181,8 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", - "unicode-xid 0.2.1", + "syn 1.0.72", + "unicode-xid 0.2.2", ] [[package]] @@ -3272,7 +3255,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3321,7 +3304,7 @@ dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", "standback", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3395,7 +3378,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", ] [[package]] @@ -3466,9 +3449,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", "log", @@ -3478,9 +3461,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -3575,9 +3558,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "untrusted" @@ -3606,9 +3589,9 @@ checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" [[package]] name = "utf8-width" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" +checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b" [[package]] name = "uuid" @@ -3699,7 +3682,7 @@ dependencies = [ "log", "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "wasm-bindgen-shared", ] @@ -3733,7 +3716,7 @@ checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.70", + "syn 1.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3874,12 +3857,12 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ "proc-macro2 1.0.26", - "syn 1.0.70", + "syn 1.0.72", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 68f76ebf9..7ac3ecb38 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,7 +49,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.0" } mime = "0.3.16"