fix test set

fix dump tests

Cargo.lock

@@ -210,7 +210,7 @@ dependencies = [
  "rustls 0.18.1",
  "tokio-rustls",
  "webpki",
- "webpki-roots",
+ "webpki-roots 0.20.0",
 ]
 
 [[package]]
@@ -299,6 +299,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
 
+[[package]]
+name = "ahash"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
+
 [[package]]
 name = "ahash"
 version = "0.4.7"
@@ -332,7 +338,7 @@ checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
 [[package]]
 name = "assert-json-diff"
 version = "1.0.1"
-source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4"
+source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4"
 dependencies = [
  "serde",
  "serde_json",
@@ -383,7 +389,7 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "base64 0.13.0",
- "bytes",
+ "bytes 0.5.6",
  "cfg-if 1.0.0",
  "derive_more",
  "futures-core",
@@ -570,6 +576,15 @@ version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
 
+[[package]]
+name = "cedarwood"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -582,6 +597,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "character_converter"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c"
+dependencies = [
+ "bincode",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.19"
@@ -790,12 +814,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "data-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "993a608597367c6377b258c25d7120740f00ed23a2252b729b1932dd7866f908"
-
 [[package]]
 name = "debugid"
 version = "0.7.2"
@@ -968,6 +986,16 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "fs_extra"
 version = "1.2.0"
@@ -1179,13 +1207,23 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
 
+[[package]]
+name = "hashbrown"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf"
+dependencies = [
+ "ahash 0.3.8",
+ "autocfg",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 dependencies = [
- "ahash",
+ "ahash 0.4.7",
  "serde",
 ]
 
@@ -1225,9 +1263,9 @@ checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c"
 
 [[package]]
 name = "heed-types"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72fc61caee13e85ea330eabf0c6c7098c511ff173bcb57a760b1eda3bba9f6eb"
+checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405"
 dependencies = [
  "bincode",
  "heed-traits",
@@ -1376,7 +1414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.9.1",
  "serde",
 ]
 
@@ -1470,6 +1508,21 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "jieba-rs"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a"
+dependencies = [
+ "cedarwood",
+ "fxhash",
+ "hashbrown 0.9.1",
+ "lazy_static",
+ "phf",
+ "phf_codegen",
+ "regex",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.46"
@@ -1512,9 +1565,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
+checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
 
 [[package]]
 name = "linked-hash-map"
@@ -1602,7 +1655,7 @@ dependencies = [
  "either",
  "env_logger 0.8.2",
  "fst",
- "hashbrown",
+ "hashbrown 0.9.1",
  "heed",
  "indexmap",
  "intervaltree",
@@ -1612,6 +1665,7 @@ dependencies = [
  "log",
  "meilisearch-error",
  "meilisearch-schema",
+ "meilisearch-tokenizer",
  "meilisearch-types",
  "once_cell",
  "ordered-float",
@@ -1987,6 +2041,44 @@ dependencies = [
  "sha-1 0.8.2",
 ]
 
+[[package]]
+name = "phf"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
+dependencies = [
+ "phf_shared",
+ "rand 0.7.3",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.27"
@@ -2153,6 +2245,7 @@ dependencies = [
  "rand_chacha 0.2.2",
  "rand_core 0.5.1",
  "rand_hc 0.2.0",
+ "rand_pcg",
 ]
 
 [[package]]
@@ -2238,6 +2331,15 @@ dependencies = [
  "rand_core 0.6.1",
 ]
 
+[[package]]
+name = "rand_pcg"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
 [[package]]
 name = "rand_xoshiro"
 version = "0.4.0"
@@ -2355,7 +2457,6 @@ dependencies = [
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
- "wasm-bindgen-test",
  "web-sys",
  "webpki-roots 0.20.0",
  "winreg 0.7.0",
@@ -2462,12 +2563,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -3397,30 +3492,6 @@ name = "web-sys"
 version = "0.3.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3"
-dependencies = [
- "console_error_panic_hook",
- "js-sys",
- "scoped-tls",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "wasm-bindgen-test-macro",
-]
-
-[[package]]
-name = "webpki"
-version = "0.21.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3436,6 +3507,15 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "webpki-roots"
 version = "0.21.0"
@@ -3445,6 +3525,15 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "whatlang"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
+dependencies = [
+ "hashbrown 0.7.2",
+]
+
 [[package]]
 name = "whoami"
 version = "1.0.3"

meilisearch-core/src/query_builder.rs

@@ -444,7 +444,7 @@ mod tests {
                 for index in indexes {
                     let name = index.attribute.to_string();
                     schema.insert(&name).unwrap();
-                    let indexed_pos = schema.set_indexed(&name).unwrap().1;
+                    let indexed_pos = schema.insert_with_position(&name).unwrap().1;
                     let index = DocIndex {
                         attribute: indexed_pos.0,
                         ..*index

meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl

@@ -74,4 +74,4 @@
 {"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]}
 {"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]}
 {"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]}
-{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]}
+{"id":77,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Red","name":"孫武","gender":"male","email":"SunTzu@chorizon.com","phone":"+1 (810) 407-3258","address":"吴國","about":"孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]}

meilisearch-http/tests/assets/test_set.json

@@ -1596,12 +1596,12 @@
     "picture": "http://placehold.it/32x32",
     "age": 25,
     "color": "Red",
-    "name": "孙武",
+    "name": "孫武",
     "gender": "male",
     "email": "SunTzu@chorizon.com",
     "phone": "+1 (810) 407-3258",
-    "address": "吴国",
+    "address": "吴國",
-    "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孙子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为乐安孙氏始祖，次子孙明为富春孙氏始祖。\r\n",
+    "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n",
     "registered": "2014-10-20T10:13:32 -02:00",
     "latitude": 17.11935,
     "longitude": 65.38197,

meilisearch-http/tests/search.rs

@@ -376,12 +376,12 @@ async fn search_with_attribute_to_highlight_wildcard_chinese() {
             "picture": "http://placehold.it/32x32",
             "age": 25,
             "color": "Red",
-            "name": "孙武",
+            "name": "孫武",
             "gender": "male",
             "email": "SunTzu@chorizon.com",
             "phone": "+1 (810) 407-3258",
-            "address": "吴国",
+            "address": "吴國",
-            "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孙子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为乐安孙氏始祖，次子孙明为富春孙氏始祖。\r\n",
+            "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n",
             "registered": "2014-10-20T10:13:32 -02:00",
             "latitude": 17.11935,
             "longitude": 65.38197,
@@ -396,12 +396,12 @@ async fn search_with_attribute_to_highlight_wildcard_chinese() {
             "picture": "http://placehold.it/32x32",
             "age": 25,
             "color": "Red",
-            "name": "<em>孙武</em>",
+            "name": "<em>孫武</em>",
             "gender": "male",
             "email": "SunTzu@chorizon.com",
             "phone": "+1 (810) 407-3258",
-            "address": "吴国",
+            "address": "吴國",
-            "about": "<em>孫武</em>（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《<em>孙子</em>兵法》的作者，後人尊稱為<em>孫子</em>、兵聖、東方兵聖，山東、蘇州等地尚有祀奉<em>孫武</em>的廟宇兵聖廟。其族人为乐安<em>孙氏</em>始祖，次<em>子孙</em>明为富春孙氏始祖。\r\n",
+            "about": "<em>孫武</em>（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《<em>孫子</em>兵法》的作者，後人尊稱為<em>孫子</em>、兵聖、東方兵聖，山東、蘇州等地尚有祀奉<em>孫武</em>的廟宇兵聖廟。其族人为樂安<em>孫氏</em>始祖，次<em>子孙</em>明为富春孫氏始祖。\r\n",
             "registered": "2014-10-20T10:13:32 -02:00",
             "latitude": 17.11935,
             "longitude": 65.38197,