searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 14:54:27 +01:00
Code Issues Releases Wiki Activity

use meta from milli

Browse Source
This commit is contained in:
mpostma 2021-03-11 19:40:18 +01:00
parent 1fad72e019
commit 79a4bc8129
No known key found for this signature in database
GPG Key ID: CBC8A7C1D7A28C3A
5 changed files with 207 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
Cargo.lock generated
View File

@ -159,8 +159,8 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655"
dependencies = [ dependencies = [
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -169,8 +169,8 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef"
dependencies = [ dependencies = [
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -363,9 +363,9 @@ version = "0.5.0-beta.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -462,9 +462,9 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -473,9 +473,9 @@ version = "0.1.48"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -925,9 +925,9 @@ version = "0.99.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -982,9 +982,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595"
dependencies = [ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -1029,9 +1029,9 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
"synstructure", "synstructure",
] ]
@ -1170,9 +1170,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -1721,6 +1721,28 @@ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
] ]
[[package]]
name = "logging_timer"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793"
dependencies = [
"log",
"logging_timer_proc_macros",
]
[[package]]
name = "logging_timer_proc_macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07"
dependencies = [
"log",
"proc-macro2 0.4.30",
"quote 0.6.13",
"syn 0.15.44",
]
[[package]] [[package]]
name = "lru-cache" name = "lru-cache"
version = "0.1.2" version = "0.1.2"
@ -1866,11 +1888,11 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bstr", "bstr",
"byteorder", "byteorder",
"chrono",
"crossbeam-channel", "crossbeam-channel",
"csv", "csv",
"either", "either",
@ -1884,6 +1906,7 @@ dependencies = [
"levenshtein_automata", "levenshtein_automata",
"linked-hash-map", "linked-hash-map",
"log", "log",
"logging_timer",
"meilisearch-tokenizer", "meilisearch-tokenizer",
"memmap", "memmap",
"num-traits", "num-traits",
@ -1897,9 +1920,11 @@ dependencies = [
"roaring", "roaring",
"serde", "serde",
"serde_json", "serde_json",
"slice-group-by",
"smallstr", "smallstr",
"smallvec", "smallvec",
"tempfile", "tempfile",
"tinytemplate",
"uuid", "uuid",
] ]
@ -2170,9 +2195,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55"
dependencies = [ dependencies = [
"pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_meta", "pest_meta",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -2248,9 +2273,9 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -2259,9 +2284,9 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -2301,9 +2326,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [ dependencies = [
"proc-macro-error-attr", "proc-macro-error-attr",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
"version_check", "version_check",
] ]
@ -2313,8 +2338,8 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"version_check", "version_check",
] ]
@ -2330,13 +2355,22 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
[[package]]
name = "proc-macro2"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
dependencies = [
"unicode-xid 0.1.0",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.24" version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
dependencies = [ dependencies = [
"unicode-xid", "unicode-xid 0.2.1",
] ]
[[package]] [[package]]
@ -2345,13 +2379,22 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quote"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
dependencies = [
"proc-macro2 0.4.30",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.9" version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
] ]
[[package]] [[package]]
@ -2799,9 +2842,9 @@ version = "1.0.124"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -2987,11 +3030,11 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"serde", "serde",
"serde_derive", "serde_derive",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -3001,13 +3044,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
dependencies = [ dependencies = [
"base-x", "base-x",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"serde", "serde",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"sha1", "sha1",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -3041,9 +3084,20 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90"
dependencies = [ dependencies = [
"heck", "heck",
"proc-macro-error", "proc-macro-error",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
]
[[package]]
name = "syn"
version = "0.15.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
dependencies = [
"proc-macro2 0.4.30",
"quote 0.6.13",
"unicode-xid 0.1.0",
] ]
[[package]] [[package]]
@ -3052,9 +3106,9 @@ version = "1.0.63"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"unicode-xid", "unicode-xid 0.2.1",
] ]
[[package]] [[package]]
@ -3072,10 +3126,10 @@ version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
"unicode-xid", "unicode-xid 0.2.1",
] ]
[[package]] [[package]]
@ -3146,9 +3200,9 @@ version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -3212,10 +3266,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"standback", "standback",
"syn", "syn 1.0.63",
]
[[package]]
name = "tinytemplate"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f"
dependencies = [
"serde",
"serde_json",
] ]
[[package]] [[package]]
@ -3281,9 +3345,9 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
] ]
[[package]] [[package]]
@ -3479,6 +3543,12 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"
version = "0.2.1" version = "0.2.1"
@ -3592,9 +3662,9 @@ dependencies = [
"bumpalo", "bumpalo",
"lazy_static", "lazy_static",
"log", "log",
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3616,7 +3686,7 @@ version = "0.2.71"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b"
dependencies = [ dependencies = [
"quote", "quote 1.0.9",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
] ]
@ -3626,9 +3696,9 @@ version = "0.2.71"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.9",
"syn", "syn 1.0.63",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3788,8 +3858,8 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"syn", "syn 1.0.63",
"synstructure", "synstructure",
] ]

3
meilisearch-http/Cargo.toml
View File

@ -38,7 +38,8 @@ main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" } meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
memmap = "0.7.0" memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } milli = { path = "../../milli/milli" }
#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" }
mime = "0.3.16" mime = "0.3.16"
once_cell = "1.5.2" once_cell = "1.5.2"
rand = "0.7.3" rand = "0.7.3"

12
meilisearch-http/src/index/search.rs
View File

@ -6,7 +6,7 @@ use either::Either;
use anyhow::bail; use anyhow::bail;
use heed::RoTxn; use heed::RoTxn;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use milli::{FacetCondition, facet::FacetValue}; use milli::{FacetCondition, MatchingWords, facet::FacetValue};
use serde::{Serialize, Deserialize}; use serde::{Serialize, Deserialize};
use serde_json::{Value, Map}; use serde_json::{Value, Map};
@ -71,7 +71,7 @@ impl Index {
let milli::SearchResult { let milli::SearchResult {
documents_ids, documents_ids,
found_words, matching_words,
candidates, candidates,
.. ..
} = search.execute()?; } = search.execute()?;
@ -102,7 +102,7 @@ impl Index {
for (_id, obkv) in self.documents(&rtxn, documents_ids)? { for (_id, obkv) in self.documents(&rtxn, documents_ids)? {
let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap();
if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { if let Some(ref attributes_to_highlight) = query.attributes_to_highlight {
highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight);
} }
documents.push(object); documents.push(object);
} }
@ -173,7 +173,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
Self { analyzer } Self { analyzer }
} }
pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet<String>) -> Value { pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value {
match value { match value {
Value::Null => Value::Null, Value::Null => Value::Null,
Value::Bool(boolean) => Value::Bool(boolean), Value::Bool(boolean) => Value::Bool(boolean),
@ -183,7 +183,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
let analyzed = self.analyzer.analyze(&old_string); let analyzed = self.analyzer.analyze(&old_string);
for (word, token) in analyzed.reconstruct() { for (word, token) in analyzed.reconstruct() {
if token.is_word() { if token.is_word() {
let to_highlight = words_to_highlight.contains(token.text()); let to_highlight = words_to_highlight.matches(token.text());
if to_highlight { if to_highlight {
string.push_str("<mark>") string.push_str("<mark>")
} }
@ -215,7 +215,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
pub fn highlight_record( pub fn highlight_record(
&self, &self,
object: &mut Map<String, Value>, object: &mut Map<String, Value>,
words_to_highlight: &HashSet<String>, words_to_highlight: &MatchingWords,
attributes_to_highlight: &HashSet<String>, attributes_to_highlight: &HashSet<String>,
) { ) {
// TODO do we need to create a string for element that are not and needs to be highlight? // TODO do we need to create a string for element that are not and needs to be highlight?

2
meilisearch-http/src/index/updates.rs
View File

@ -12,7 +12,7 @@ use super::Index;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub enum UpdateResult { pub enum UpdateResult {
DocumentsAddition(DocumentAdditionResult), DocumentsAddition(DocumentAdditionResult),
DocumentDeletion { deleted: usize }, DocumentDeletion { deleted: u64 },
Other, Other,
} }

172
meilisearch-http/src/index_controller/index_actor.rs
View File

@ -1,5 +1,5 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::{create_dir_all, remove_dir_all, File}; use std::fs::{create_dir_all, File};
use std::future::Future; use std::future::Future;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
@ -8,15 +8,13 @@ use async_stream::stream;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use futures::pin_mut; use futures::pin_mut;
use futures::stream::StreamExt; use futures::stream::StreamExt;
use heed::{ use heed::EnvOpenOptions;
types::{ByteSlice, SerdeBincode},
Database, Env, EnvOpenOptions,
};
use log::debug; use log::debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use tokio::{sync::{mpsc, oneshot, RwLock}}; use tokio::sync::{mpsc, oneshot, RwLock};
use tokio::task::spawn_blocking; use tokio::task::spawn_blocking;
use tokio::fs::remove_dir_all;
use uuid::Uuid; use uuid::Uuid;
use super::get_arc_ownership_blocking; use super::get_arc_ownership_blocking;
@ -36,12 +34,21 @@ type UpdateResult = std::result::Result<Processed<UpdateMeta, UResult>, Failed<U
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct IndexMeta { pub struct IndexMeta {
uuid: Uuid,
created_at: DateTime<Utc>, created_at: DateTime<Utc>,
updated_at: DateTime<Utc>, updated_at: DateTime<Utc>,
primary_key: Option<String>, primary_key: Option<String>,
} }
impl IndexMeta {
fn new(index: &Index) -> Result<Self> {
let txn = index.read_txn()?;
let created_at = index.created_at(&txn)?;
let updated_at = index.updated_at(&txn)?;
let primary_key = index.primary_key(&txn)?.map(String::from);
Ok(Self { primary_key, updated_at, created_at })
}
}
enum IndexMsg { enum IndexMsg {
CreateIndex { CreateIndex {
uuid: Uuid, uuid: Uuid,
@ -106,14 +113,9 @@ pub enum IndexError {
#[async_trait::async_trait] #[async_trait::async_trait]
trait IndexStore { trait IndexStore {
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>; async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index>;
async fn update_index<R, F>(&self, uuid: Uuid, f: F) -> Result<R>
where
F: FnOnce(Index) -> Result<R> + Send + Sync + 'static,
R: Sync + Send + 'static;
async fn get(&self, uuid: Uuid) -> Result<Option<Index>>; async fn get(&self, uuid: Uuid) -> Result<Option<Index>>;
async fn delete(&self, uuid: Uuid) -> Result<Option<Index>>; async fn delete(&self, uuid: Uuid) -> Result<Option<Index>>;
async fn get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>>;
} }
impl<S: IndexStore + Sync + Send> IndexActor<S> { impl<S: IndexStore + Sync + Send> IndexActor<S> {
@ -245,7 +247,11 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
uuid: Uuid, uuid: Uuid,
primary_key: Option<String>, primary_key: Option<String>,
) -> Result<IndexMeta> { ) -> Result<IndexMeta> {
self.store.create_index(uuid, primary_key).await let index = self.store.create(uuid, primary_key).await?;
let meta = spawn_blocking(move || IndexMeta::new(&index))
.await
.map_err(|e| IndexError::Error(e.into()))??;
Ok(meta)
} }
async fn handle_update( async fn handle_update(
@ -256,16 +262,13 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
debug!("Processing update {}", meta.id()); debug!("Processing update {}", meta.id());
let uuid = meta.index_uuid().clone(); let uuid = meta.index_uuid().clone();
let update_handler = self.update_handler.clone(); let update_handler = self.update_handler.clone();
let handle = self let index = match self.store.get(uuid.clone()).await? {
.store Some(index) => index,
.update_index(uuid, |index| { None => self.store.create(uuid, None).await?,
let handle = };
spawn_blocking(move || update_handler.handle_update(meta, data, index)); spawn_blocking(move || update_handler.handle_update(meta, data, index))
Ok(handle) .await
}) .map_err(|e| IndexError::Error(e.into()))
.await?;
handle.await.map_err(|e| IndexError::Error(e.into()))
} }
async fn handle_settings(&self, uuid: Uuid) -> Result<Settings> { async fn handle_settings(&self, uuid: Uuid) -> Result<Settings> {
@ -338,8 +341,15 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
} }
async fn handle_get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>> { async fn handle_get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>> {
let result = self.store.get_meta(uuid).await?; match self.store.get(uuid).await? {
Ok(result) Some(index) => {
let meta = spawn_blocking(move || IndexMeta::new(&index))
.await
.map_err(|e| IndexError::Error(e.into()))??;
Ok(Some(meta))
}
None => Ok(None),
}
} }
} }
@ -451,24 +461,15 @@ impl IndexActorHandle {
} }
struct HeedIndexStore { struct HeedIndexStore {
env: Env,
db: Database<ByteSlice, SerdeBincode<IndexMeta>>,
index_store: AsyncMap<Uuid, Index>, index_store: AsyncMap<Uuid, Index>,
path: PathBuf, path: PathBuf,
} }
impl HeedIndexStore { impl HeedIndexStore {
fn new(path: impl AsRef<Path>) -> anyhow::Result<Self> { fn new(path: impl AsRef<Path>) -> anyhow::Result<Self> {
let mut options = EnvOpenOptions::new();
options.map_size(1_073_741_824); //1GB
let path = path.as_ref().join("indexes/"); let path = path.as_ref().join("indexes/");
create_dir_all(&path)?;
let env = options.open(&path)?;
let db = env.create_database(None)?;
let index_store = Arc::new(RwLock::new(HashMap::new())); let index_store = Arc::new(RwLock::new(HashMap::new()));
Ok(Self { Ok(Self {
env,
db,
index_store, index_store,
path, path,
}) })
@ -477,76 +478,22 @@ impl HeedIndexStore {
#[async_trait::async_trait] #[async_trait::async_trait]
impl IndexStore for HeedIndexStore { impl IndexStore for HeedIndexStore {
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> { async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index> {
let path = self.path.join(format!("index-{}", uuid)); let path = self.path.join(format!("index-{}", uuid));
if path.exists() { if path.exists() {
return Err(IndexError::IndexAlreadyExists); return Err(IndexError::IndexAlreadyExists);
} }
let env = self.env.clone(); let index = spawn_blocking(move || -> Result<Index> {
let db = self.db.clone();
let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> {
let now = Utc::now();
let meta = IndexMeta {
uuid: uuid.clone(),
created_at: now.clone(),
updated_at: now,
primary_key,
};
let mut txn = env.write_txn()?;
db.put(&mut txn, uuid.as_bytes(), &meta)?;
txn.commit()?;
let index = open_index(&path, 4096 * 100_000)?; let index = open_index(&path, 4096 * 100_000)?;
Ok(index)
Ok((index, meta))
}) })
.await .await
.expect("thread died")?; .map_err(|e| IndexError::Error(e.into()))??;
self.index_store.write().await.insert(uuid.clone(), index); self.index_store.write().await.insert(uuid.clone(), index.clone());
Ok(meta) Ok(index)
}
async fn update_index<R, F>(&self, uuid: Uuid, f: F) -> Result<R>
where
F: FnOnce(Index) -> Result<R> + Send + Sync + 'static,
R: Sync + Send + 'static,
{
let guard = self.index_store.read().await;
let index = match guard.get(&uuid) {
Some(index) => index.clone(),
None => {
drop(guard);
self.create_index(uuid.clone(), None).await?;
self.index_store
.read()
.await
.get(&uuid)
.expect("Index should exist")
.clone()
}
};
let env = self.env.clone();
let db = self.db.clone();
spawn_blocking(move || {
let mut txn = env.write_txn()?;
let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index");
match f(index) {
Ok(r) => {
meta.updated_at = Utc::now();
db.put(&mut txn, uuid.as_bytes(), &meta)?;
txn.commit()?;
Ok(r)
}
Err(e) => Err(e),
}
})
.await
.expect("thread died")
} }
async fn get(&self, uuid: Uuid) -> Result<Option<Index>> { async fn get(&self, uuid: Uuid) -> Result<Option<Index>> {
@ -561,54 +508,33 @@ impl IndexStore for HeedIndexStore {
return Ok(None); return Ok(None);
} }
// TODO: set this info from the database
let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) let index = spawn_blocking(|| open_index(path, 4096 * 100_000))
.await .await
.expect("thread died")?; .map_err(|e| IndexError::Error(e.into()))??;
self.index_store self.index_store
.write() .write()
.await .await
.insert(uuid.clone(), index.clone()); .insert(uuid.clone(), index.clone());
println!("here");
Ok(Some(index)) Ok(Some(index))
} }
} }
} }
async fn delete(&self, uuid: Uuid) -> Result<Option<Index>> { async fn delete(&self, uuid: Uuid) -> Result<Option<Index>> {
let env = self.env.clone();
let db = self.db.clone();
let db_path = self.path.join(format!("index-{}", uuid)); let db_path = self.path.join(format!("index-{}", uuid));
spawn_blocking(move || -> Result<()> { remove_dir_all(db_path).await
let mut txn = env.write_txn()?; .map_err(|e| IndexError::Error(e.into()))?;
db.delete(&mut txn, uuid.as_bytes())?;
txn.commit()?;
remove_dir_all(db_path).unwrap();
Ok(())
})
.await
.expect("thread died")?;
let index = self.index_store.write().await.remove(&uuid); let index = self.index_store.write().await.remove(&uuid);
Ok(index) Ok(index)
} }
async fn get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>> {
let env = self.env.clone();
let db = self.db.clone();
spawn_blocking(move || {
let txn = env.read_txn()?;
let meta = db.get(&txn, uuid.as_bytes())?;
Ok(meta)
})
.await
.expect("thread died")
}
} }
fn open_index(path: impl AsRef<Path>, size: usize) -> Result<Index> { fn open_index(path: impl AsRef<Path>, size: usize) -> Result<Index> {
create_dir_all(&path).expect("can't create db"); create_dir_all(&path)
.map_err(|e| IndexError::Error(e.into()))?;
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(size); options.map_size(size);
let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; let index = milli::Index::new(options, &path)
.map_err(|e| IndexError::Error(e))?;
Ok(Index(Arc::new(index))) Ok(Index(Arc::new(index)))
} }