mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-22 21:04:27 +01:00
use meta from milli
This commit is contained in:
mpostma
parent
1fad72e019
commit
79a4bc8129
228
Cargo.lock
generated
228
Cargo.lock
generated
@ -159,8 +159,8 @@ version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -169,8 +169,8 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -363,9 +363,9 @@ version = "0.5.0-beta.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -462,9 +462,9 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -473,9 +473,9 @@ version = "0.1.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -925,9 +925,9 @@ version = "0.99.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -982,9 +982,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1029,9 +1029,9 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@ -1170,9 +1170,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7"
|
||||
dependencies = [
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1721,6 +1721,28 @@ dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "logging_timer"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793"
|
||||
dependencies = [
|
||||
"log",
|
||||
"logging_timer_proc_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "logging_timer_proc_macros"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07"
|
||||
dependencies = [
|
||||
"log",
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"syn 0.15.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-cache"
|
||||
version = "0.1.2"
|
||||
@ -1866,11 +1888,11 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "milli"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bstr",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
"csv",
|
||||
"either",
|
||||
@ -1884,6 +1906,7 @@ dependencies = [
|
||||
"levenshtein_automata",
|
||||
"linked-hash-map",
|
||||
"log",
|
||||
"logging_timer",
|
||||
"meilisearch-tokenizer",
|
||||
"memmap",
|
||||
"num-traits",
|
||||
@ -1897,9 +1920,11 @@ dependencies = [
|
||||
"roaring",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"slice-group-by",
|
||||
"smallstr",
|
||||
"smallvec",
|
||||
"tempfile",
|
||||
"tinytemplate",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@ -2170,9 +2195,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55"
|
||||
dependencies = [
|
||||
"pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pest_meta",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2248,9 +2273,9 @@ version = "0.4.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2259,9 +2284,9 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2301,9 +2326,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
@ -2313,8 +2338,8 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
@ -2330,13 +2355,22 @@ version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
dependencies = [
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2345,13 +2379,22 @@ version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "0.6.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"proc-macro2 1.0.24",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2799,9 +2842,9 @@ version = "1.0.124"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2987,11 +3030,11 @@ version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"syn",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3001,13 +3044,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
|
||||
dependencies = [
|
||||
"base-x",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"sha1",
|
||||
"syn",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3041,9 +3084,20 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "0.15.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3052,9 +3106,9 @@ version = "1.0.63"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3072,10 +3126,10 @@ version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"unicode-xid",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3146,9 +3200,9 @@ version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3212,10 +3266,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa"
|
||||
dependencies = [
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"standback",
|
||||
"syn",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3281,9 +3345,9 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3479,6 +3543,12 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.1"
|
||||
@ -3592,9 +3662,9 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@ -3616,7 +3686,7 @@ version = "0.2.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"quote 1.0.9",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
@ -3626,9 +3696,9 @@ version = "0.2.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.9",
|
||||
"syn 1.0.63",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@ -3788,8 +3858,8 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"syn 1.0.63",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
|
||||
@ -38,7 +38,8 @@ main_error = "0.1.0"
|
||||
meilisearch-error = { path = "../meilisearch-error" }
|
||||
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
|
||||
memmap = "0.7.0"
|
||||
milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" }
|
||||
milli = { path = "../../milli/milli" }
|
||||
#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" }
|
||||
mime = "0.3.16"
|
||||
once_cell = "1.5.2"
|
||||
rand = "0.7.3"
|
||||
|
||||
@ -6,7 +6,7 @@ use either::Either;
|
||||
use anyhow::bail;
|
||||
use heed::RoTxn;
|
||||
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
|
||||
use milli::{FacetCondition, facet::FacetValue};
|
||||
use milli::{FacetCondition, MatchingWords, facet::FacetValue};
|
||||
use serde::{Serialize, Deserialize};
|
||||
use serde_json::{Value, Map};
|
||||
|
||||
@ -71,7 +71,7 @@ impl Index {
|
||||
|
||||
let milli::SearchResult {
|
||||
documents_ids,
|
||||
found_words,
|
||||
matching_words,
|
||||
candidates,
|
||||
..
|
||||
} = search.execute()?;
|
||||
@ -102,7 +102,7 @@ impl Index {
|
||||
for (_id, obkv) in self.documents(&rtxn, documents_ids)? {
|
||||
let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap();
|
||||
if let Some(ref attributes_to_highlight) = query.attributes_to_highlight {
|
||||
highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight);
|
||||
highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight);
|
||||
}
|
||||
documents.push(object);
|
||||
}
|
||||
@ -173,7 +173,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
|
||||
Self { analyzer }
|
||||
}
|
||||
|
||||
pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet<String>) -> Value {
|
||||
pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value {
|
||||
match value {
|
||||
Value::Null => Value::Null,
|
||||
Value::Bool(boolean) => Value::Bool(boolean),
|
||||
@ -183,7 +183,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
|
||||
let analyzed = self.analyzer.analyze(&old_string);
|
||||
for (word, token) in analyzed.reconstruct() {
|
||||
if token.is_word() {
|
||||
let to_highlight = words_to_highlight.contains(token.text());
|
||||
let to_highlight = words_to_highlight.matches(token.text());
|
||||
if to_highlight {
|
||||
string.push_str("<mark>")
|
||||
}
|
||||
@ -215,7 +215,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
|
||||
pub fn highlight_record(
|
||||
&self,
|
||||
object: &mut Map<String, Value>,
|
||||
words_to_highlight: &HashSet<String>,
|
||||
words_to_highlight: &MatchingWords,
|
||||
attributes_to_highlight: &HashSet<String>,
|
||||
) {
|
||||
// TODO do we need to create a string for element that are not and needs to be highlight?
|
||||
|
||||
@ -12,7 +12,7 @@ use super::Index;
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum UpdateResult {
|
||||
DocumentsAddition(DocumentAdditionResult),
|
||||
DocumentDeletion { deleted: usize },
|
||||
DocumentDeletion { deleted: u64 },
|
||||
Other,
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{create_dir_all, remove_dir_all, File};
|
||||
use std::fs::{create_dir_all, File};
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
@ -8,15 +8,13 @@ use async_stream::stream;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::pin_mut;
|
||||
use futures::stream::StreamExt;
|
||||
use heed::{
|
||||
types::{ByteSlice, SerdeBincode},
|
||||
Database, Env, EnvOpenOptions,
|
||||
};
|
||||
use heed::EnvOpenOptions;
|
||||
use log::debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use tokio::{sync::{mpsc, oneshot, RwLock}};
|
||||
use tokio::sync::{mpsc, oneshot, RwLock};
|
||||
use tokio::task::spawn_blocking;
|
||||
use tokio::fs::remove_dir_all;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::get_arc_ownership_blocking;
|
||||
@ -36,12 +34,21 @@ type UpdateResult = std::result::Result<Processed<UpdateMeta, UResult>, Failed<U
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct IndexMeta {
|
||||
uuid: Uuid,
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
primary_key: Option<String>,
|
||||
}
|
||||
|
||||
impl IndexMeta {
|
||||
fn new(index: &Index) -> Result<Self> {
|
||||
let txn = index.read_txn()?;
|
||||
let created_at = index.created_at(&txn)?;
|
||||
let updated_at = index.updated_at(&txn)?;
|
||||
let primary_key = index.primary_key(&txn)?.map(String::from);
|
||||
Ok(Self { primary_key, updated_at, created_at })
|
||||
}
|
||||
}
|
||||
|
||||
enum IndexMsg {
|
||||
CreateIndex {
|
||||
uuid: Uuid,
|
||||
@ -106,14 +113,9 @@ pub enum IndexError {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
trait IndexStore {
|
||||
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>;
|
||||
async fn update_index<R, F>(&self, uuid: Uuid, f: F) -> Result<R>
|
||||
where
|
||||
F: FnOnce(Index) -> Result<R> + Send + Sync + 'static,
|
||||
R: Sync + Send + 'static;
|
||||
async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index>;
|
||||
async fn get(&self, uuid: Uuid) -> Result<Option<Index>>;
|
||||
async fn delete(&self, uuid: Uuid) -> Result<Option<Index>>;
|
||||
async fn get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>>;
|
||||
}
|
||||
|
||||
impl<S: IndexStore + Sync + Send> IndexActor<S> {
|
||||
@ -245,7 +247,11 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
|
||||
uuid: Uuid,
|
||||
primary_key: Option<String>,
|
||||
) -> Result<IndexMeta> {
|
||||
self.store.create_index(uuid, primary_key).await
|
||||
let index = self.store.create(uuid, primary_key).await?;
|
||||
let meta = spawn_blocking(move || IndexMeta::new(&index))
|
||||
.await
|
||||
.map_err(|e| IndexError::Error(e.into()))??;
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
async fn handle_update(
|
||||
@ -256,16 +262,13 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
|
||||
debug!("Processing update {}", meta.id());
|
||||
let uuid = meta.index_uuid().clone();
|
||||
let update_handler = self.update_handler.clone();
|
||||
let handle = self
|
||||
.store
|
||||
.update_index(uuid, |index| {
|
||||
let handle =
|
||||
spawn_blocking(move || update_handler.handle_update(meta, data, index));
|
||||
Ok(handle)
|
||||
})
|
||||
.await?;
|
||||
|
||||
handle.await.map_err(|e| IndexError::Error(e.into()))
|
||||
let index = match self.store.get(uuid.clone()).await? {
|
||||
Some(index) => index,
|
||||
None => self.store.create(uuid, None).await?,
|
||||
};
|
||||
spawn_blocking(move || update_handler.handle_update(meta, data, index))
|
||||
.await
|
||||
.map_err(|e| IndexError::Error(e.into()))
|
||||
}
|
||||
|
||||
async fn handle_settings(&self, uuid: Uuid) -> Result<Settings> {
|
||||
@ -338,8 +341,15 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
|
||||
}
|
||||
|
||||
async fn handle_get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>> {
|
||||
let result = self.store.get_meta(uuid).await?;
|
||||
Ok(result)
|
||||
match self.store.get(uuid).await? {
|
||||
Some(index) => {
|
||||
let meta = spawn_blocking(move || IndexMeta::new(&index))
|
||||
.await
|
||||
.map_err(|e| IndexError::Error(e.into()))??;
|
||||
Ok(Some(meta))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -451,24 +461,15 @@ impl IndexActorHandle {
|
||||
}
|
||||
|
||||
struct HeedIndexStore {
|
||||
env: Env,
|
||||
db: Database<ByteSlice, SerdeBincode<IndexMeta>>,
|
||||
index_store: AsyncMap<Uuid, Index>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl HeedIndexStore {
|
||||
fn new(path: impl AsRef<Path>) -> anyhow::Result<Self> {
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(1_073_741_824); //1GB
|
||||
let path = path.as_ref().join("indexes/");
|
||||
create_dir_all(&path)?;
|
||||
let env = options.open(&path)?;
|
||||
let db = env.create_database(None)?;
|
||||
let index_store = Arc::new(RwLock::new(HashMap::new()));
|
||||
Ok(Self {
|
||||
env,
|
||||
db,
|
||||
index_store,
|
||||
path,
|
||||
})
|
||||
@ -477,76 +478,22 @@ impl HeedIndexStore {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl IndexStore for HeedIndexStore {
|
||||
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
|
||||
async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index> {
|
||||
let path = self.path.join(format!("index-{}", uuid));
|
||||
|
||||
if path.exists() {
|
||||
return Err(IndexError::IndexAlreadyExists);
|
||||
}
|
||||
|
||||
let env = self.env.clone();
|
||||
let db = self.db.clone();
|
||||
let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> {
|
||||
let now = Utc::now();
|
||||
let meta = IndexMeta {
|
||||
uuid: uuid.clone(),
|
||||
created_at: now.clone(),
|
||||
updated_at: now,
|
||||
primary_key,
|
||||
};
|
||||
let mut txn = env.write_txn()?;
|
||||
db.put(&mut txn, uuid.as_bytes(), &meta)?;
|
||||
txn.commit()?;
|
||||
|
||||
let index = spawn_blocking(move || -> Result<Index> {
|
||||
let index = open_index(&path, 4096 * 100_000)?;
|
||||
|
||||
Ok((index, meta))
|
||||
Ok(index)
|
||||
})
|
||||
.await
|
||||
.expect("thread died")?;
|
||||
.map_err(|e| IndexError::Error(e.into()))??;
|
||||
|
||||
self.index_store.write().await.insert(uuid.clone(), index);
|
||||
self.index_store.write().await.insert(uuid.clone(), index.clone());
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
async fn update_index<R, F>(&self, uuid: Uuid, f: F) -> Result<R>
|
||||
where
|
||||
F: FnOnce(Index) -> Result<R> + Send + Sync + 'static,
|
||||
R: Sync + Send + 'static,
|
||||
{
|
||||
let guard = self.index_store.read().await;
|
||||
let index = match guard.get(&uuid) {
|
||||
Some(index) => index.clone(),
|
||||
None => {
|
||||
drop(guard);
|
||||
self.create_index(uuid.clone(), None).await?;
|
||||
self.index_store
|
||||
.read()
|
||||
.await
|
||||
.get(&uuid)
|
||||
.expect("Index should exist")
|
||||
.clone()
|
||||
}
|
||||
};
|
||||
|
||||
let env = self.env.clone();
|
||||
let db = self.db.clone();
|
||||
spawn_blocking(move || {
|
||||
let mut txn = env.write_txn()?;
|
||||
let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index");
|
||||
match f(index) {
|
||||
Ok(r) => {
|
||||
meta.updated_at = Utc::now();
|
||||
db.put(&mut txn, uuid.as_bytes(), &meta)?;
|
||||
txn.commit()?;
|
||||
Ok(r)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("thread died")
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
async fn get(&self, uuid: Uuid) -> Result<Option<Index>> {
|
||||
@ -561,54 +508,33 @@ impl IndexStore for HeedIndexStore {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// TODO: set this info from the database
|
||||
let index = spawn_blocking(|| open_index(path, 4096 * 100_000))
|
||||
.await
|
||||
.expect("thread died")?;
|
||||
.map_err(|e| IndexError::Error(e.into()))??;
|
||||
self.index_store
|
||||
.write()
|
||||
.await
|
||||
.insert(uuid.clone(), index.clone());
|
||||
println!("here");
|
||||
Ok(Some(index))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete(&self, uuid: Uuid) -> Result<Option<Index>> {
|
||||
let env = self.env.clone();
|
||||
let db = self.db.clone();
|
||||
let db_path = self.path.join(format!("index-{}", uuid));
|
||||
spawn_blocking(move || -> Result<()> {
|
||||
let mut txn = env.write_txn()?;
|
||||
db.delete(&mut txn, uuid.as_bytes())?;
|
||||
txn.commit()?;
|
||||
remove_dir_all(db_path).unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.expect("thread died")?;
|
||||
remove_dir_all(db_path).await
|
||||
.map_err(|e| IndexError::Error(e.into()))?;
|
||||
let index = self.index_store.write().await.remove(&uuid);
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
async fn get_meta(&self, uuid: Uuid) -> Result<Option<IndexMeta>> {
|
||||
let env = self.env.clone();
|
||||
let db = self.db.clone();
|
||||
spawn_blocking(move || {
|
||||
let txn = env.read_txn()?;
|
||||
let meta = db.get(&txn, uuid.as_bytes())?;
|
||||
Ok(meta)
|
||||
})
|
||||
.await
|
||||
.expect("thread died")
|
||||
}
|
||||
}
|
||||
|
||||
fn open_index(path: impl AsRef<Path>, size: usize) -> Result<Index> {
|
||||
create_dir_all(&path).expect("can't create db");
|
||||
create_dir_all(&path)
|
||||
.map_err(|e| IndexError::Error(e.into()))?;
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(size);
|
||||
let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?;
|
||||
let index = milli::Index::new(options, &path)
|
||||
.map_err(|e| IndexError::Error(e))?;
|
||||
Ok(Index(Arc::new(index)))
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user