From 61ce749122acf0cf1b8521b19ace855167b8ce74 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 09:10:04 +0100 Subject: [PATCH 01/69] update tokio and disable all routes --- Cargo.lock | 532 +++++++++++++++++++++++++--------- Cargo.toml | 16 +- src/data/mod.rs | 29 +- src/data/search.rs | 150 +++++----- src/data/updates.rs | 100 +++---- src/error.rs | 9 +- src/helpers/authentication.rs | 24 +- src/helpers/normalize_path.rs | 2 +- src/index_controller/mod.rs | 46 ++- src/lib.rs | 52 +--- src/main.rs | 55 +++- src/routes/document.rs | 122 ++++---- src/routes/index.rs | 2 +- src/routes/key.rs | 2 +- 14 files changed, 680 insertions(+), 461 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e4054d87..da8d649dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,8 +12,24 @@ dependencies = [ "futures-sink", "log", "pin-project 0.4.27", - "tokio", - "tokio-util", + "tokio 0.2.24", + "tokio-util 0.3.1", +] + +[[package]] +name = "actix-codec" +version = "0.4.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" +dependencies = [ + "bitflags", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", + "tokio-util 0.6.3", ] [[package]] @@ -22,27 +38,22 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-utils 2.0.0", "derive_more", "either", "futures-util", "http", "log", - "rustls", - "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", - "webpki", ] [[package]] name = "actix-cors" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3a3d5493dbc9b8769fe88c030d057ef8d2edc5728e5e26267780e8fc5db0be" +version = "0.5.4" dependencies = [ "actix-web", "derive_more", @@ -58,28 +69,25 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-connect", - "actix-rt", - "actix-service", + "actix-rt 1.1.1", + "actix-service 1.0.6", "actix-threadpool", - "actix-tls", - "actix-utils", + "actix-utils 2.0.0", "base64 0.13.0", "bitflags", - "brotli2", "bytes 0.5.6", "cookie", "copyless", "derive_more", "either", "encoding_rs", - "flate2", "futures-channel", "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -100,6 +108,52 @@ dependencies = [ "time 0.2.23", ] +[[package]] +name = "actix-http" +version = "3.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a12706e793a92377f85cec219514b72625b3b89f9b4912d8bfb53ab6a615bf0" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "ahash 0.7.0", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 1.0.1", + "bytestring", + "cookie", + "derive_more", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.0", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.8.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "smallvec", + "time 0.2.23", +] + [[package]] name = "actix-macros" version = "0.1.3" @@ -111,10 +165,20 @@ dependencies = [ ] [[package]] -name = "actix-router" -version = "0.2.5" +name = "actix-macros" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", @@ -129,33 +193,42 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" dependencies = [ - "actix-macros", + "actix-macros 0.1.3", "actix-threadpool", "copyless", "futures-channel", "futures-util", "smallvec", - "tokio", + "tokio 0.2.24", +] + +[[package]] +name = "actix-rt" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" +dependencies = [ + "actix-macros 0.2.0", + "futures-core", + "tokio 1.2.0", ] [[package]] name = "actix-server" -version = "1.0.4" +version = "2.0.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "futures-channel", - "futures-util", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "futures-core", "log", - "mio", - "mio-uds", + "mio 0.7.9", "num_cpus", "slab", - "socket2", + "tokio 1.2.0", ] [[package]] @@ -169,17 +242,13 @@ dependencies = [ ] [[package]] -name = "actix-testing" -version = "1.0.1" +name = "actix-service" +version = "2.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" dependencies = [ - "actix-macros", - "actix-rt", - "actix-server", - "actix-service", - "log", - "socket2", + "futures-core", + "pin-project-lite 0.2.0", ] [[package]] @@ -199,18 +268,21 @@ dependencies = [ [[package]] name = "actix-tls" -version = "2.0.0" +version = "3.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" dependencies = [ - "actix-codec", - "actix-service", - "actix-utils", - "futures-util", - "rustls", - "tokio-rustls", - "webpki", - "webpki-roots", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "derive_more", + "futures-core", + "http", + "log", + "tokio-rustls 0.22.0", + "tokio-util 0.6.3", + "webpki-roots 0.21.0", ] [[package]] @@ -219,9 +291,9 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", "bitflags", "bytes 0.5.6", "either", @@ -234,50 +306,63 @@ dependencies = [ ] [[package]] -name = "actix-web" -version = "3.3.2" +name = "actix-utils" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-web" +version = "4.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9683dc8c3037ea524e0fec6032d34e1cb1ee72c4eb8689f428a60c2a544ea3" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.3", + "actix-macros 0.2.0", "actix-router", - "actix-rt", + "actix-rt 2.1.0", "actix-server", - "actix-service", - "actix-testing", - "actix-threadpool", + "actix-service 2.0.0-beta.4", "actix-tls", - "actix-utils", + "actix-utils 3.0.0-beta.2", "actix-web-codegen", + "ahash 0.7.0", "awc", - "bytes 0.5.6", + "bytes 1.0.1", "derive_more", + "either", "encoding_rs", - "futures-channel", "futures-core", "futures-util", - "fxhash", "log", "mime", "pin-project 1.0.2", "regex", - "rustls", + "rustls 0.19.0", "serde", "serde_json", "serde_urlencoded", + "smallvec", "socket2", "time 0.2.23", - "tinyvec", "url", ] [[package]] name = "actix-web-codegen" -version = "0.4.0" +version = "0.5.0-beta.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +checksum = "8313dc4cbcae1785a7f14c3dfb7dfeb25fe96a03b20e5c38fe026786def5aa70" dependencies = [ "proc-macro2", "quote", @@ -305,6 +390,17 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +[[package]] +name = "ahash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.15" @@ -348,7 +444,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.24", ] [[package]] @@ -381,24 +477,24 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "awc" -version = "2.0.3" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" +checksum = "da7225ad81fbad09ef56ccc61e0688abe8494a68722c5d0df5e2fc8b724a200b" dependencies = [ - "actix-codec", - "actix-http", - "actix-rt", - "actix-service", + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.3", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", "base64 0.13.0", - "bytes 0.5.6", + "bytes 1.0.1", "cfg-if 1.0.0", "derive_more", "futures-core", "log", "mime", "percent-encoding", - "rand 0.7.3", - "rustls", + "rand 0.8.3", + "rustls 0.19.0", "serde", "serde_json", "serde_urlencoded", @@ -563,12 +659,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" [[package]] -name = "bytestring" -version = "0.1.5" +name = "bytes" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", ] [[package]] @@ -1171,8 +1273,28 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", - "tokio-util", + "tokio 0.2.24", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + +[[package]] +name = "h2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.2.0", + "tokio-util 0.6.3", "tracing", "tracing-futures", ] @@ -1183,7 +1305,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" dependencies = [ - "ahash", + "ahash 0.3.8", "autocfg", ] @@ -1262,11 +1384,11 @@ dependencies = [ [[package]] name = "http" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "itoa", ] @@ -1324,7 +1446,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.2.7", "http", "http-body", "httparse", @@ -1332,7 +1454,7 @@ dependencies = [ "itoa", "pin-project 1.0.2", "socket2", - "tokio", + "tokio 0.2.24", "tower-service", "tracing", "want", @@ -1348,9 +1470,9 @@ dependencies = [ "futures-util", "hyper", "log", - "rustls", - "tokio", - "tokio-rustls", + "rustls 0.18.1", + "tokio 0.2.24", + "tokio-rustls 0.14.1", "webpki", ] @@ -1537,9 +1659,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.81" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "linked-hash-map" @@ -1610,24 +1732,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "meilisearch-error" -version = "0.19.0" -dependencies = [ - "actix-http", -] - -[[package]] -name = "meilisearch-http" +name = "meilisearch" version = "0.17.0" dependencies = [ "actix-cors", - "actix-http", - "actix-rt", - "actix-service", + "actix-service 1.0.6", "actix-web", "anyhow", "assert-json-diff", "async-compression", + "async-trait", "byte-unit", "bytes 0.6.0", "chrono", @@ -1656,7 +1770,7 @@ dependencies = [ "rand 0.7.3", "rayon", "regex", - "rustls", + "rustls 0.19.0", "sentry", "serde", "serde_json", @@ -1668,11 +1782,20 @@ dependencies = [ "tar", "tempdir", "tempfile", - "tokio", + "thiserror", + "tokio 0.2.24", + "tokio 1.2.0", "uuid", "vergen", ] +[[package]] +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + [[package]] name = "meilisearch-tokenizer" version = "0.1.1" @@ -1799,14 +1922,15 @@ dependencies = [ ] [[package]] -name = "mio-named-pipes" -version = "0.1.7" +name = "mio" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" dependencies = [ + "libc", "log", - "mio", "miow 0.3.6", + "ntapi", "winapi 0.3.9", ] @@ -1818,7 +1942,7 @@ checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" dependencies = [ "iovec", "libc", - "mio", + "mio 0.6.23", ] [[package]] @@ -1866,6 +1990,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -2215,12 +2348,24 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.15", "libc", - "rand_chacha", + "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc", + "rand_hc 0.2.0", "rand_pcg", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2231,6 +2376,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -2255,6 +2410,15 @@ dependencies = [ "getrandom 0.1.15", ] +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -2264,6 +2428,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_pcg" version = "0.2.1" @@ -2381,17 +2554,17 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite 0.2.0", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", - "tokio", - "tokio-rustls", + "tokio 0.2.24", + "tokio-rustls 0.14.1", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.20.0", "winreg 0.7.0", ] @@ -2457,6 +2630,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -2680,9 +2866,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" @@ -2883,18 +3069,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2", "quote", @@ -2996,14 +3182,33 @@ dependencies = [ "lazy_static", "libc", "memchr", - "mio", - "mio-named-pipes", + "mio 0.6.23", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", "signal-hook-registry", "slab", - "tokio-macros", + "tokio-macros 0.2.6", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes 1.0.1", + "libc", + "memchr", + "mio 0.7.9", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.0", + "signal-hook-registry", + "tokio-macros 1.1.0", "winapi 0.3.9", ] @@ -3018,6 +3223,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.14.1" @@ -3025,8 +3241,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls", - "tokio", + "rustls 0.18.1", + "tokio 0.2.24", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.0", + "tokio 1.2.0", "webpki", ] @@ -3041,7 +3268,21 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.24", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", ] [[package]] @@ -3097,7 +3338,7 @@ dependencies = [ "rand 0.7.3", "smallvec", "thiserror", - "tokio", + "tokio 0.2.24", "url", ] @@ -3117,7 +3358,7 @@ dependencies = [ "resolv-conf", "smallvec", "thiserror", - "tokio", + "tokio 0.2.24", "trust-dns-proto", ] @@ -3369,6 +3610,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + [[package]] name = "whatlang" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 993c80946..126f859e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Quentin de Quelen ", "Clément Renault , + pub index_controller: Arc, pub api_keys: ApiKeys, options: Opt, } @@ -59,14 +60,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let indexer_opts = options.indexer_options.clone(); + //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = LocalIndexController::new( - &path, - indexer_opts, - options.max_mdb_size.get_bytes(), - options.max_udb_size.get_bytes(), - )?; + let index_controller = ActorIndexController::new(); let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { @@ -85,7 +81,7 @@ impl Data { pub fn settings>(&self, index_uid: S) -> anyhow::Result { let index = self.index_controller - .index(&index_uid)? + .index(index_uid.as_ref().to_string())? .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; let txn = index.read_txn()?; @@ -119,19 +115,20 @@ impl Data { } pub fn index(&self, name: impl AsRef) -> anyhow::Result> { - Ok(self - .list_indexes()? - .into_iter() - .find(|i| i.uid == name.as_ref())) + todo!() + //Ok(self + //.list_indexes()? + //.into_iter() + //.find(|i| i.uid == name.as_ref())) } - pub fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { name: Some(name.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; - let meta = self.index_controller.create_index(settings)?; + let meta = self.index_controller.create_index(settings).await?; Ok(meta) } diff --git a/src/data/search.rs b/src/data/search.rs index f26730fcf..23c6b463e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -2,7 +2,7 @@ use std::collections::{HashSet, BTreeMap}; use std::mem; use std::time::Instant; -use anyhow::{bail, Context}; +use anyhow::bail; use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::Data; -use crate::index_controller::IndexController; pub const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -202,107 +201,110 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { impl Data { pub fn search>( &self, - index: S, - search_query: SearchQuery, + _index: S, + _search_query: SearchQuery, ) -> anyhow::Result { - match self.index_controller.index(&index)? { - Some(index) => Ok(search_query.perform(index)?), - None => bail!("index {:?} doesn't exists", index.as_ref()), - } + todo!() + //match self.index_controller.index(&index)? { + //Some(index) => Ok(search_query.perform(index)?), + //None => bail!("index {:?} doesn't exists", index.as_ref()), + //} } pub async fn retrieve_documents( &self, - index: impl AsRef + Send + Sync + 'static, - offset: usize, - limit: usize, - attributes_to_retrieve: Option>, + _index: String, + _offset: usize, + _limit: usize, + _attributes_to_retrieve: Option>, ) -> anyhow::Result>> where S: AsRef + Send + Sync + 'static, { - let index_controller = self.index_controller.clone(); - let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - let index = index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + todo!() + //let index_controller = self.index_controller.clone(); + //let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + //let index = index_controller + //.index(index.clone())? + //.with_context(|| format!("Index {:?} doesn't exist", index))?; - let txn = index.read_txn()?; + //let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + //let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + //let attributes_to_retrieve_ids = match attributes_to_retrieve { + //Some(attrs) => attrs + //.iter() + //.filter_map(|f| fields_ids_map.id(f.as_ref())) + //.collect::>(), + //None => fields_ids_map.iter().map(|(id, _)| id).collect(), + //}; - let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); + //let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); - let mut documents = Vec::new(); + //let mut documents = Vec::new(); - for entry in iter { - let (_id, obkv) = entry?; - let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; - documents.push(object); - } + //for entry in iter { + //let (_id, obkv) = entry?; + //let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + //documents.push(object); + //} - Ok(documents) - }) - .await?; - documents + //Ok(documents) + //}) + //.await?; + //documents } pub async fn retrieve_document( &self, - index: impl AsRef + Sync + Send + 'static, - document_id: impl AsRef + Sync + Send + 'static, - attributes_to_retrieve: Option>, + _index: impl AsRef + Sync + Send + 'static, + _document_id: impl AsRef + Sync + Send + 'static, + _attributes_to_retrieve: Option>, ) -> anyhow::Result> where S: AsRef + Sync + Send + 'static, { - let index_controller = self.index_controller.clone(); - let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - let index = index_controller - .index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let txn = index.read_txn()?; + todo!() + //let index_controller = self.index_controller.clone(); + //let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { + //let index = index_controller + //.index(&index)? + //.with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + //let txn = index.read_txn()?; - let fields_ids_map = index.fields_ids_map(&txn)?; + //let fields_ids_map = index.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + //let attributes_to_retrieve_ids = match attributes_to_retrieve { + //Some(attrs) => attrs + //.iter() + //.filter_map(|f| fields_ids_map.id(f.as_ref())) + //.collect::>(), + //None => fields_ids_map.iter().map(|(id, _)| id).collect(), + //}; - let internal_id = index - .external_documents_ids(&txn)? - .get(document_id.as_ref().as_bytes()) - .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; + //let internal_id = index + //.external_documents_ids(&txn)? + //.get(document_id.as_ref().as_bytes()) + //.with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - let document = index - .documents(&txn, std::iter::once(internal_id))? - .into_iter() - .next() - .map(|(_, d)| d); + //let document = index + //.documents(&txn, std::iter::once(internal_id))? + //.into_iter() + //.next() + //.map(|(_, d)| d); - match document { - Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, - &fields_ids_map, - document, - )?), - None => bail!("Document with id {} not found", document_id.as_ref()), - } - }) - .await?; - document + //match document { + //Some(document) => Ok(obkv_to_json( + //&attributes_to_retrieve_ids, + //&fields_ids_map, + //document, + //)?), + //None => bail!("Document with id {} not found", document_id.as_ref()), + //} + //}) + //.await?; + //document } } diff --git a/src/data/updates.rs b/src/data/updates.rs index fbb9be801..5cdaf7db1 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,12 +1,13 @@ use std::ops::Deref; -use async_compression::tokio_02::write::GzipEncoder; -use futures_util::stream::StreamExt; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use tokio::io::AsyncWriteExt; +//use async_compression::tokio_02::write::GzipEncoder; +//use futures_util::stream::StreamExt; +//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use tokio::io::AsyncWriteExt; +use actix_web::web::Payload; use crate::index_controller::UpdateStatus; -use crate::index_controller::{IndexController, Settings, IndexSettings, IndexMetadata}; +use crate::index_controller::{Settings, IndexMetadata}; use super::Data; impl Data { @@ -15,88 +16,68 @@ impl Data { index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - mut stream: impl futures::Stream> + Unpin, + stream: Payload, primary_key: Option, ) -> anyhow::Result where B: Deref, E: std::error::Error + Send + Sync + 'static, { - let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; - let file = tokio::fs::File::from_std(file?); - let mut encoder = GzipEncoder::new(file); - - let mut empty_update = true; - while let Some(result) = stream.next().await { - empty_update = false; - let bytes = &*result?; - encoder.write_all(&bytes[..]).await?; - } - - encoder.shutdown().await?; - let mut file = encoder.into_inner(); - file.sync_all().await?; - let file = file.into_std().await; - - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move ||{ - let mmap; - let bytes = if empty_update { - &[][..] - } else { - mmap = unsafe { memmap::Mmap::map(&file)? }; - &mmap - }; - index_controller.add_documents(index, method, format, &bytes, primary_key) - }).await??; - Ok(update.into()) + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + Ok(update_status) } pub async fn update_settings( &self, - index: impl AsRef + Send + Sync + 'static, - settings: Settings + _index: impl AsRef + Send + Sync + 'static, + _settings: Settings ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; + //Ok(update.into()) } pub async fn clear_documents( &self, - index: impl AsRef + Sync + Send + 'static, + _index: impl AsRef + Sync + Send + 'static, ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; + //Ok(update.into()) } pub async fn delete_documents( &self, - index: impl AsRef + Sync + Send + 'static, - document_ids: Vec, + _index: impl AsRef + Sync + Send + 'static, + _document_ids: Vec, ) -> anyhow::Result { - let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; - Ok(update.into()) + todo!() + //let index_controller = self.index_controller.clone(); + //let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; + //Ok(update.into()) } pub async fn delete_index( &self, - index: impl AsRef + Send + Sync + 'static, + _index: impl AsRef + Send + Sync + 'static, ) -> anyhow::Result<()> { - let index_controller = self.index_controller.clone(); - tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; - Ok(()) + todo!() + //let index_controller = self.index_controller.clone(); + //tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; + //Ok(()) } #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - self.index_controller.update_status(index, uid) + todo!() + //self.index_controller.update_status(index, uid) } pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - self.index_controller.all_update_status(index) + todo!() + //self.index_controller.all_update_status(index) } pub fn update_index( @@ -105,11 +86,12 @@ impl Data { primary_key: Option>, new_name: Option> ) -> anyhow::Result { - let settings = IndexSettings { - name: new_name.map(|s| s.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), - }; + todo!() + //let settings = IndexSettings { + //name: new_name.map(|s| s.as_ref().to_string()), + //primary_key: primary_key.map(|s| s.as_ref().to_string()), + //}; - self.index_controller.update_index(name, settings) + //self.index_controller.update_index(name, settings) } } diff --git a/src/error.rs b/src/error.rs index 33aa06d3c..70e9c4ed8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,7 +1,8 @@ use std::error; use std::fmt; -use actix_http::ResponseBuilder; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::Error as HttpError; use actix_web as aweb; use actix_web::error::{JsonPayloadError, QueryPayloadError}; use actix_web::http::StatusCode; @@ -66,7 +67,7 @@ impl Serialize for ResponseError { impl aweb::error::ResponseError for ResponseError { fn error_response(&self) -> aweb::HttpResponse { - ResponseBuilder::new(self.status_code()).json(&self) + HttpResponseBuilder::new(self.status_code()).json(&self) } fn status_code(&self) -> StatusCode { @@ -260,8 +261,8 @@ impl From for Error { } } -impl From for Error { - fn from(err: actix_http::Error) -> Error { +impl From for Error { + fn from(err: HttpError) -> Error { Error::Internal(err.to_string()) } } diff --git a/src/helpers/authentication.rs b/src/helpers/authentication.rs index 0de709ba3..4fdbdeef5 100644 --- a/src/helpers/authentication.rs +++ b/src/helpers/authentication.rs @@ -3,27 +3,26 @@ use std::pin::Pin; use std::rc::Rc; use std::task::{Context, Poll}; -use actix_service::{Service, Transform}; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web}; +use actix_web::dev::{Transform, Service, ServiceResponse, ServiceRequest}; +use actix_web::web; use futures::future::{err, ok, Future, Ready}; use crate::error::{Error, ResponseError}; use crate::Data; -#[derive(Clone)] +#[derive(Clone, Copy)] pub enum Authentication { Public, Private, Admin, } -impl Transform for Authentication +impl Transform for Authentication where - S: Service, Error = actix_web::Error>, + S: Service, Error = actix_web::Error>, S::Future: 'static, B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = actix_web::Error; type InitError = (); @@ -32,7 +31,7 @@ where fn new_transform(&self, service: S) -> Self::Future { ok(LoggingMiddleware { - acl: self.clone(), + acl: *self, service: Rc::new(RefCell::new(service)), }) } @@ -44,23 +43,22 @@ pub struct LoggingMiddleware { } #[allow(clippy::type_complexity)] -impl Service for LoggingMiddleware +impl Service for LoggingMiddleware where - S: Service, Error = actix_web::Error> + 'static, + S: Service, Error = actix_web::Error> + 'static, S::Future: 'static, B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = actix_web::Error; type Future = Pin>>>; - fn poll_ready(&mut self, cx: &mut Context) -> Poll> { + fn poll_ready(&self, cx: &mut Context) -> Poll> { self.service.poll_ready(cx) } - fn call(&mut self, req: ServiceRequest) -> Self::Future { - let mut svc = self.service.clone(); + fn call(&self, req: ServiceRequest) -> Self::Future { + let svc = self.service.clone(); // This unwrap is left because this error should never appear. If that's the case, then // it means that actix-web has an issue or someone changes the type `Data`. let data = req.app_data::>().unwrap(); diff --git a/src/helpers/normalize_path.rs b/src/helpers/normalize_path.rs index e669b9d94..51c64c52b 100644 --- a/src/helpers/normalize_path.rs +++ b/src/helpers/normalize_path.rs @@ -1,5 +1,5 @@ /// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 -use actix_http::Error; +use actix_web::http::Error; use actix_service::{Service, Transform}; use actix_web::{ dev::ServiceRequest, diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index b20e43749..ed085456b 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,8 +1,7 @@ -mod local_index_controller; +pub mod actor_index_controller; +//mod local_index_controller; mod updates; -pub use local_index_controller::LocalIndexController; - use std::collections::HashMap; use std::num::NonZeroUsize; use std::sync::Arc; @@ -13,6 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; +use actix_web::web::Payload; pub use updates::{Processed, Processing, Failed}; @@ -21,7 +21,6 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - pub uid: String, uuid: Uuid, created_at: DateTime, updated_at: DateTime, @@ -114,6 +113,7 @@ pub struct IndexSettings { /// be provided. This allows the implementer to define the behaviour of write accesses to the /// indices, and abstract the scheduling of the updates. The implementer must be able to provide an /// instance of `IndexStore` +#[async_trait::async_trait] pub trait IndexController { /* @@ -126,59 +126,47 @@ pub trait IndexController { /// Perform document addition on the database. If the provided index does not exist, it will be /// created when the addition is applied to the index. - fn add_documents>( + async fn add_documents( &self, - index: S, + index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: &[u8], + data: Payload, primary_key: Option, ) -> anyhow::Result; /// Clear all documents in the given index. - fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; + fn clear_documents(&self, index: String) -> anyhow::Result; /// Delete all documents in `document_ids`. - fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. - fn create_index(&self, index_settings: IndexSettings) -> Result; + async fn create_index(&self, index_settings: IndexSettings) -> Result; /// Delete index with the given `index_uid`, attempting to close it beforehand. - fn delete_index>(&self, index_uid: S) -> Result<()>; + fn delete_index(&self, index_uid: String) -> Result<()>; /// Swap two indexes, concretely, it simply swaps the index the names point to. - fn swap_indices, S2: AsRef>(&self, index1_uid: S1, index2_uid: S2) -> Result<()>; - - /// Apply an update to the given index. This method can be called when an update is ready to be - /// processed - fn handle_update>( - &self, - _index: S, - _update_id: u64, - _meta: Processing, - _content: &[u8] - ) -> Result, Failed> { - todo!() - } + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> Result<()>; /// Returns, if it exists, the `Index` with the povided name. - fn index(&self, name: impl AsRef) -> anyhow::Result>>; + fn index(&self, name: String) -> anyhow::Result>>; /// Returns the udpate status an update - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; + fn update_status(&self, index: String, id: u64) -> anyhow::Result>; /// Returns all the udpate status for an index - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; + fn all_update_status(&self, index: String) -> anyhow::Result>; /// List all the indexes fn list_indexes(&self) -> anyhow::Result>; - fn update_index(&self, name: impl AsRef, index_settings: IndexSettings) -> anyhow::Result; + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result; } diff --git a/src/lib.rs b/src/lib.rs index d542fd6d7..65c4e8eb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,7 @@ #![allow(clippy::or_fun_call)] +#![allow(unused_must_use)] +#![allow(unused_variables)] +#![allow(dead_code)] pub mod data; pub mod error; @@ -7,54 +10,5 @@ pub mod option; pub mod routes; mod index_controller; -use actix_http::Error; -use actix_service::ServiceFactory; -use actix_web::{dev, web, App}; - pub use option::Opt; pub use self::data::Data; -use self::error::payload_error_handler; - -pub fn create_app( - data: &Data, - enable_frontend: bool, -) -> App< - impl ServiceFactory< - Config = (), - Request = dev::ServiceRequest, - Response = dev::ServiceResponse, - Error = Error, - InitError = (), - >, - actix_http::body::Body, -> { - let app = App::new() - .data(data.clone()) - .app_data( - web::JsonConfig::default() - .limit(data.http_payload_size_limit()) - .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ) - .configure(routes::document::services) - .configure(routes::index::services) - .configure(routes::search::services) - .configure(routes::settings::services) - .configure(routes::stop_words::services) - .configure(routes::synonym::services) - .configure(routes::health::services) - .configure(routes::stats::services) - .configure(routes::key::services); - //.configure(routes::dump::services); - if enable_frontend { - app - .service(routes::load_html) - .service(routes::load_css) - } else { - app - } -} diff --git a/src/main.rs b/src/main.rs index 7e50225be..c9042b88e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,13 @@ use std::env; use actix_cors::Cors; -use actix_web::{middleware, HttpServer}; +use actix_web::{middleware, HttpServer, web, web::ServiceConfig}; use main_error::MainError; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::{create_app, Data, Opt}; +use meilisearch::{Data, Opt}; use structopt::StructOpt; +use actix_web::App; +use meilisearch::error::payload_error_handler; +use actix_web::middleware::TrailingSlash; //mod analytics; @@ -74,9 +76,34 @@ async fn main() -> Result<(), MainError> { print_launch_resume(&opt, &data); let enable_frontend = opt.env != "production"; + + run_http(data, opt, enable_frontend).await?; + + Ok(()) +} + +async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { let http_server = HttpServer::new(move || { - create_app(&data, enable_frontend) - .wrap( + let app = App::new() + .configure(|c| configure_data(c, &data)) + .configure(meilisearch::routes::document::services) + .configure(meilisearch::routes::index::services) + .configure(meilisearch::routes::search::services) + .configure(meilisearch::routes::settings::services) + .configure(meilisearch::routes::stop_words::services) + .configure(meilisearch::routes::synonym::services) + .configure(meilisearch::routes::health::services) + .configure(meilisearch::routes::stats::services) + .configure(meilisearch::routes::key::services); + //.configure(routes::dump::services); + let app = if enable_frontend { + app + .service(meilisearch::routes::load_html) + .service(meilisearch::routes::load_css) + } else { + app + }; + app.wrap( Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) @@ -84,7 +111,7 @@ async fn main() -> Result<(), MainError> { ) .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) - .wrap(NormalizePath) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) }); if let Some(config) = opt.get_ssl_config()? { @@ -95,10 +122,24 @@ async fn main() -> Result<(), MainError> { } else { http_server.bind(opt.http_addr)?.run().await?; } - Ok(()) } +fn configure_data(config: &mut ServiceConfig, data: &Data) { + config + .data(data.clone()) + .app_data( + web::JsonConfig::default() + .limit(data.http_payload_size_limit()) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()) + ); +} + pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 diff --git a/src/routes/document.rs b/src/routes/document.rs index 43d88e4b9..2cb87074e 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -3,7 +3,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; use log::error; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; @@ -142,25 +142,26 @@ async fn add_documents_json( params: web::Query, body: Payload, ) -> Result { - let addition_result = data - .add_documents( - path.into_inner().index_uid, - IndexDocumentsMethod::ReplaceDocuments, - UpdateFormat::Json, - body, - params.primary_key.clone(), - ).await; + todo!() + //let addition_result = data + //.add_documents( + //path.into_inner().index_uid, + //IndexDocumentsMethod::ReplaceDocuments, + //UpdateFormat::Json, + //body, + //params.primary_key.clone(), + //).await; - match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match addition_result { + //Ok(update) => { + //let value = serde_json::to_string(&update).unwrap(); + //let response = HttpResponse::Ok().body(value); + //Ok(response) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } @@ -199,25 +200,26 @@ async fn update_documents( params: web::Query, body: web::Payload, ) -> Result { - let addition_result = data - .add_documents( - path.into_inner().index_uid, - IndexDocumentsMethod::UpdateDocuments, - UpdateFormat::Json, - body, - params.primary_key.clone(), - ).await; + todo!() + //let addition_result = data + //.add_documents( + //path.into_inner().index_uid, + //IndexDocumentsMethod::UpdateDocuments, + //UpdateFormat::Json, + //body, + //params.primary_key.clone(), + //).await; - match addition_result { - Ok(update) => { - let value = serde_json::to_string(&update).unwrap(); - let response = HttpResponse::Ok().body(value); - Ok(response) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match addition_result { + //Ok(update) => { + //let value = serde_json::to_string(&update).unwrap(); + //let response = HttpResponse::Ok().body(value); + //Ok(response) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[post( @@ -229,20 +231,21 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { - let ids = body - .iter() - .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) - .collect(); + todo!() + //let ids = body + //.iter() + //.map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + //.collect(); - match data.delete_documents(path.index_uid.clone(), ids).await { - Ok(result) => { - let json = serde_json::to_string(&result).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + //match data.delete_documents(path.index_uid.clone(), ids).await { + //Ok(result) => { + //let json = serde_json::to_string(&result).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] @@ -250,13 +253,14 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - match data.clear_documents(path.index_uid.clone()).await { - Ok(update) => { - let json = serde_json::to_string(&update).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + todo!() + //match data.clear_documents(path.index_uid.clone()).await { + //Ok(update) => { + //let json = serde_json::to_string(&update).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } diff --git a/src/routes/index.rs b/src/routes/index.rs index 2c478432b..813256517 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -61,7 +61,7 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - match data.create_index(&body.uid, body.primary_key.clone()) { + match data.create_index(&body.uid, body.primary_key.clone()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/src/routes/key.rs b/src/routes/key.rs index a0cbaccc3..a051ee0f9 100644 --- a/src/routes/key.rs +++ b/src/routes/key.rs @@ -19,7 +19,7 @@ struct KeysResponse { #[get("/keys", wrap = "Authentication::Admin")] async fn list(data: web::Data) -> HttpResponse { let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(KeysResponse { + HttpResponse::Ok().json(&KeysResponse { private: api_keys.private, public: api_keys.public, }) From 672a4b54005cca57a737117ac2e7631f910f5a4b Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 09:10:36 +0100 Subject: [PATCH 02/69] add actors/ support index creation --- .../actor_index_controller/index_actor.rs | 132 ++++++++++++++++++ .../actor_index_controller/mod.rs | 93 ++++++++++++ .../actor_index_controller/update_actor.rs | 16 +++ .../actor_index_controller/uuid_resolver.rs | 116 +++++++++++++++ 4 files changed, 357 insertions(+) create mode 100644 src/index_controller/actor_index_controller/index_actor.rs create mode 100644 src/index_controller/actor_index_controller/mod.rs create mode 100644 src/index_controller/actor_index_controller/update_actor.rs create mode 100644 src/index_controller/actor_index_controller/uuid_resolver.rs diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs new file mode 100644 index 000000000..6de0f2b77 --- /dev/null +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -0,0 +1,132 @@ +use uuid::Uuid; +use std::path::{PathBuf, Path}; +use chrono::Utc; +use tokio::sync::{mpsc, oneshot, RwLock}; +use thiserror::Error; +use std::collections::HashMap; +use std::sync::Arc; +use milli::Index; +use std::collections::hash_map::Entry; +use std::fs::create_dir_all; +use heed::EnvOpenOptions; +use crate::index_controller::IndexMetadata; + +pub type Result = std::result::Result; +type AsyncMap = Arc>>; + +enum IndexMsg { + CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, +} + +struct IndexActor { + inbox: mpsc::Receiver, + store: S, +} + +#[derive(Error, Debug)] +pub enum IndexError { + #[error("error with index: {0}")] + Error(#[from] anyhow::Error), + #[error("index already exists")] + IndexAlreadyExists, +} + +#[async_trait::async_trait] +trait IndexStore { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; +} + +impl IndexActor { + fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + async fn run(mut self) { + loop { + match self.inbox.recv().await { + Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, + None => break, + } + } + } + + async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { + let result = self.store.create_index(uuid, primary_key).await; + let _ = ret.send(result); + } +} + +#[derive(Clone)] +pub struct IndexActorHandle { + sender: mpsc::Sender, +} + +impl IndexActorHandle { + pub fn new() -> Self { + let (sender, receiver) = mpsc::channel(100); + + let store = MapIndexStore::new("data.ms"); + let actor = IndexActor::new(receiver, store); + tokio::task::spawn(actor.run()); + Self { sender } + } + + pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::CreateIndex { ret, uuid, primary_key }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } +} + +struct MapIndexStore { + root: PathBuf, + meta_store: AsyncMap, + index_store: AsyncMap, +} + +#[async_trait::async_trait] +impl IndexStore for MapIndexStore { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let meta = match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(entry) => { + let meta = IndexMetadata { + uuid, + created_at: Utc::now(), + updated_at: Utc::now(), + primary_key, + }; + entry.insert(meta).clone() + } + Entry::Occupied(_) => return Err(IndexError::IndexAlreadyExists), + }; + + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + + println!("before blocking"); + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; + Ok(index) + }).await.expect("thread died"); + println!("after blocking"); + + self.index_store.write().await.insert(meta.uuid.clone(), index?); + + Ok(meta) + } +} + +impl MapIndexStore { + fn new(root: impl AsRef) -> Self { + let mut root = root.as_ref().to_owned(); + root.push("indexes/"); + let meta_store = Arc::new(RwLock::new(HashMap::new())); + let index_store = Arc::new(RwLock::new(HashMap::new())); + Self { meta_store, index_store, root } + } +} diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs new file mode 100644 index 000000000..bfcac7a3f --- /dev/null +++ b/src/index_controller/actor_index_controller/mod.rs @@ -0,0 +1,93 @@ +mod index_actor; +mod update_actor; +mod uuid_resolver; + +use tokio::fs::File; +use tokio::sync::oneshot; +use super::IndexController; +use uuid::Uuid; +use super::IndexMetadata; + + +pub struct ActorIndexController { + uuid_resolver: uuid_resolver::UuidResolverHandle, + index_actor: index_actor::IndexActorHandle, +} + +impl ActorIndexController { + pub fn new() -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(); + Self { uuid_resolver, index_actor } + } +} + +enum IndexControllerMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Shutdown, +} + +#[async_trait::async_trait] +impl IndexController for ActorIndexController { + async fn add_documents( + &self, + index: String, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + data: File, + primary_key: Option, + ) -> anyhow::Result { + todo!() + } + + fn clear_documents(&self, index: String) -> anyhow::Result { + todo!() + } + + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + todo!() + } + + fn update_settings(&self, index_uid: String, settings: super::Settings) -> anyhow::Result { + todo!() + } + + async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { + let super::IndexSettings { name, primary_key } = index_settings; + let uuid = self.uuid_resolver.create(name.unwrap()).await?; + let index_meta = self.index_actor.create_index(uuid, primary_key).await?; + Ok(index_meta) + } + + fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn index(&self, name: String) -> anyhow::Result>> { + todo!() + } + + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + todo!() + } + + fn all_update_status(&self, index: String) -> anyhow::Result> { + todo!() + } + + fn list_indexes(&self) -> anyhow::Result> { + todo!() + } + + fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { + todo!() + } +} diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs new file mode 100644 index 000000000..9fd3cc39f --- /dev/null +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -0,0 +1,16 @@ +use super::index_actor::IndexActorHandle; +use uuid::Uuid; +use tokio::sync::{mpsc, oneshot}; + +enum UpdateMsg { + CreateIndex{ + uuid: Uuid, + ret: oneshot::Sender>, + } +} + +struct UpdateActor { + update_store: S, + inbox: mpsc::Receiver, + index_actor: IndexActorHandle, +} diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs new file mode 100644 index 000000000..d5756d05e --- /dev/null +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -0,0 +1,116 @@ +use thiserror::Error; +use tokio::sync::{RwLock, mpsc, oneshot}; +use uuid::Uuid; +use std::collections::HashMap; +use std::sync::Arc; +use std::collections::hash_map::Entry; +use log::info; + +pub type Result = std::result::Result; + +#[derive(Debug)] +enum UuidResolveMsg { + Resolve { + name: String, + ret: oneshot::Sender>>, + }, + Create { + name: String, + ret: oneshot::Sender>, + }, + Shutdown, + +} + +struct UuidResolverActor { + inbox: mpsc::Receiver, + store: S, +} + +impl UuidResolverActor { + fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + async fn run(mut self) { + use UuidResolveMsg::*; + + info!("uuid resolver started"); + + // TODO: benchmark and use buffered streams to improve throughput. + loop { + match self.inbox.recv().await { + Some(Create { name, ret }) => self.handle_create(name, ret).await, + Some(_) => (), + // all senders have ned dropped, need to quit. + None => break, + } + } + } + + async fn handle_create(&self, name: String, ret: oneshot::Sender>) { + let result = self.store.create_uuid(name).await; + let _ = ret.send(result); + } +} + +#[derive(Clone)] +pub struct UuidResolverHandle { + sender: mpsc::Sender, +} + +impl UuidResolverHandle { + pub fn new() -> Self { + let (sender, reveiver) = mpsc::channel(100); + let store = MapUuidStore(Arc::new(RwLock::new(HashMap::new()))); + let actor = UuidResolverActor::new(reveiver, store); + tokio::spawn(actor.run()); + Self { sender } + } + + pub async fn resolve(&self, name: String) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Resolve { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } + + pub async fn create(&self, name: String) -> anyhow::Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Create { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } +} + +#[derive(Clone, Debug, Error)] +pub enum UuidError { + #[error("Name already exist.")] + NameAlreadyExist, +} + +#[async_trait::async_trait] +trait UuidStore { + async fn create_uuid(&self, name: String) -> Result; + async fn get_uuid(&self, name: String) -> Result>; +} + +struct MapUuidStore(Arc>>); + +#[async_trait::async_trait] +impl UuidStore for MapUuidStore { + async fn create_uuid(&self, name: String) -> Result { + match self.0.write().await.entry(name) { + Entry::Occupied(_) => Err(UuidError::NameAlreadyExist), + Entry::Vacant(entry) => { + let uuid = Uuid::new_v4(); + let uuid = entry.insert(uuid); + Ok(uuid.clone()) + } + } + } + + async fn get_uuid(&self, name: String) -> Result> { + Ok(self.0.read().await.get(&name).cloned()) + } +} From 6bcc30295028076efe09fc7f78e5daccc09ada1d Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 17:14:11 +0100 Subject: [PATCH 03/69] receive update --- src/data/updates.rs | 22 +- .../actor_index_controller/index_actor.rs | 19 +- .../actor_index_controller/mod.rs | 19 +- .../actor_index_controller/update_actor.rs | 91 +++- .../actor_index_controller/uuid_resolver.rs | 37 +- .../local_index_controller/update_store.rs | 407 ------------------ src/index_controller/mod.rs | 6 +- src/routes/document.rs | 39 +- 8 files changed, 181 insertions(+), 459 deletions(-) delete mode 100644 src/index_controller/local_index_controller/update_store.rs diff --git a/src/data/updates.rs b/src/data/updates.rs index 5cdaf7db1..a559d812a 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,29 +1,33 @@ -use std::ops::Deref; - //use async_compression::tokio_02::write::GzipEncoder; //use futures_util::stream::StreamExt; -//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; +use tokio::fs::File; +use tokio::io::{AsyncWriteExt, AsyncSeekExt}; +use futures::prelude::stream::StreamExt; use crate::index_controller::UpdateStatus; use crate::index_controller::{Settings, IndexMetadata}; use super::Data; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - stream: Payload, + mut stream: Payload, primary_key: Option, ) -> anyhow::Result - where - B: Deref, - E: std::error::Error + Send + Sync + 'static, { - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + let file = tempfile::tempfile_in(".")?; + let mut file = File::from_std(file); + while let Some(Ok(bytes)) = stream.next().await { + file.write(bytes.as_ref()).await; + } + file.seek(std::io::SeekFrom::Start(0)).await?; + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 6de0f2b77..40f8de279 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -9,13 +9,15 @@ use milli::Index; use std::collections::hash_map::Entry; use std::fs::create_dir_all; use heed::EnvOpenOptions; -use crate::index_controller::IndexMetadata; +use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; pub type Result = std::result::Result; type AsyncMap = Arc>>; +type UpdateResult = std::result::Result, Failed>; enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, + Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, } struct IndexActor { @@ -45,6 +47,7 @@ impl IndexActor { loop { match self.inbox.recv().await { Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, + Some(IndexMsg::Update { ret, meta, data }) => self.handle_update().await, None => break, } } @@ -54,6 +57,10 @@ impl IndexActor { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); } + + async fn handle_update(&self) { + println!("processing update!!!"); + } } #[derive(Clone)] @@ -77,6 +84,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } + + pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Update { ret, meta, data }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } } struct MapIndexStore { @@ -103,8 +117,6 @@ impl IndexStore for MapIndexStore { let db_path = self.root.join(format!("index-{}", meta.uuid)); - - println!("before blocking"); let index: Result = tokio::task::spawn_blocking(move || { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); @@ -113,7 +125,6 @@ impl IndexStore for MapIndexStore { .map_err(|e| IndexError::Error(e))?; Ok(index) }).await.expect("thread died"); - println!("after blocking"); self.index_store.write().await.insert(meta.uuid.clone(), index?); diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index bfcac7a3f..621bbb1d6 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -1,24 +1,28 @@ mod index_actor; mod update_actor; mod uuid_resolver; +mod update_store; -use tokio::fs::File; use tokio::sync::oneshot; use super::IndexController; use uuid::Uuid; use super::IndexMetadata; +use tokio::fs::File; +use super::UpdateMeta; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, - index_actor: index_actor::IndexActorHandle, + index_handle: index_actor::IndexActorHandle, + update_handle: update_actor::UpdateActorHandle, } impl ActorIndexController { pub fn new() -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); let index_actor = index_actor::IndexActorHandle::new(); - Self { uuid_resolver, index_actor } + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone()); + Self { uuid_resolver, index_handle: index_actor, update_handle } } } @@ -31,7 +35,7 @@ enum IndexControllerMsg { Shutdown, } -#[async_trait::async_trait] +#[async_trait::async_trait(?Send)] impl IndexController for ActorIndexController { async fn add_documents( &self, @@ -41,7 +45,10 @@ impl IndexController for ActorIndexController { data: File, primary_key: Option, ) -> anyhow::Result { - todo!() + let uuid = self.uuid_resolver.get_or_create(index).await?; + let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let status = self.update_handle.update(meta, Some(data), uuid).await?; + Ok(status) } fn clear_documents(&self, index: String) -> anyhow::Result { @@ -59,7 +66,7 @@ impl IndexController for ActorIndexController { async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { let super::IndexSettings { name, primary_key } = index_settings; let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_actor.create_index(uuid, primary_key).await?; + let index_meta = self.index_handle.create_index(uuid, primary_key).await?; Ok(index_meta) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 9fd3cc39f..a0bf011c5 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -1,16 +1,99 @@ use super::index_actor::IndexActorHandle; use uuid::Uuid; use tokio::sync::{mpsc, oneshot}; +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; +use thiserror::Error; +use tokio::io::AsyncReadExt; +use log::info; +use tokio::fs::File; +use std::path::PathBuf; +use std::fs::create_dir_all; +use std::sync::Arc; + +pub type Result = std::result::Result; +type UpdateStore = super::update_store::UpdateStore; + +#[derive(Debug, Error)] +pub enum UpdateError {} enum UpdateMsg { CreateIndex{ uuid: Uuid, - ret: oneshot::Sender>, + ret: oneshot::Sender>, + }, + Update { + uuid: Uuid, + meta: UpdateMeta, + payload: Option, + ret: oneshot::Sender> } } -struct UpdateActor { - update_store: S, +struct UpdateActor { + store: Arc, inbox: mpsc::Receiver, - index_actor: IndexActorHandle, + index_handle: IndexActorHandle, +} + +impl UpdateActor { + fn new(store: Arc, inbox: mpsc::Receiver, index_handle: IndexActorHandle) -> Self { + Self { store, inbox, index_handle } + } + + async fn run(mut self) { + + info!("started update actor."); + + loop { + match self.inbox.recv().await { + Some(UpdateMsg::Update { uuid, meta, payload, ret }) => self.handle_update(uuid, meta, payload, ret).await, + Some(_) => {} + None => {} + } + } + } + + async fn handle_update(&self, _uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { + let mut buf = Vec::new(); + let mut payload = payload.unwrap(); + payload.read_to_end(&mut buf).await.unwrap(); + let result = self.store.register_update(meta, &buf).unwrap(); + let _ = ret.send(Ok(UpdateStatus::Pending(result))); + } +} + +#[derive(Clone)] +pub struct UpdateActorHandle { + sender: mpsc::Sender, +} + +impl UpdateActorHandle { + pub fn new(index_handle: IndexActorHandle) -> Self { + let (sender, receiver) = mpsc::channel(100); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let mut path = PathBuf::new(); + path.push("data.ms"); + path.push("updates"); + create_dir_all(&path).unwrap(); + let index_handle_clone = index_handle.clone(); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle_clone.update(meta, file)) + }).unwrap(); + let actor = UpdateActor::new(store, receiver, index_handle); + tokio::task::spawn_local(actor.run()); + Self { sender } + } + + pub async fn update(&self, meta: UpdateMeta, payload: Option, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Update { + uuid, + payload, + meta, + ret, + }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs index d5756d05e..b75c0402c 100644 --- a/src/index_controller/actor_index_controller/uuid_resolver.rs +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -14,6 +14,10 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>>, }, + GetOrCreate { + name: String, + ret: oneshot::Sender>, + }, Create { name: String, ret: oneshot::Sender>, @@ -41,15 +45,21 @@ impl UuidResolverActor { loop { match self.inbox.recv().await { Some(Create { name, ret }) => self.handle_create(name, ret).await, - Some(_) => (), - // all senders have ned dropped, need to quit. + Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, + Some(_) => {} + // all senders have been dropped, need to quit. None => break, } } } async fn handle_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name).await; + let result = self.store.create_uuid(name, true).await; + let _ = ret.send(result); + } + + async fn handle_get_or_create(&self, name: String, ret: oneshot::Sender>) { + let result = self.store.create_uuid(name, false).await; let _ = ret.send(result); } } @@ -75,6 +85,13 @@ impl UuidResolverHandle { Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + pub async fn get_or_create(&self, name: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetOrCreate { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } + pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Create { name, ret }; @@ -91,7 +108,9 @@ pub enum UuidError { #[async_trait::async_trait] trait UuidStore { - async fn create_uuid(&self, name: String) -> Result; + // Create a new entry for `name`. Return an error if `err` and the entry already exists, return + // the uuid otherwise. + async fn create_uuid(&self, name: String, err: bool) -> Result; async fn get_uuid(&self, name: String) -> Result>; } @@ -99,9 +118,15 @@ struct MapUuidStore(Arc>>); #[async_trait::async_trait] impl UuidStore for MapUuidStore { - async fn create_uuid(&self, name: String) -> Result { + async fn create_uuid(&self, name: String, err: bool) -> Result { match self.0.write().await.entry(name) { - Entry::Occupied(_) => Err(UuidError::NameAlreadyExist), + Entry::Occupied(entry) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + Ok(entry.get().clone()) + } + }, Entry::Vacant(entry) => { let uuid = Uuid::new_v4(); let uuid = entry.insert(uuid); diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs deleted file mode 100644 index b025ff090..000000000 --- a/src/index_controller/local_index_controller/update_store.rs +++ /dev/null @@ -1,407 +0,0 @@ -use std::path::Path; -use std::sync::{Arc, RwLock}; - -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; - -use crate::index_controller::updates::*; - -type BEU64 = heed::zerocopy::U64; - -#[derive(Clone)] -pub struct UpdateStore { - env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, - notification_sender: Sender<()>, -} - -pub trait HandleUpdate { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed>; -} - -impl UpdateStore -where - M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, - N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, - E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, -{ - pub fn open( - mut options: EnvOpenOptions, - path: P, - mut update_handler: U, - ) -> heed::Result> - where - P: AsRef, - U: HandleUpdate + Send + 'static, - { - options.max_dbs(5); - - let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); - - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); - // Send a first notification to trigger the process. - let _ = notification_sender.send(()); - - let update_store = Arc::new(UpdateStore { - env, - pending, - pending_meta, - processed_meta, - aborted_meta, - notification_sender, - failed_meta, - processing, - }); - - // We need a weak reference so we can take ownership on the arc later when we - // want to close the index. - let update_store_weak = Arc::downgrade(&update_store); - std::thread::spawn(move || { - // Block and wait for something to process. - 'outer: for _ in notification_receiver { - loop { - match update_store_weak.upgrade() { - Some(update_store) => { - match update_store.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), - } - } - // the ownership on the arc has been taken, we need to exit. - None => break 'outer, - } - } - } - }); - - Ok(update_store) - } - - pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { - self.env.prepare_for_closing() - } - - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_processed = self.processed_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_aborted = self.aborted_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } - } - - /// Registers the update content in the pending store and the meta - /// into the pending-meta store. Returns the new unique update id. - pub fn register_update( - &self, - meta: M, - content: &[u8] - ) -> heed::Result> { - let mut wtxn = self.env.write_txn()?; - - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); - - let meta = Pending::new(meta, update_id); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; - - wtxn.commit()?; - - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } - Ok(meta) - } - /// Executes the user provided function on the next pending update (the one with the lowest id). - /// This is asynchronous as it let the user process the update with a read-only txn and - /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> - where - U: HandleUpdate + Send + 'static, - { - // Create a read transaction to be able to retrieve the pending update in order. - let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; - - // If there is a pending update we process and only keep - // a reader while processing it, not a writer. - match first_meta { - Some((first_id, pending)) => { - let first_content = self.pending - .get(&rtxn, &first_id)? - .expect("associated update content"); - - // we change the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure - let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); - // Process the pending update using the provided user function. - let result = handler.handle_update(processing, first_content); - drop(rtxn); - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; - self.pending.delete(&mut wtxn, &first_id)?; - match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, - } - wtxn.commit()?; - - Ok(Some(())) - }, - None => Ok(None) - } - } - - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { - let rtxn = self.env.read_txn()?; - - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; - - // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) - } - - /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); - - if let Some(ref meta) = *self.processing.read().unwrap() { - if meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); - } - } - - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); - } - - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Failed(meta))); - } - - Ok(None) - } - - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - #[allow(dead_code)] - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { - let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); - - // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { - return Ok(None); - } - - let pending = match self.pending_meta.get(&wtxn, &key)? { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - #[allow(dead_code)] - pub fn abort_pendings(&self) -> heed::Result)>> { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::{Duration, Instant}; - - impl HandleUpdate for F - where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { - self(meta, content) - } - } - - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } - - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let before_register = Instant::now(); - - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - thread::sleep(Duration::from_millis(400 * 3 + 100)); - - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index ed085456b..dc6cc3863 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -12,7 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use actix_web::web::Payload; +use tokio::fs::File; pub use updates::{Processed, Processing, Failed}; @@ -113,7 +113,7 @@ pub struct IndexSettings { /// be provided. This allows the implementer to define the behaviour of write accesses to the /// indices, and abstract the scheduling of the updates. The implementer must be able to provide an /// instance of `IndexStore` -#[async_trait::async_trait] +#[async_trait::async_trait(?Send)] pub trait IndexController { /* @@ -131,7 +131,7 @@ pub trait IndexController { index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: Payload, + data: File, primary_key: Option, ) -> anyhow::Result; diff --git a/src/routes/document.rs b/src/routes/document.rs index 2cb87074e..00d037359 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -3,7 +3,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use indexmap::IndexMap; use log::error; -//use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; @@ -142,26 +142,25 @@ async fn add_documents_json( params: web::Query, body: Payload, ) -> Result { - todo!() - //let addition_result = data - //.add_documents( - //path.into_inner().index_uid, - //IndexDocumentsMethod::ReplaceDocuments, - //UpdateFormat::Json, - //body, - //params.primary_key.clone(), - //).await; + let addition_result = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::ReplaceDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ).await; - //match addition_result { - //Ok(update) => { - //let value = serde_json::to_string(&update).unwrap(); - //let response = HttpResponse::Ok().body(value); - //Ok(response) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } From 658166c05e451b9b53af50e469fa9b194fb3f80e Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Feb 2021 18:11:43 +0100 Subject: [PATCH 04/69] implement document push --- .../actor_index_controller/index_actor.rs | 62 +++-- .../actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_actor.rs | 4 +- .../local_index_controller/update_handler.rs | 255 ------------------ src/index_controller/updates.rs | 9 +- 5 files changed, 58 insertions(+), 274 deletions(-) delete mode 100644 src/index_controller/local_index_controller/update_handler.rs diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 40f8de279..6123ca774 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,15 +1,20 @@ -use uuid::Uuid; +use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; -use chrono::Utc; -use tokio::sync::{mpsc, oneshot, RwLock}; -use thiserror::Error; -use std::collections::HashMap; use std::sync::Arc; -use milli::Index; -use std::collections::hash_map::Entry; -use std::fs::create_dir_all; + +use chrono::Utc; use heed::EnvOpenOptions; +use milli::Index; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; +use log::info; + +use super::update_handler::UpdateHandler; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; +use crate::option::IndexerOpts; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -22,6 +27,7 @@ enum IndexMsg { struct IndexActor { inbox: mpsc::Receiver, + update_handler: Arc, store: S, } @@ -36,18 +42,22 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn get_or_create(&self, uuid: Uuid) -> Result>; } -impl IndexActor { +impl IndexActor { fn new(inbox: mpsc::Receiver, store: S) -> Self { - Self { inbox, store } + let options = IndexerOpts::default(); + let update_handler = UpdateHandler::new(&options).unwrap(); + let update_handler = Arc::new(update_handler); + Self { inbox, store, update_handler } } async fn run(mut self) { loop { match self.inbox.recv().await { Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, - Some(IndexMsg::Update { ret, meta, data }) => self.handle_update().await, + Some(IndexMsg::Update { ret, meta, data }) => self.handle_update(meta, data, ret).await, None => break, } } @@ -58,8 +68,14 @@ impl IndexActor { let _ = ret.send(result); } - async fn handle_update(&self) { - println!("processing update!!!"); + async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { + info!("processing update"); + let uuid = meta.index_uuid().clone(); + let index = self.store.get_or_create(uuid).await.unwrap(); + let update_handler = self.update_handler.clone(); + let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index.as_ref())).await; + let result = result.unwrap(); + let _ = ret.send(result); } } @@ -96,7 +112,7 @@ impl IndexActorHandle { struct MapIndexStore { root: PathBuf, meta_store: AsyncMap, - index_store: AsyncMap, + index_store: AsyncMap>, } #[async_trait::async_trait] @@ -126,10 +142,26 @@ impl IndexStore for MapIndexStore { Ok(index) }).await.expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), index?); + self.index_store.write().await.insert(meta.uuid.clone(), Arc::new(index?)); Ok(meta) } + + async fn get_or_create(&self, uuid: Uuid) -> Result> { + match self.index_store.write().await.entry(uuid.clone()) { + Entry::Vacant(entry) => { + match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(_) => { + todo!() + } + Entry::Occupied(entry) => { + todo!() + } + } + } + Entry::Occupied(entry) => Ok(entry.get().clone()), + } + } } impl MapIndexStore { diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 621bbb1d6..2936c59ea 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -2,6 +2,7 @@ mod index_actor; mod update_actor; mod uuid_resolver; mod update_store; +mod update_handler; use tokio::sync::oneshot; use super::IndexController; @@ -10,7 +11,6 @@ use super::IndexMetadata; use tokio::fs::File; use super::UpdateMeta; - pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index a0bf011c5..d182ef1c8 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -53,11 +53,11 @@ impl UpdateActor { } } - async fn handle_update(&self, _uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { let mut buf = Vec::new(); let mut payload = payload.unwrap(); payload.read_to_end(&mut buf).await.unwrap(); - let result = self.store.register_update(meta, &buf).unwrap(); + let result = self.store.register_update(meta, &buf, uuid).unwrap(); let _ = ret.send(Ok(UpdateStatus::Pending(result))); } } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs deleted file mode 100644 index 5781a2806..000000000 --- a/src/index_controller/local_index_controller/update_handler.rs +++ /dev/null @@ -1,255 +0,0 @@ -use std::collections::HashMap; -use std::io; -use std::sync::Arc; - -use anyhow::Result; -use flate2::read::GzDecoder; -use grenad::CompressionType; -use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use milli::Index; -use rayon::ThreadPool; - -use super::update_store::HandleUpdate; -use crate::index_controller::updates::{Failed, Processed, Processing}; -use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; -use crate::option::IndexerOpts; - -pub struct UpdateHandler { - index: Arc, - max_nb_chunks: Option, - chunk_compression_level: Option, - thread_pool: Arc, - log_frequency: usize, - max_memory: usize, - linked_hash_map_size: usize, - chunk_compression_type: CompressionType, - chunk_fusing_shrink_size: u64, -} - -impl UpdateHandler { - pub fn new( - opt: &IndexerOpts, - index: Arc, - thread_pool: Arc, - ) -> anyhow::Result { - Ok(Self { - index, - max_nb_chunks: opt.max_nb_chunks, - chunk_compression_level: opt.chunk_compression_level, - thread_pool, - log_frequency: opt.log_every_n, - max_memory: opt.max_memory.get_bytes() as usize, - linked_hash_map_size: opt.linked_hash_map_size, - chunk_compression_type: opt.chunk_compression_type, - chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - }) - } - - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { - // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); - if let Some(max_nb_chunks) = self.max_nb_chunks { - update_builder.max_nb_chunks(max_nb_chunks); - } - if let Some(chunk_compression_level) = self.chunk_compression_level { - update_builder.chunk_compression_level(chunk_compression_level); - } - update_builder.thread_pool(&self.thread_pool); - update_builder.log_every_n(self.log_frequency); - update_builder.max_memory(self.max_memory); - update_builder.linked_hash_map_size(self.linked_hash_map_size); - update_builder.chunk_compression_type(self.chunk_compression_type); - update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); - update_builder - } - - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: &[u8], - update_builder: UpdateBuilder, - primary_key: Option<&str>, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - - // Set the primary key if not set already, ignore if already set. - match (self.index.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - self.index.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), - } - - let mut builder = update_builder.index_documents(&mut wtxn, &self.index); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = true; - let reader = if gzipped && !content.is_empty() { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, &self.index); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_settings( - &self, - settings: &Settings, - update_builder: UpdateBuilder, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, &self.index); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = self.index.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, &self.index); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn delete_documents( - &self, - document_ids: &[u8], - update_builder: UpdateBuilder, - ) -> anyhow::Result { - let ids: Vec = serde_json::from_slice(document_ids)?; - let mut txn = self.index.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, &self.index)?; - - // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); - - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } -} - -impl HandleUpdate for UpdateHandler { - fn handle_update( - &mut self, - meta: Processing, - content: &[u8], - ) -> Result, Failed> { - use UpdateMeta::*; - - let update_id = meta.id(); - - let update_builder = self.update_buidler(update_id); - - let result = match meta.meta() { - DocumentsAddition { - method, - format, - primary_key, - } => self.update_documents( - *format, - *method, - content, - update_builder, - primary_key.as_deref(), - ), - ClearDocuments => self.clear_documents(update_builder), - DeleteDocuments => self.delete_documents(content, update_builder), - Settings(settings) => self.update_settings(settings, update_builder), - Facets(levels) => self.update_facets(levels, update_builder), - }; - - match result { - Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), - } - } -} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index b15593b58..b2ad54a14 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -1,5 +1,6 @@ use chrono::{Utc, DateTime}; use serde::{Serialize, Deserialize}; +use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -7,14 +8,16 @@ pub struct Pending { pub update_id: u64, pub meta: M, pub enqueued_at: DateTime, + pub index_uuid: Uuid, } impl Pending { - pub fn new(meta: M, update_id: u64) -> Self { + pub fn new(meta: M, update_id: u64, index_uuid: Uuid) -> Self { Self { enqueued_at: Utc::now(), meta, update_id, + index_uuid, } } @@ -73,6 +76,10 @@ impl Processing { self.from.meta() } + pub fn index_uuid(&self) -> &Uuid { + &self.from.index_uuid + } + pub fn process(self, meta: N) -> Processed { Processed { success: meta, From c4dfd5f0c3707e968cd2192e33d4ae42f4b6ca0b Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 27 Feb 2021 10:19:05 +0100 Subject: [PATCH 05/69] implement search and fix document addition --- src/data/mod.rs | 2 +- src/data/search.rs | 34 +- .../actor_index_controller/index_actor.rs | 128 +++++- .../actor_index_controller/mod.rs | 7 + .../actor_index_controller/update_handler.rs | 260 +++++++++++ .../actor_index_controller/update_store.rs | 423 ++++++++++++++++++ .../actor_index_controller/uuid_resolver.rs | 13 +- src/index_controller/mod.rs | 4 + src/routes/search.rs | 4 +- 9 files changed, 842 insertions(+), 33 deletions(-) create mode 100644 src/index_controller/actor_index_controller/update_handler.rs create mode 100644 src/index_controller/actor_index_controller/update_store.rs diff --git a/src/data/mod.rs b/src/data/mod.rs index 3d6858763..ecc8a7e2e 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,4 +1,4 @@ -mod search; +pub mod search; mod updates; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; diff --git a/src/data/search.rs b/src/data/search.rs index 23c6b463e..6ab792073 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -123,27 +123,27 @@ impl SearchQuery { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { - hits: Vec>, - nb_hits: u64, - query: String, - limit: usize, - offset: usize, - processing_time_ms: u128, + pub hits: Vec>, + pub nb_hits: u64, + pub query: String, + pub limit: usize, + pub offset: usize, + pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] - facet_distributions: Option>>, + pub facet_distributions: Option>>, } -struct Highlighter<'a, A> { +pub struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, } impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - fn new(stop_words: &'a fst::Set) -> Self { + pub fn new(stop_words: &'a fst::Set) -> Self { let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); Self { analyzer } } - fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -182,7 +182,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } } - fn highlight_record( + pub fn highlight_record( &self, object: &mut Map, words_to_highlight: &HashSet, @@ -199,16 +199,12 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } impl Data { - pub fn search>( + pub async fn search>( &self, - _index: S, - _search_query: SearchQuery, + index: S, + search_query: SearchQuery, ) -> anyhow::Result { - todo!() - //match self.index_controller.index(&index)? { - //Some(index) => Ok(search_query.perform(index)?), - //None => bail!("index {:?} doesn't exists", index.as_ref()), - //} + self.index_controller.search(index.as_ref().to_string(), search_query).await } pub async fn retrieve_documents( diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 6123ca774..0cb057a9b 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,6 +1,7 @@ use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; +use std::time::Instant; use chrono::Utc; use heed::EnvOpenOptions; @@ -11,8 +12,12 @@ use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use log::info; +use crate::data::SearchQuery; +use futures::stream::{StreamExt, Stream}; use super::update_handler::UpdateHandler; +use async_stream::stream; +use crate::data::SearchResult; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; @@ -22,7 +27,8 @@ type UpdateResult = std::result::Result, Failed, ret: oneshot::Sender> }, - Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, + Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, + Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, } struct IndexActor { @@ -43,6 +49,7 @@ pub enum IndexError { trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn get(&self, uuid: Uuid) -> Result>>; } impl IndexActor { @@ -54,13 +61,109 @@ impl IndexActor { } async fn run(mut self) { - loop { - match self.inbox.recv().await { - Some(IndexMsg::CreateIndex { uuid, primary_key, ret }) => self.handle_create_index(uuid, primary_key, ret).await, - Some(IndexMsg::Update { ret, meta, data }) => self.handle_update(meta, data, ret).await, - None => break, + let stream = stream! { + loop { + match self.inbox.recv().await { + Some(msg) => yield msg, + None => break, + } } - } + }; + + stream.for_each_concurent(Some(10), |msg| { + match msg { + IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret), + IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret), + IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret), + } + }) + } + + async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + + let before_search = Instant::now(); + let rtxn = index.read_txn().unwrap(); + + let mut search = index.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + //if let Some(ref facets) = query.facet_filters { + //if let Some(facets) = parse_facets(facets, index, &rtxn)? { + //search.facet_condition(facets); + //} + //} + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute().unwrap(); + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = crate::data::search::Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = index.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute().unwrap()) + } + None => None, + }; + + let result = Ok(SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }); + + ret.send(result) + }); + } async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { @@ -107,6 +210,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } + + pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Search { uuid, query, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -162,6 +272,10 @@ impl IndexStore for MapIndexStore { Entry::Occupied(entry) => Ok(entry.get().clone()), } } + + async fn get(&self, uuid: Uuid) -> Result>> { + Ok(self.index_store.read().await.get(&uuid).cloned()) + } } impl MapIndexStore { diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 2936c59ea..646d9cf45 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use super::IndexMetadata; use tokio::fs::File; use super::UpdateMeta; +use crate::data::{SearchResult, SearchQuery}; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, @@ -97,4 +98,10 @@ impl IndexController for ActorIndexController { fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { todo!() } + + async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let result = self.index_handle.search(uuid, query).await?; + Ok(result) + } } diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs new file mode 100644 index 000000000..d9ac2f866 --- /dev/null +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -0,0 +1,260 @@ +use std::collections::HashMap; +use std::io; +use std::fs::File; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use log::info; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::Index; +use rayon::ThreadPool; + +use crate::index_controller::updates::{Failed, Processed, Processing}; +use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::option::IndexerOpts; + +pub struct UpdateHandler { + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: ThreadPool, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new( + opt: &IndexerOpts, + ) -> anyhow::Result { + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + Ok(Self { + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: File, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + index: &Index, + ) -> anyhow::Result { + info!("performing document addition"); + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (index.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + index.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + + let mut builder = update_builder.index_documents(&mut wtxn, index); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = false; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); + + info!("document addition done: {:?}", result); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, index); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, index); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = index.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, index); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + fn delete_documents( + &self, + document_ids: File, + update_builder: UpdateBuilder, + index: &Index, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_reader(document_ids)?; + let mut txn = index.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, index)?; + + // We ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + pub fn handle_update( + &self, + meta: Processing, + content: File, + index: &Index, + ) -> Result, Failed> { + use UpdateMeta::*; + + let update_id = meta.id(); + + let update_builder = self.update_buidler(update_id); + + let result = match meta.meta() { + DocumentsAddition { + method, + format, + primary_key, + } => self.update_documents( + *format, + *method, + content, + update_builder, + primary_key.as_deref(), + index, + ), + ClearDocuments => self.clear_documents(update_builder, index), + DeleteDocuments => self.delete_documents(content, update_builder, index), + Settings(settings) => self.update_settings(settings, update_builder, index), + Facets(levels) => self.update_facets(levels, update_builder, index), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } + } +} diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs new file mode 100644 index 000000000..371ac7bd9 --- /dev/null +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -0,0 +1,423 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; +use std::io::{Cursor, SeekFrom, Seek}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; +use std::fs::File; +use uuid::Uuid; + +use crate::index_controller::updates::*; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait HandleUpdate { + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; +} + +impl HandleUpdate for F +where F: FnMut(Processing, File) -> Result, Failed> +{ + fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { + self(meta, content) + } +} + +impl UpdateStore +where + M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, + N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, + E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, +{ + pub fn open( + mut options: EnvOpenOptions, + path: P, + mut update_handler: U, + ) -> heed::Result> + where + P: AsRef, + U: HandleUpdate + Send + 'static, + { + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + // We need a weak reference so we can take ownership on the arc later when we + // want to close the index. + let update_store_weak = Arc::downgrade(&update_store); + std::thread::spawn(move || { + // Block and wait for something to process. + 'outer: for _ in notification_receiver { + loop { + match update_store_weak.upgrade() { + Some(update_store) => { + match update_store.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + // the ownership on the arc has been taken, we need to exit. + None => break 'outer, + } + } + } + }); + + Ok(update_store) + } + + pub fn prepare_for_closing(self) -> heed::EnvClosingEvent { + self.env.prepare_for_closing() + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: M, + content: &[u8], + index_uuid: Uuid, + ) -> heed::Result> { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id, index_uuid); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: HandleUpdate + Send + 'static, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we change the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + let mut cursor = Cursor::new(first_content); + let mut file = tempfile::tempfile()?; + std::io::copy(&mut cursor, &mut file)?; + file.seek(SeekFrom::Start(0))?; + // Process the pending update using the provided user function. + let result = handler.handle_update(processing, file); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + #[allow(dead_code)] + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + #[allow(dead_code)] + pub fn abort_pendings(&self) -> heed::Result)>> { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + impl HandleUpdate for F + where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { + self(meta, content) + } + } + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/actor_index_controller/uuid_resolver.rs index b75c0402c..3143ae8fc 100644 --- a/src/index_controller/actor_index_controller/uuid_resolver.rs +++ b/src/index_controller/actor_index_controller/uuid_resolver.rs @@ -4,7 +4,7 @@ use uuid::Uuid; use std::collections::HashMap; use std::sync::Arc; use std::collections::hash_map::Entry; -use log::info; +use log::{info, warn}; pub type Result = std::result::Result; @@ -22,8 +22,6 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>, }, - Shutdown, - } struct UuidResolverActor { @@ -46,11 +44,13 @@ impl UuidResolverActor { match self.inbox.recv().await { Some(Create { name, ret }) => self.handle_create(name, ret).await, Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, - Some(_) => {} + Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, // all senders have been dropped, need to quit. None => break, } } + + warn!("exiting uuid resolver loop"); } async fn handle_create(&self, name: String, ret: oneshot::Sender>) { @@ -62,6 +62,11 @@ impl UuidResolverActor { let result = self.store.create_uuid(name, false).await; let _ = ret.send(result); } + + async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { + let result = self.store.get_uuid(name).await; + let _ = ret.send(result); + } } #[derive(Clone)] diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dc6cc3863..cee276ea0 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -13,6 +13,8 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; use tokio::fs::File; +use crate::data::SearchResult; +use crate::data::SearchQuery; pub use updates::{Processed, Processing, Failed}; @@ -135,6 +137,8 @@ pub trait IndexController { primary_key: Option, ) -> anyhow::Result; + async fn search(&self, name: String, query: SearchQuery) -> Result; + /// Clear all documents in the given index. fn clear_documents(&self, index: String) -> anyhow::Result; diff --git a/src/routes/search.rs b/src/routes/search.rs index 7919c5412..1f4218555 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -83,7 +83,7 @@ async fn search_with_url_query( return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } }; - let search_result = data.search(&path.index_uid, query); + let search_result = data.search(&path.index_uid, query).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); @@ -101,7 +101,7 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(&path.index_uid, params.into_inner()); + let search_result = data.search(&path.index_uid, params.into_inner()).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); From 62532b8f79c54f9b318fbed777387173ad43756f Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Mar 2021 15:45:35 +0100 Subject: [PATCH 06/69] WIP concurent index store --- Cargo.lock | 125 ++++++++++++------ Cargo.toml | 1 + src/data/updates.rs | 5 +- .../actor_index_controller/index_actor.rs | 21 +-- .../actor_index_controller/update_actor.rs | 1 - .../actor_index_controller/update_store.rs | 6 +- 6 files changed, 106 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da8d649dd..7f3b20e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -421,9 +421,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" @@ -447,6 +447,27 @@ dependencies = [ "tokio 0.2.24", ] +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.42" @@ -609,9 +630,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" dependencies = [ "lazy_static", "memchr", @@ -641,10 +662,16 @@ dependencies = [ ] [[package]] -name = "byteorder" -version = "1.3.4" +name = "bytemuck" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" @@ -1022,15 +1049,15 @@ checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "winapi 0.3.9", ] [[package]] name = "flate2" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -1665,9 +1692,9 @@ checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "linked-hash-map" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lmdb-rkv-sys" @@ -1691,11 +1718,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1741,6 +1768,7 @@ dependencies = [ "anyhow", "assert-json-diff", "async-compression", + "async-stream", "async-trait", "byte-unit", "bytes 0.6.0", @@ -1853,7 +1881,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools 0.9.0", + "itertools 0.10.0", "levenshtein_automata", "linked-hash-map", "log", @@ -2060,9 +2088,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" dependencies = [ "num-traits", ] @@ -2097,7 +2125,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi 0.3.9", ] @@ -2496,10 +2524,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] -name = "regex" -version = "1.4.2" +name = "redox_syscall" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -2518,9 +2555,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remove_dir_all" @@ -2578,6 +2615,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + [[package]] name = "ring" version = "0.16.19" @@ -2595,11 +2638,13 @@ dependencies = [ [[package]] name = "roaring" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d60b41c8f25d07cecab125cb46ebbf234fc055effc61ca2392a3ef4f9422304" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" dependencies = [ + "bytemuck", "byteorder", + "retain_mut", ] [[package]] @@ -2721,18 +2766,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ "proc-macro2", "quote", @@ -2741,9 +2786,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -2983,9 +3028,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.55" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a571a711dddd09019ccc628e1b17fe87c59b09d513c06c026877aa708334f37a" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ "proc-macro2", "quote", @@ -3021,7 +3066,7 @@ checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" dependencies = [ "filetime", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "xattr", ] @@ -3037,14 +3082,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand 0.7.3", - "redox_syscall", + "rand 0.8.3", + "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", ] diff --git a/Cargo.toml b/Cargo.toml index 126f859e7..aaf571ef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ itertools = "0.10.0" either = "1.6.1" async-trait = "0.1.42" thiserror = "1.0.24" +async-stream = "0.3.0" [dependencies.sentry] default-features = false diff --git a/src/data/updates.rs b/src/data/updates.rs index a559d812a..7db04cd5d 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -23,10 +23,11 @@ impl Data { { let file = tempfile::tempfile_in(".")?; let mut file = File::from_std(file); - while let Some(Ok(bytes)) = stream.next().await { - file.write(bytes.as_ref()).await; + while let Some(item) = stream.next().await { + file.write_all(&item?).await?; } file.seek(std::io::SeekFrom::Start(0)).await?; + file.flush().await?; let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 0cb057a9b..7100edef6 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -13,7 +13,7 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use log::info; use crate::data::SearchQuery; -use futures::stream::{StreamExt, Stream}; +use futures::stream::StreamExt; use super::update_handler::UpdateHandler; use async_stream::stream; @@ -32,7 +32,7 @@ enum IndexMsg { } struct IndexActor { - inbox: mpsc::Receiver, + inbox: Option>, update_handler: Arc, store: S, } @@ -57,26 +57,31 @@ impl IndexActor { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); + let inbox = Some(inbox); Self { inbox, store, update_handler } } async fn run(mut self) { + let mut inbox = self.inbox.take().expect("Index Actor must have a inbox at this point."); + let stream = stream! { loop { - match self.inbox.recv().await { + match inbox.recv().await { Some(msg) => yield msg, None => break, } } }; - stream.for_each_concurent(Some(10), |msg| { + let fut = stream.for_each_concurrent(Some(10), |msg| async { match msg { - IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret), - IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret), - IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret), + IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, + IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, } - }) + }); + + fut.await; } async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index d182ef1c8..3d783009f 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -41,7 +41,6 @@ impl UpdateActor { } async fn run(mut self) { - info!("started update actor."); loop { diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs index 371ac7bd9..d14f47f05 100644 --- a/src/index_controller/actor_index_controller/update_store.rs +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -1,6 +1,6 @@ use std::path::Path; use std::sync::{Arc, RwLock}; -use std::io::{Cursor, SeekFrom, Seek}; +use std::io::{Cursor, SeekFrom, Seek, Write}; use crossbeam_channel::Sender; use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; @@ -192,7 +192,9 @@ where .replace(processing.clone()); let mut cursor = Cursor::new(first_content); let mut file = tempfile::tempfile()?; - std::io::copy(&mut cursor, &mut file)?; + let n = std::io::copy(&mut cursor, &mut file)?; + println!("copied count: {}", n); + file.flush()?; file.seek(SeekFrom::Start(0))?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); From 9aca6fab88857dec3ce55b3bc424446d35bb7834 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 10:57:13 +0100 Subject: [PATCH 07/69] completely file backed udpates --- src/data/mod.rs | 2 +- src/data/updates.rs | 14 +--- .../actor_index_controller/mod.rs | 28 +++++-- .../actor_index_controller/update_actor.rs | 84 ++++++++++++------- .../actor_index_controller/update_store.rs | 22 ++--- src/index_controller/mod.rs | 4 +- 6 files changed, 90 insertions(+), 64 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index ecc8a7e2e..0b57f9a55 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -62,7 +62,7 @@ impl Data { let path = options.db_path.clone(); //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = ActorIndexController::new(); + let index_controller = ActorIndexController::new(&path); let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { diff --git a/src/data/updates.rs b/src/data/updates.rs index 7db04cd5d..01f5174a2 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -3,9 +3,6 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; -use tokio::fs::File; -use tokio::io::{AsyncWriteExt, AsyncSeekExt}; -use futures::prelude::stream::StreamExt; use crate::index_controller::UpdateStatus; use crate::index_controller::{Settings, IndexMetadata}; @@ -17,18 +14,11 @@ impl Data { index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, - mut stream: Payload, + stream: Payload, primary_key: Option, ) -> anyhow::Result { - let file = tempfile::tempfile_in(".")?; - let mut file = File::from_std(file); - while let Some(item) = stream.next().await { - file.write_all(&item?).await?; - } - file.seek(std::io::SeekFrom::Start(0)).await?; - file.flush().await?; - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, file, primary_key).await?; + let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; Ok(update_status) } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 646d9cf45..58e58a45a 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -4,25 +4,29 @@ mod uuid_resolver; mod update_store; mod update_handler; -use tokio::sync::oneshot; +use std::path::Path; + +use tokio::sync::{mpsc, oneshot}; use super::IndexController; use uuid::Uuid; use super::IndexMetadata; -use tokio::fs::File; +use futures::stream::StreamExt; +use actix_web::web::Payload; use super::UpdateMeta; use crate::data::{SearchResult, SearchQuery}; +use actix_web::web::Bytes; pub struct ActorIndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, - update_handle: update_actor::UpdateActorHandle, + update_handle: update_actor::UpdateActorHandle, } impl ActorIndexController { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); let index_actor = index_actor::IndexActorHandle::new(); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone()); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Self { uuid_resolver, index_handle: index_actor, update_handle } } } @@ -43,12 +47,22 @@ impl IndexController for ActorIndexController { index: String, method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, - data: File, + mut payload: Payload, primary_key: Option, ) -> anyhow::Result { let uuid = self.uuid_resolver.get_or_create(index).await?; let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let status = self.update_handle.update(meta, Some(data), uuid).await?; + let (sender, receiver) = mpsc::channel(10); + + // It is necessary to spawn a local task to senf the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + while let Some(bytes) = payload.next().await { + sender.send(bytes.unwrap()).await; + } + }); + let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 3d783009f..6fc873715 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -1,22 +1,24 @@ -use super::index_actor::IndexActorHandle; -use uuid::Uuid; -use tokio::sync::{mpsc, oneshot}; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; -use thiserror::Error; -use tokio::io::AsyncReadExt; -use log::info; -use tokio::fs::File; -use std::path::PathBuf; use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use log::info; +use super::index_actor::IndexActorHandle; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; + +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult, updates::Pending}; + pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; #[derive(Debug, Error)] pub enum UpdateError {} -enum UpdateMsg { +enum UpdateMsg { CreateIndex{ uuid: Uuid, ret: oneshot::Sender>, @@ -24,20 +26,30 @@ enum UpdateMsg { Update { uuid: Uuid, meta: UpdateMeta, - payload: Option, + data: mpsc::Receiver, ret: oneshot::Sender> } } -struct UpdateActor { +struct UpdateActor { + path: PathBuf, store: Arc, - inbox: mpsc::Receiver, + inbox: mpsc::Receiver>, index_handle: IndexActorHandle, } -impl UpdateActor { - fn new(store: Arc, inbox: mpsc::Receiver, index_handle: IndexActorHandle) -> Self { - Self { store, inbox, index_handle } +impl UpdateActor +where D: AsRef<[u8]> + Sized + 'static, +{ + fn new( + store: Arc, + inbox: mpsc::Receiver>, + index_handle: IndexActorHandle, + path: impl AsRef, + ) -> Self { + let path = path.as_ref().to_owned().join("update_files"); + create_dir_all(&path).unwrap(); + Self { store, inbox, index_handle, path } } async fn run(mut self) { @@ -45,29 +57,43 @@ impl UpdateActor { loop { match self.inbox.recv().await { - Some(UpdateMsg::Update { uuid, meta, payload, ret }) => self.handle_update(uuid, meta, payload, ret).await, + Some(UpdateMsg::Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, Some(_) => {} None => {} } } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, payload: Option, ret: oneshot::Sender>) { - let mut buf = Vec::new(); - let mut payload = payload.unwrap(); - payload.read_to_end(&mut buf).await.unwrap(); - let result = self.store.register_update(meta, &buf, uuid).unwrap(); + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver, ret: oneshot::Sender>) { + let store = self.store.clone(); + let update_file_id = uuid::Uuid::new_v4(); + let path = self.path.join(format!("update_{}", update_file_id)); + let mut file = File::create(&path).await.unwrap(); + + while let Some(bytes) = payload.recv().await { + file.write_all(bytes.as_ref()).await; + } + + file.flush().await; + + let file = file.into_std().await; + + let result = tokio::task::spawn_blocking(move || -> anyhow::Result> { + Ok(store.register_update(meta, path, uuid)?) + }).await.unwrap().unwrap(); let _ = ret.send(Ok(UpdateStatus::Pending(result))); } } #[derive(Clone)] -pub struct UpdateActorHandle { - sender: mpsc::Sender, +pub struct UpdateActorHandle { + sender: mpsc::Sender>, } -impl UpdateActorHandle { - pub fn new(index_handle: IndexActorHandle) -> Self { +impl UpdateActorHandle +where D: AsRef<[u8]> + Sized + 'static, +{ + pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let (sender, receiver) = mpsc::channel(100); let mut options = heed::EnvOpenOptions::new(); options.map_size(4096 * 100_000); @@ -79,16 +105,16 @@ impl UpdateActorHandle { let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle_clone.update(meta, file)) }).unwrap(); - let actor = UpdateActor::new(store, receiver, index_handle); + let actor = UpdateActor::new(store, receiver, index_handle, path); tokio::task::spawn_local(actor.run()); Self { sender } } - pub async fn update(&self, meta: UpdateMeta, payload: Option, uuid: Uuid) -> Result { + pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, - payload, + data, meta, ret, }; diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/actor_index_controller/update_store.rs index d14f47f05..ae4bfb8d8 100644 --- a/src/index_controller/actor_index_controller/update_store.rs +++ b/src/index_controller/actor_index_controller/update_store.rs @@ -1,9 +1,9 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::io::{Cursor, SeekFrom, Seek, Write}; +use std::fs::remove_file; use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson}; use heed::{EnvOpenOptions, Env, Database}; use serde::{Serialize, Deserialize}; use std::fs::File; @@ -17,7 +17,7 @@ type BEU64 = heed::zerocopy::U64; pub struct UpdateStore { env: Env, pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, + pending: Database, SerdeJson>, processed_meta: Database, SerdeJson>>, failed_meta: Database, SerdeJson>>, aborted_meta: Database, SerdeJson>>, @@ -140,7 +140,7 @@ where pub fn register_update( &self, meta: M, - content: &[u8], + content: impl AsRef, index_uuid: Uuid, ) -> heed::Result> { let mut wtxn = self.env.write_txn()?; @@ -154,7 +154,7 @@ where let meta = Pending::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; + self.pending.put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; wtxn.commit()?; @@ -178,7 +178,7 @@ where // a reader while processing it, not a writer. match first_meta { Some((first_id, pending)) => { - let first_content = self.pending + let content_path = self.pending .get(&rtxn, &first_id)? .expect("associated update content"); @@ -190,12 +190,7 @@ where .write() .unwrap() .replace(processing.clone()); - let mut cursor = Cursor::new(first_content); - let mut file = tempfile::tempfile()?; - let n = std::io::copy(&mut cursor, &mut file)?; - println!("copied count: {}", n); - file.flush()?; - file.seek(SeekFrom::Start(0))?; + let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); drop(rtxn); @@ -209,6 +204,7 @@ where .unwrap() .take(); self.pending_meta.delete(&mut wtxn, &first_id)?; + remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; match result { Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index cee276ea0..c3a09b2aa 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -12,7 +12,7 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use tokio::fs::File; +use actix_web::web::Payload; use crate::data::SearchResult; use crate::data::SearchQuery; @@ -133,7 +133,7 @@ pub trait IndexController { index: String, method: IndexDocumentsMethod, format: UpdateFormat, - data: File, + data: Payload, primary_key: Option, ) -> anyhow::Result; From 7c7143d435dee28f1772c2532f446c31c14e940a Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 11:43:51 +0100 Subject: [PATCH 08/69] remove IndexController interface --- src/data/mod.rs | 10 +- .../actor_index_controller/mod.rs | 32 ++-- src/index_controller/mod.rs | 168 ------------------ 3 files changed, 18 insertions(+), 192 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 0b57f9a55..f2798c868 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexController, IndexMetadata, Settings, IndexSettings}; -use crate::index_controller::actor_index_controller::ActorIndexController; +use crate::index_controller::{IndexMetadata, Settings, IndexSettings}; +use crate::index_controller::actor_index_controller::IndexController; use crate::option::Opt; #[derive(Clone)] @@ -26,9 +26,8 @@ impl Deref for Data { } } -#[derive(Clone)] pub struct DataInner { - pub index_controller: Arc, + pub index_controller: IndexController, pub api_keys: ApiKeys, options: Opt, } @@ -62,8 +61,7 @@ impl Data { let path = options.db_path.clone(); //let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = ActorIndexController::new(&path); - let index_controller = Arc::new(index_controller); + let index_controller = IndexController::new(&path); let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 58e58a45a..34fcf55f0 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -7,7 +7,6 @@ mod update_handler; use std::path::Path; use tokio::sync::{mpsc, oneshot}; -use super::IndexController; use uuid::Uuid; use super::IndexMetadata; use futures::stream::StreamExt; @@ -16,21 +15,12 @@ use super::UpdateMeta; use crate::data::{SearchResult, SearchQuery}; use actix_web::web::Bytes; -pub struct ActorIndexController { +pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, update_handle: update_actor::UpdateActorHandle, } -impl ActorIndexController { - pub fn new(path: impl AsRef) -> Self { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } - } -} - enum IndexControllerMsg { CreateIndex { uuid: Uuid, @@ -40,9 +30,15 @@ enum IndexControllerMsg { Shutdown, } -#[async_trait::async_trait(?Send)] -impl IndexController for ActorIndexController { - async fn add_documents( +impl IndexController { + pub fn new(path: impl AsRef) -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + Self { uuid_resolver, index_handle: index_actor, update_handle } + } + + pub async fn add_documents( &self, index: String, method: milli::update::IndexDocumentsMethod, @@ -78,7 +74,7 @@ impl IndexController for ActorIndexController { todo!() } - async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { + pub async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { let super::IndexSettings { name, primary_key } = index_settings; let uuid = self.uuid_resolver.create(name.unwrap()).await?; let index_meta = self.index_handle.create_index(uuid, primary_key).await?; @@ -93,7 +89,7 @@ impl IndexController for ActorIndexController { todo!() } - fn index(&self, name: String) -> anyhow::Result>> { + pub fn index(&self, name: String) -> anyhow::Result>> { todo!() } @@ -105,7 +101,7 @@ impl IndexController for ActorIndexController { todo!() } - fn list_indexes(&self) -> anyhow::Result> { + pub fn list_indexes(&self) -> anyhow::Result> { todo!() } @@ -113,7 +109,7 @@ impl IndexController for ActorIndexController { todo!() } - async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); let result = self.index_handle.search(uuid, query).await?; Ok(result) diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index c3a09b2aa..fbe7a161d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,20 +1,14 @@ pub mod actor_index_controller; -//mod local_index_controller; mod updates; use std::collections::HashMap; use std::num::NonZeroUsize; -use std::sync::Arc; use anyhow::Result; use chrono::{DateTime, Utc}; -use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; use uuid::Uuid; -use actix_web::web::Payload; -use crate::data::SearchResult; -use crate::data::SearchQuery; pub use updates::{Processed, Processing, Failed}; @@ -109,165 +103,3 @@ pub struct IndexSettings { pub name: Option, pub primary_key: Option, } - -/// The `IndexController` is in charge of the access to the underlying indices. It splits the logic -/// for read access which is provided thanks to an handle to the index, and write access which must -/// be provided. This allows the implementer to define the behaviour of write accesses to the -/// indices, and abstract the scheduling of the updates. The implementer must be able to provide an -/// instance of `IndexStore` -#[async_trait::async_trait(?Send)] -pub trait IndexController { - - /* - * Write operations - * - * Logic for the write operation need to be provided by the implementer, since they can be made - * asynchronous thanks to an update_store for example. - * - * */ - - /// Perform document addition on the database. If the provided index does not exist, it will be - /// created when the addition is applied to the index. - async fn add_documents( - &self, - index: String, - method: IndexDocumentsMethod, - format: UpdateFormat, - data: Payload, - primary_key: Option, - ) -> anyhow::Result; - - async fn search(&self, name: String, query: SearchQuery) -> Result; - - /// Clear all documents in the given index. - fn clear_documents(&self, index: String) -> anyhow::Result; - - /// Delete all documents in `document_ids`. - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result; - - /// Updates an index settings. If the index does not exist, it will be created when the update - /// is applied to the index. - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result; - - /// Create an index with the given `index_uid`. - async fn create_index(&self, index_settings: IndexSettings) -> Result; - - /// Delete index with the given `index_uid`, attempting to close it beforehand. - fn delete_index(&self, index_uid: String) -> Result<()>; - - /// Swap two indexes, concretely, it simply swaps the index the names point to. - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> Result<()>; - - /// Returns, if it exists, the `Index` with the povided name. - fn index(&self, name: String) -> anyhow::Result>>; - - /// Returns the udpate status an update - fn update_status(&self, index: String, id: u64) -> anyhow::Result>; - - /// Returns all the udpate status for an index - fn all_update_status(&self, index: String) -> anyhow::Result>; - - /// List all the indexes - fn list_indexes(&self) -> anyhow::Result>; - - fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result; -} - - -#[cfg(test)] -#[macro_use] -pub(crate) mod test { - use super::*; - - #[macro_export] - macro_rules! make_index_controller_tests { - ($controller_buider:block) => { - #[test] - fn test_create_and_list_indexes() { - crate::index_controller::test::create_and_list_indexes($controller_buider); - } - - #[test] - fn test_create_index_with_no_name_is_error() { - crate::index_controller::test::create_index_with_no_name_is_error($controller_buider); - } - - #[test] - fn test_update_index() { - crate::index_controller::test::update_index($controller_buider); - } - }; - } - - pub(crate) fn create_and_list_indexes(controller: impl IndexController) { - let settings1 = IndexSettings { - name: Some(String::from("test_index")), - primary_key: None, - }; - - let settings2 = IndexSettings { - name: Some(String::from("test_index2")), - primary_key: Some(String::from("foo")), - }; - - controller.create_index(settings1).unwrap(); - controller.create_index(settings2).unwrap(); - - let indexes = controller.list_indexes().unwrap(); - assert_eq!(indexes.len(), 2); - assert_eq!(indexes[0].uid, "test_index"); - assert_eq!(indexes[1].uid, "test_index2"); - assert_eq!(indexes[1].primary_key.clone().unwrap(), "foo"); - } - - pub(crate) fn create_index_with_no_name_is_error(controller: impl IndexController) { - let settings = IndexSettings { - name: None, - primary_key: None, - }; - assert!(controller.create_index(settings).is_err()); - } - - pub(crate) fn update_index(controller: impl IndexController) { - - let settings = IndexSettings { - name: Some(String::from("test")), - primary_key: None, - }; - - assert!(controller.create_index(settings).is_ok()); - - // perform empty update returns index meta unchanged - let settings = IndexSettings { - name: None, - primary_key: None, - }; - - let result = controller.update_index("test", settings).unwrap(); - assert_eq!(result.uid, "test"); - assert_eq!(result.created_at, result.updated_at); - assert!(result.primary_key.is_none()); - - // Changing the name trigger an error - let settings = IndexSettings { - name: Some(String::from("bar")), - primary_key: None, - }; - - assert!(controller.update_index("test", settings).is_err()); - - // Update primary key - let settings = IndexSettings { - name: None, - primary_key: Some(String::from("foo")), - }; - - let result = controller.update_index("test", settings.clone()).unwrap(); - assert_eq!(result.uid, "test"); - assert!(result.created_at < result.updated_at); - assert_eq!(result.primary_key.unwrap(), "foo"); - - // setting the primary key again is an error - assert!(controller.update_index("test", settings).is_err()); - } -} From 70d935a2da2fc73db469aa9ccc73a87f6e06ade5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 11:53:01 +0100 Subject: [PATCH 09/69] refactor index serach for better error handling --- .../actor_index_controller/index_actor.rs | 163 +++++++++--------- 1 file changed, 83 insertions(+), 80 deletions(-) diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 7100edef6..48b10c8a3 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -87,85 +87,7 @@ impl IndexActor { async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - - let before_search = Instant::now(); - let rtxn = index.read_txn().unwrap(); - - let mut search = index.search(&rtxn); - - if let Some(ref query) = query.q { - search.query(query); - } - - search.limit(query.limit); - search.offset(query.offset.unwrap_or_default()); - - //if let Some(ref facets) = query.facet_filters { - //if let Some(facets) = parse_facets(facets, index, &rtxn)? { - //search.facet_condition(facets); - //} - //} - let milli::SearchResult { - documents_ids, - found_words, - candidates, - .. - } = search.execute().unwrap(); - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = crate::data::search::Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); - if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match query.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute().unwrap()) - } - None => None, - }; - - let result = Ok(SearchResult { - hits: documents, - nb_hits, - query: query.q.clone().unwrap_or_default(), - limit: query.limit, - offset: query.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }); - + let result = perform_search(&index, query); ret.send(result) }); @@ -177,7 +99,7 @@ impl IndexActor { } async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { - info!("processing update"); + info!("processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); @@ -187,6 +109,87 @@ impl IndexActor { } } +fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result { + let before_search = Instant::now(); + let rtxn = index.read_txn()?; + + let mut search = index.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + //if let Some(ref facets) = query.facet_filters { + //if let Some(facets) = parse_facets(facets, index, &rtxn)? { + //search.facet_condition(facets); + //} + //} + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute()?; + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = crate::data::search::Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids)? { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = index.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute()?) + } + None => None, + }; + + let result = SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }; + Ok(result) +} + #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, From e285404c3e7fc5d9f490085f685d260a52096936 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 12:16:16 +0100 Subject: [PATCH 10/69] handle errors when sendign payload to actor --- .../actor_index_controller/mod.rs | 9 ++++- .../actor_index_controller/update_actor.rs | 35 +++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 34fcf55f0..be63b42d4 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -55,9 +55,16 @@ impl IndexController { // registered and the update_actor that waits for the the payload to be sent to it. tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { - sender.send(bytes.unwrap()).await; + match bytes { + Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Err(e) => { + let error: Box = Box::new(e); + sender.send(Err(error)).await; }, + } } }); + + // This must be done *AFTER* spawning the task. let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 6fc873715..1541cd4f7 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -10,13 +10,17 @@ use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult, updates::Pending}; +use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; +type PayloadData = std::result::Result>; #[derive(Debug, Error)] -pub enum UpdateError {} +pub enum UpdateError { + #[error("error with update: {0}")] + Error(Box), +} enum UpdateMsg { CreateIndex{ @@ -26,7 +30,7 @@ enum UpdateMsg { Update { uuid: Uuid, meta: UpdateMeta, - data: mpsc::Receiver, + data: mpsc::Receiver>, ret: oneshot::Sender> } } @@ -64,24 +68,35 @@ where D: AsRef<[u8]> + Sized + 'static, } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver, ret: oneshot::Sender>) { + async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, ret: oneshot::Sender>) { let store = self.store.clone(); let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); let mut file = File::create(&path).await.unwrap(); while let Some(bytes) = payload.recv().await { - file.write_all(bytes.as_ref()).await; + match bytes { + Ok(bytes) => { + file.write_all(bytes.as_ref()).await; + } + Err(e) => { + ret.send(Err(UpdateError::Error(e))); + return + } + } } file.flush().await; let file = file.into_std().await; - let result = tokio::task::spawn_blocking(move || -> anyhow::Result> { - Ok(store.register_update(meta, path, uuid)?) - }).await.unwrap().unwrap(); - let _ = ret.send(Ok(UpdateStatus::Pending(result))); + let result = tokio::task::spawn_blocking(move || { + let result = store + .register_update(meta, path, uuid) + .map(|pending| UpdateStatus::Pending(pending)) + .map_err(|e| UpdateError::Error(Box::new(e))); + let _ = ret.send(result); + }).await; } } @@ -110,7 +125,7 @@ where D: AsRef<[u8]> + Sized + 'static, Self { sender } } - pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver, uuid: Uuid) -> Result { + pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver>, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, From 3cd799a74442985347720b2c16a0686667711b67 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 14:39:44 +0100 Subject: [PATCH 11/69] fix update files created in the wrong place --- src/data/mod.rs | 2 ++ .../actor_index_controller/index_actor.rs | 4 ++-- src/index_controller/actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_actor.rs | 9 ++++++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index f2798c868..aa601a670 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -59,7 +59,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); + //let indexer_opts = options.indexer_options.clone(); + create_dir_all(&path)?; let index_controller = IndexController::new(&path); diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 48b10c8a3..04920260e 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -196,10 +196,10 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> Self { let (sender, receiver) = mpsc::channel(100); - let store = MapIndexStore::new("data.ms"); + let store = MapIndexStore::new(path); let actor = IndexActor::new(receiver, store); tokio::task::spawn(actor.run()); Self { sender } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index be63b42d4..882b20963 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -33,7 +33,7 @@ enum IndexControllerMsg { impl IndexController { pub fn new(path: impl AsRef) -> Self { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(&path); let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Self { uuid_resolver, index_handle: index_actor, update_handle } } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index 1541cd4f7..e82f01092 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -112,9 +112,12 @@ where D: AsRef<[u8]> + Sized + 'static, let (sender, receiver) = mpsc::channel(100); let mut options = heed::EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let mut path = PathBuf::new(); - path.push("data.ms"); - path.push("updates"); + + let path = path + .as_ref() + .to_owned() + .join("updates"); + create_dir_all(&path).unwrap(); let index_handle_clone = index_handle.clone(); let store = UpdateStore::open(options, &path, move |meta, file| { From eff8570f591fe32a6106087807e3fe8c18e8e5e4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Mar 2021 15:10:00 +0100 Subject: [PATCH 12/69] handle ctrl-c shutdown --- src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c9042b88e..c5e2d11e6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -112,7 +112,9 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box .wrap(middleware::Logger::default()) .wrap(middleware::Compress::default()) .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - }); + }) + // Disable signals allows the server to terminate immediately when a user enter CTRL-C + .disable_signals(); if let Some(config) = opt.get_ssl_config()? { http_server From ae3c8af56cbd938666b4a3b4bb23db871c5c38c5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 10:40:37 +0100 Subject: [PATCH 13/69] enable faceted search --- .../actor_index_controller/index_actor.rs | 76 +++++++++++++++---- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 04920260e..b45b683bf 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -1,23 +1,24 @@ +use std::collections::{HashMap, hash_map::Entry}; use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; use std::time::Instant; +use anyhow::bail; +use either::Either; +use async_stream::stream; use chrono::Utc; -use heed::EnvOpenOptions; -use milli::Index; -use std::collections::HashMap; -use std::collections::hash_map::Entry; +use futures::stream::StreamExt; +use heed::{EnvOpenOptions, RoTxn}; +use log::info; +use milli::{FacetCondition, Index}; +use serde_json::Value; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use log::info; -use crate::data::SearchQuery; -use futures::stream::StreamExt; use super::update_handler::UpdateHandler; -use async_stream::stream; -use crate::data::SearchResult; +use crate::data::{SearchQuery, SearchResult}; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; @@ -99,7 +100,7 @@ impl IndexActor { } async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { - info!("processing update {}", meta.id()); + info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); @@ -122,11 +123,12 @@ fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result anyhow::Result, +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, index, ands) +} + +fn parse_facets( + facets: &Value, + index: &Index, + txn: &RoTxn, +) -> anyhow::Result> { + match facets { + // Disabled for now + //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::Array(arr) => parse_facets_array(txn, index, arr), + v => bail!( + "Invalid facet expression, expected Array, found: {:?}", + v + ), + } +} #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, From 9e2a95b1a372ee2b8909807e16207fa8ed989b2f Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 11:23:41 +0100 Subject: [PATCH 14/69] refactor search --- src/data/mod.rs | 2 - src/data/search.rs | 243 +----------------- .../actor_index_controller/index_actor.rs | 154 +---------- .../actor_index_controller/mod.rs | 2 +- .../actor_index_controller/update_handler.rs | 14 +- src/lib.rs | 1 + src/routes/search.rs | 2 +- 7 files changed, 23 insertions(+), 395 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index aa601a670..58acb105a 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,8 +1,6 @@ pub mod search; mod updates; -pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; - use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; diff --git a/src/data/search.rs b/src/data/search.rs index 6ab792073..d6bf8438e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,203 +1,8 @@ -use std::collections::{HashSet, BTreeMap}; -use std::mem; -use std::time::Instant; - -use anyhow::bail; -use either::Either; -use heed::RoTxn; -use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{obkv_to_json, FacetCondition, Index, facet::FacetValue}; -use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use crate::index::{SearchQuery, SearchResult}; use super::Data; -pub const DEFAULT_SEARCH_LIMIT: usize = 20; - -const fn default_search_limit() -> usize { - DEFAULT_SEARCH_LIMIT -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -#[allow(dead_code)] -pub struct SearchQuery { - pub q: Option, - pub offset: Option, - #[serde(default = "default_search_limit")] - pub limit: usize, - pub attributes_to_retrieve: Option>, - pub attributes_to_crop: Option>, - pub crop_length: Option, - pub attributes_to_highlight: Option>, - pub filters: Option, - pub matches: Option, - pub facet_filters: Option, - pub facet_distributions: Option>, -} - -impl SearchQuery { - pub fn perform(&self, index: impl AsRef) -> anyhow::Result { - let index = index.as_ref(); - let before_search = Instant::now(); - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - - if let Some(ref query) = self.q { - search.query(query); - } - - search.limit(self.limit); - search.offset(self.offset.unwrap_or_default()); - - if let Some(ref facets) = self.facet_filters { - if let Some(facets) = parse_facets(facets, index, &rtxn)? { - search.facet_condition(facets); - } - } - - let milli::SearchResult { - documents_ids, - found_words, - candidates, - } = search.execute()?; - - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn)?; - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn)?; - - let attributes_to_retrieve_ids = match self.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids)? { - let mut object = obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv)?; - if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match self.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute()?) - } - None => None, - }; - - Ok(SearchResult { - hits: documents, - nb_hits, - query: self.q.clone().unwrap_or_default(), - limit: self.limit, - offset: self.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }) - } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchResult { - pub hits: Vec>, - pub nb_hits: u64, - pub query: String, - pub limit: usize, - pub offset: usize, - pub processing_time_ms: u128, - #[serde(skip_serializing_if = "Option::is_none")] - pub facet_distributions: Option>>, -} - -pub struct Highlighter<'a, A> { - analyzer: Analyzer<'a, A>, -} - -impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - pub fn new(stop_words: &'a fst::Set) -> Self { - let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); - Self { analyzer } - } - - pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { - match value { - Value::Null => Value::Null, - Value::Bool(boolean) => Value::Bool(boolean), - Value::Number(number) => Value::Number(number), - Value::String(old_string) => { - let mut string = String::new(); - let analyzed = self.analyzer.analyze(&old_string); - for (word, token) in analyzed.reconstruct() { - if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); - if to_highlight { - string.push_str("") - } - string.push_str(word); - if to_highlight { - string.push_str("") - } - } else { - string.push_str(word); - } - } - Value::String(string) - } - Value::Array(values) => Value::Array( - values - .into_iter() - .map(|v| self.highlight_value(v, words_to_highlight)) - .collect(), - ), - Value::Object(object) => Value::Object( - object - .into_iter() - .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) - .collect(), - ), - } - } - - pub fn highlight_record( - &self, - object: &mut Map, - words_to_highlight: &HashSet, - attributes_to_highlight: &HashSet, - ) { - // TODO do we need to create a string for element that are not and needs to be highlight? - for (key, value) in object.iter_mut() { - if attributes_to_highlight.contains(key) { - let old_value = mem::take(value); - *value = self.highlight_value(old_value, words_to_highlight); - } - } - } -} - impl Data { pub async fn search>( &self, @@ -303,49 +108,3 @@ impl Data { //document } } - -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &Vec, -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FacetCondition::from_array(txn, index, ands) -} - -fn parse_facets( - facets: &Value, - index: &Index, - txn: &RoTxn, -) -> anyhow::Result> { - match facets { - // Disabled for now - //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), - Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), - } -} - diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index b45b683bf..27d206c06 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -2,25 +2,20 @@ use std::collections::{HashMap, hash_map::Entry}; use std::fs::{File, create_dir_all}; use std::path::{PathBuf, Path}; use std::sync::Arc; -use std::time::Instant; -use anyhow::bail; -use either::Either; use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; -use heed::{EnvOpenOptions, RoTxn}; +use heed::EnvOpenOptions; use log::info; -use milli::{FacetCondition, Index}; -use serde_json::Value; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::data::{SearchQuery, SearchResult}; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; use crate::option::IndexerOpts; +use crate::index::{Index, SearchQuery, SearchResult}; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -49,8 +44,8 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn get(&self, uuid: Uuid) -> Result>>; + async fn get_or_create(&self, uuid: Uuid) -> Result; + async fn get(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -88,7 +83,7 @@ impl IndexActor { async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = perform_search(&index, query); + let result = index.perform_search(query); ret.send(result) }); @@ -104,138 +99,12 @@ impl IndexActor { let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index.as_ref())).await; + let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index)).await; let result = result.unwrap(); let _ = ret.send(result); } } -fn perform_search(index: &Index, query: SearchQuery) -> anyhow::Result { - let before_search = Instant::now(); - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - - if let Some(ref query) = query.q { - search.query(query); - } - - search.limit(query.limit); - search.offset(query.offset.unwrap_or_default()); - - if let Some(ref facets) = query.facet_filters { - if let Some(facets) = parse_facets(facets, index, &rtxn)? { - search.facet_condition(facets); - } - } - - let milli::SearchResult { - documents_ids, - found_words, - candidates, - .. - } = search.execute()?; - let mut documents = Vec::new(); - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - - let displayed_fields_ids = index.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let stop_words = fst::Set::default(); - let highlighter = crate::data::search::Highlighter::new(&stop_words); - - for (_id, obkv) in index.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); - if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); - } - documents.push(object); - } - - let nb_hits = candidates.len(); - - let facet_distributions = match query.facet_distributions { - Some(ref fields) => { - let mut facet_distribution = index.facets_distribution(&rtxn); - if fields.iter().all(|f| f != "*") { - facet_distribution.facets(fields); - } - Some(facet_distribution.candidates(candidates).execute()?) - } - None => None, - }; - - let result = SearchResult { - hits: documents, - nb_hits, - query: query.q.clone().unwrap_or_default(), - limit: query.limit, - offset: query.offset.unwrap_or_default(), - processing_time_ms: before_search.elapsed().as_millis(), - facet_distributions, - }; - Ok(result) -} - -fn parse_facets_array( - txn: &RoTxn, - index: &Index, - arr: &Vec, -) -> anyhow::Result> { - let mut ands = Vec::new(); - for value in arr { - match value { - Value::String(s) => ands.push(Either::Right(s.clone())), - Value::Array(arr) => { - let mut ors = Vec::new(); - for value in arr { - match value { - Value::String(s) => ors.push(s.clone()), - v => bail!("Invalid facet expression, expected String, found: {:?}", v), - } - } - ands.push(Either::Left(ors)); - } - v => bail!( - "Invalid facet expression, expected String or [String], found: {:?}", - v - ), - } - } - - FacetCondition::from_array(txn, index, ands) -} - -fn parse_facets( - facets: &Value, - index: &Index, - txn: &RoTxn, -) -> anyhow::Result> { - match facets { - // Disabled for now - //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), - Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), - } -} #[derive(Clone)] pub struct IndexActorHandle { sender: mpsc::Sender, @@ -276,7 +145,7 @@ impl IndexActorHandle { struct MapIndexStore { root: PathBuf, meta_store: AsyncMap, - index_store: AsyncMap>, + index_store: AsyncMap, } #[async_trait::async_trait] @@ -301,17 +170,18 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = Index::new(options, &db_path) + let index = milli::Index::new(options, &db_path) .map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); Ok(index) }).await.expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), Arc::new(index?)); + self.index_store.write().await.insert(meta.uuid.clone(), index?); Ok(meta) } - async fn get_or_create(&self, uuid: Uuid) -> Result> { + async fn get_or_create(&self, uuid: Uuid) -> Result { match self.index_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { match self.meta_store.write().await.entry(uuid.clone()) { @@ -327,7 +197,7 @@ impl IndexStore for MapIndexStore { } } - async fn get(&self, uuid: Uuid) -> Result>> { + async fn get(&self, uuid: Uuid) -> Result> { Ok(self.index_store.read().await.get(&uuid).cloned()) } } diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index 882b20963..b893b62e4 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -12,7 +12,7 @@ use super::IndexMetadata; use futures::stream::StreamExt; use actix_web::web::Payload; use super::UpdateMeta; -use crate::data::{SearchResult, SearchQuery}; +use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; pub struct IndexController { diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs index d9ac2f866..c42a532ea 100644 --- a/src/index_controller/actor_index_controller/update_handler.rs +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -7,7 +7,7 @@ use flate2::read::GzDecoder; use grenad::CompressionType; use log::info; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; -use milli::Index; +use crate::index::Index; use rayon::ThreadPool; use crate::index_controller::updates::{Failed, Processed, Processing}; @@ -225,7 +225,7 @@ impl UpdateHandler { &self, meta: Processing, content: File, - index: &Index, + index: Index, ) -> Result, Failed> { use UpdateMeta::*; @@ -244,12 +244,12 @@ impl UpdateHandler { content, update_builder, primary_key.as_deref(), - index, + &index, ), - ClearDocuments => self.clear_documents(update_builder, index), - DeleteDocuments => self.delete_documents(content, update_builder, index), - Settings(settings) => self.update_settings(settings, update_builder, index), - Facets(levels) => self.update_facets(levels, update_builder, index), + ClearDocuments => self.clear_documents(update_builder, &index), + DeleteDocuments => self.delete_documents(content, update_builder, &index), + Settings(settings) => self.update_settings(settings, update_builder, &index), + Facets(levels) => self.update_facets(levels, update_builder, &index), }; match result { diff --git a/src/lib.rs b/src/lib.rs index 65c4e8eb7..32c8e4266 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; +mod index; mod index_controller; pub use option::Opt; diff --git a/src/routes/search.rs b/src/routes/search.rs index 1f4218555..eb03cf94d 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -4,11 +4,11 @@ use std::convert::{TryFrom, TryInto}; use actix_web::{get, post, web, HttpResponse}; use serde::Deserialize; -use crate::data::{SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; use crate::Data; +use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); From a56db854a215eaa7f7c39a7984602d73f1c6385e Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 11:56:32 +0100 Subject: [PATCH 15/69] refactor update handler --- src/data/mod.rs | 3 +- src/data/updates.rs | 5 +- src/index/mod.rs | 19 ++ src/index/search.rs | 245 ++++++++++++++++++ src/index/updates.rs | 229 ++++++++++++++++ .../actor_index_controller/index_actor.rs | 3 +- .../actor_index_controller/mod.rs | 9 +- .../actor_index_controller/update_actor.rs | 3 +- .../actor_index_controller/update_handler.rs | 178 +------------ src/index_controller/mod.rs | 68 +---- src/routes/settings/mod.rs | 6 +- 11 files changed, 522 insertions(+), 246 deletions(-) create mode 100644 src/index/mod.rs create mode 100644 src/index/search.rs create mode 100644 src/index/updates.rs diff --git a/src/data/mod.rs b/src/data/mod.rs index 58acb105a..79572fcf7 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexMetadata, Settings, IndexSettings}; +use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::index_controller::actor_index_controller::IndexController; +use crate::index::Settings; use crate::option::Opt; #[derive(Clone)] diff --git a/src/data/updates.rs b/src/data/updates.rs index 01f5174a2..9f3e9b3fd 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -4,10 +4,11 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; //use tokio::io::AsyncWriteExt; use actix_web::web::Payload; -use crate::index_controller::UpdateStatus; -use crate::index_controller::{Settings, IndexMetadata}; +use crate::index_controller::{UpdateStatus, IndexMetadata}; +use crate::index::Settings; use super::Data; + impl Data { pub async fn add_documents( &self, diff --git a/src/index/mod.rs b/src/index/mod.rs new file mode 100644 index 000000000..f35e6b3dd --- /dev/null +++ b/src/index/mod.rs @@ -0,0 +1,19 @@ +mod search; +mod updates; + +use std::sync::Arc; +use std::ops::Deref; + +pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; +pub use updates::{Settings, Facets, UpdateResult}; + +#[derive(Clone)] +pub struct Index(pub Arc); + +impl Deref for Index { + type Target = milli::Index; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} diff --git a/src/index/search.rs b/src/index/search.rs new file mode 100644 index 000000000..1264d49d6 --- /dev/null +++ b/src/index/search.rs @@ -0,0 +1,245 @@ +use std::time::Instant; +use std::collections::{HashSet, BTreeMap}; +use std::mem; + +use either::Either; +use anyhow::bail; +use heed::RoTxn; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use milli::{FacetCondition, facet::FacetValue}; +use serde::{Serialize, Deserialize}; +use serde_json::{Value, Map}; + +use super::Index; + +pub const DEFAULT_SEARCH_LIMIT: usize = 20; + +const fn default_search_limit() -> usize { + DEFAULT_SEARCH_LIMIT +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[allow(dead_code)] +pub struct SearchQuery { + pub q: Option, + pub offset: Option, + #[serde(default = "default_search_limit")] + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + pub crop_length: Option, + pub attributes_to_highlight: Option>, + pub filters: Option, + pub matches: Option, + pub facet_filters: Option, + pub facet_distributions: Option>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + pub hits: Vec>, + pub nb_hits: u64, + pub query: String, + pub limit: usize, + pub offset: usize, + pub processing_time_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub facet_distributions: Option>>, +} + +impl Index { + pub fn perform_search(&self, query: SearchQuery) -> anyhow::Result { + let before_search = Instant::now(); + let rtxn = self.read_txn()?; + + let mut search = self.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + if let Some(ref facets) = query.facet_filters { + if let Some(facets) = parse_facets(facets, self, &rtxn)? { + search.facet_condition(facets); + } + } + + let milli::SearchResult { + documents_ids, + found_words, + candidates, + .. + } = search.execute()?; + let mut documents = Vec::new(); + let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields_ids = self.displayed_fields_ids(&rtxn).unwrap(); + + let attributes_to_retrieve_ids = match query.attributes_to_retrieve { + Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, + Some(ref attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f)) + .collect::>() + .into(), + None => None, + }; + + let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { + (_, Some(ids)) => ids, + (Some(ids), None) => ids, + (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + + for (_id, obkv) in self.documents(&rtxn, documents_ids)? { + let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + let nb_hits = candidates.len(); + + let facet_distributions = match query.facet_distributions { + Some(ref fields) => { + let mut facet_distribution = self.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facet_distribution.facets(fields); + } + Some(facet_distribution.candidates(candidates).execute()?) + } + None => None, + }; + + let result = SearchResult { + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facet_distributions, + }; + Ok(result) + } +} + +fn parse_facets_array( + txn: &RoTxn, + index: &Index, + arr: &Vec, +) -> anyhow::Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => bail!("Invalid facet expression, expected String, found: {:?}", v), + } + } + ands.push(Either::Left(ors)); + } + v => bail!( + "Invalid facet expression, expected String or [String], found: {:?}", + v + ), + } + } + + FacetCondition::from_array(txn, &index.0, ands) +} + +pub struct Highlighter<'a, A> { + analyzer: Analyzer<'a, A>, +} + +impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { + pub fn new(stop_words: &'a fst::Set) -> Self { + let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); + + Self { analyzer } + } + + pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + match value { + Value::Null => Value::Null, + Value::Bool(boolean) => Value::Bool(boolean), + Value::Number(number) => Value::Number(number), + Value::String(old_string) => { + let mut string = String::new(); + let analyzed = self.analyzer.analyze(&old_string); + for (word, token) in analyzed.reconstruct() { + if token.is_word() { + let to_highlight = words_to_highlight.contains(token.text()); + if to_highlight { + string.push_str("") + } + string.push_str(word); + if to_highlight { + string.push_str("") + } + } else { + string.push_str(word); + } + } + Value::String(string) + } + Value::Array(values) => Value::Array( + values + .into_iter() + .map(|v| self.highlight_value(v, words_to_highlight)) + .collect(), + ), + Value::Object(object) => Value::Object( + object + .into_iter() + .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .collect(), + ), + } + } + + pub fn highlight_record( + &self, + object: &mut Map, + words_to_highlight: &HashSet, + attributes_to_highlight: &HashSet, + ) { + // TODO do we need to create a string for element that are not and needs to be highlight? + for (key, value) in object.iter_mut() { + if attributes_to_highlight.contains(key) { + let old_value = mem::take(value); + *value = self.highlight_value(old_value, words_to_highlight); + } + } + } +} + +fn parse_facets( + facets: &Value, + index: &Index, + txn: &RoTxn, +) -> anyhow::Result> { + match facets { + // Disabled for now + //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), + Value::Array(arr) => parse_facets_array(txn, index, arr), + v => bail!( + "Invalid facet expression, expected Array, found: {:?}", + v + ), + } +} diff --git a/src/index/updates.rs b/src/index/updates.rs new file mode 100644 index 000000000..d339406f7 --- /dev/null +++ b/src/index/updates.rs @@ -0,0 +1,229 @@ +use std::collections::HashMap; +use std::io; +use std::num::NonZeroUsize; + +use flate2::read::GzDecoder; +use log::info; +use milli::update::{UpdateFormat, IndexDocumentsMethod, UpdateBuilder, DocumentAdditionResult}; +use serde::{Serialize, Deserialize, de::Deserializer}; + +use super::Index; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: usize }, + Other, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub searchable_attributes: Option>>, + + #[serde(default)] + pub faceted_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub criteria: Option>>, +} + +impl Settings { + pub fn cleared() -> Self { + Self { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + faceted_attributes: Some(None), + criteria: Some(None), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + pub level_group_size: Option, + pub min_level_size: Option, +} + +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + +impl Index { + pub fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: impl io::Read, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + ) -> anyhow::Result { + info!("performing document addition"); + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + + // Set the primary key if not set already, ignore if already set. + match (self.primary_key(&wtxn)?, primary_key) { + (None, Some(ref primary_key)) => { + self.put_primary_key(&mut wtxn, primary_key)?; + } + _ => (), + } + + let mut builder = update_builder.index_documents(&mut wtxn, self); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = false; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| { + info!("update {}: {:?}", update_id, indexing_step) + }); + + info!("document addition done: {:?}", result); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, self); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, self); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder + .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, self); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()), + } + } + + pub fn delete_documents( + &self, + document_ids: impl io::Read, + update_builder: UpdateBuilder, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_reader(document_ids)?; + let mut txn = self.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, self)?; + + // We ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } +} diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/actor_index_controller/index_actor.rs index 27d206c06..96e5010db 100644 --- a/src/index_controller/actor_index_controller/index_actor.rs +++ b/src/index_controller/actor_index_controller/index_actor.rs @@ -13,7 +13,8 @@ use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}, UpdateResult as UResult}; +use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; +use crate::index::UpdateResult as UResult; use crate::option::IndexerOpts; use crate::index::{Index, SearchQuery, SearchResult}; diff --git a/src/index_controller/actor_index_controller/mod.rs b/src/index_controller/actor_index_controller/mod.rs index b893b62e4..188d85580 100644 --- a/src/index_controller/actor_index_controller/mod.rs +++ b/src/index_controller/actor_index_controller/mod.rs @@ -15,6 +15,9 @@ use super::UpdateMeta; use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; +use crate::index::Settings; +use super::UpdateStatus; + pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, @@ -69,7 +72,7 @@ impl IndexController { Ok(status) } - fn clear_documents(&self, index: String) -> anyhow::Result { + fn clear_documents(&self, index: String) -> anyhow::Result { todo!() } @@ -77,7 +80,7 @@ impl IndexController { todo!() } - fn update_settings(&self, index_uid: String, settings: super::Settings) -> anyhow::Result { + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { todo!() } @@ -100,7 +103,7 @@ impl IndexController { todo!() } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { todo!() } diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/actor_index_controller/update_actor.rs index e82f01092..384c52098 100644 --- a/src/index_controller/actor_index_controller/update_actor.rs +++ b/src/index_controller/actor_index_controller/update_actor.rs @@ -10,7 +10,8 @@ use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; -use crate::index_controller::{UpdateMeta, UpdateStatus, UpdateResult}; +use crate::index_controller::{UpdateMeta, UpdateStatus}; +use crate::index::UpdateResult; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/actor_index_controller/update_handler.rs index c42a532ea..766dfc5f0 100644 --- a/src/index_controller/actor_index_controller/update_handler.rs +++ b/src/index_controller/actor_index_controller/update_handler.rs @@ -1,17 +1,14 @@ -use std::collections::HashMap; -use std::io; use std::fs::File; use anyhow::Result; -use flate2::read::GzDecoder; use grenad::CompressionType; -use log::info; -use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use milli::update::UpdateBuilder; use crate::index::Index; use rayon::ThreadPool; use crate::index_controller::updates::{Failed, Processed, Processing}; -use crate::index_controller::{Facets, Settings, UpdateMeta, UpdateResult}; +use crate::index_controller::UpdateMeta; +use crate::index::UpdateResult; use crate::option::IndexerOpts; pub struct UpdateHandler { @@ -62,164 +59,6 @@ impl UpdateHandler { update_builder } - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: File, - update_builder: UpdateBuilder, - primary_key: Option<&str>, - index: &Index, - ) -> anyhow::Result { - info!("performing document addition"); - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - - // Set the primary key if not set already, ignore if already set. - match (index.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - index.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), - } - - let mut builder = update_builder.index_documents(&mut wtxn, index); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = false; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| { - info!("update {}: {:?}", update_id, indexing_step) - }); - - info!("document addition done: {:?}", result); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder, index: &Index) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, index); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_settings( - &self, - settings: &Settings, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, index); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder - .execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn update_facets( - &self, - levels: &Facets, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - // We must use the write transaction of the update here. - let mut wtxn = index.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, index); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()), - } - } - - fn delete_documents( - &self, - document_ids: File, - update_builder: UpdateBuilder, - index: &Index, - ) -> anyhow::Result { - let ids: Vec = serde_json::from_reader(document_ids)?; - let mut txn = index.write_txn()?; - let mut builder = update_builder.delete_documents(&mut txn, index)?; - - // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); - - match builder.execute() { - Ok(deleted) => txn - .commit() - .and(Ok(UpdateResult::DocumentDeletion { deleted })) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } pub fn handle_update( &self, @@ -238,18 +77,17 @@ impl UpdateHandler { method, format, primary_key, - } => self.update_documents( + } => index.update_documents( *format, *method, content, update_builder, primary_key.as_deref(), - &index, ), - ClearDocuments => self.clear_documents(update_builder, &index), - DeleteDocuments => self.delete_documents(content, update_builder, &index), - Settings(settings) => self.update_settings(settings, update_builder, &index), - Facets(levels) => self.update_facets(levels, update_builder, &index), + ClearDocuments => index.clear_documents(update_builder), + DeleteDocuments => index.delete_documents(content, update_builder), + Settings(settings) => index.update_settings(settings, update_builder), + Facets(levels) => index.update_facets(levels, update_builder), }; match result { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index fbe7a161d..16f884137 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,16 +1,13 @@ pub mod actor_index_controller; mod updates; -use std::collections::HashMap; -use std::num::NonZeroUsize; - -use anyhow::Result; use chrono::{DateTime, Utc}; -use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; -use serde::{Serialize, Deserialize, de::Deserializer}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use serde::{Serialize, Deserialize}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; +use crate::index::{UpdateResult, Settings, Facets}; pub type UpdateStatus = updates::UpdateStatus; @@ -37,66 +34,7 @@ pub enum UpdateMeta { Facets(Facets), } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Facets { - pub level_group_size: Option, - pub min_level_size: Option, -} -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> -{ - Deserialize::deserialize(deserializer).map(Some) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Settings { - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub displayed_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub searchable_attributes: Option>>, - - #[serde(default)] - pub faceted_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub criteria: Option>>, -} - -impl Settings { - pub fn cleared() -> Self { - Self { - displayed_attributes: Some(None), - searchable_attributes: Some(None), - faceted_attributes: Some(None), - criteria: Some(None), - } - } -} -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: usize }, - Other, -} #[derive(Clone, Debug)] pub struct IndexSettings { diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index 00bc4220e..93e10674d 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -2,7 +2,7 @@ use actix_web::{web, HttpResponse, delete, get, post}; use crate::Data; use crate::error::ResponseError; -use crate::index_controller::Settings; +use crate::index::Settings; use crate::helpers::Authentication; #[macro_export] @@ -14,14 +14,14 @@ macro_rules! make_setting_route { use crate::data; use crate::error::ResponseError; use crate::helpers::Authentication; - use crate::index_controller::Settings; + use crate::index::Settings; #[actix_web::delete($route, wrap = "Authentication::Private")] pub async fn delete( data: web::Data, index_uid: web::Path, ) -> Result { - use crate::index_controller::Settings; + use crate::index::Settings; let settings = Settings { $attr: Some(None), ..Default::default() From 8432c8584a6fee1a555454693f0e9e5741d61d69 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:03:06 +0100 Subject: [PATCH 16/69] refactor index controller --- src/data/mod.rs | 2 +- .../index_actor.rs | 0 src/index_controller/mod.rs | 124 +++++++++++++++++- .../update_actor.rs | 0 .../update_handler.rs | 0 .../update_store.rs | 0 .../uuid_resolver.rs | 0 7 files changed, 122 insertions(+), 4 deletions(-) rename src/index_controller/{actor_index_controller => }/index_actor.rs (100%) rename src/index_controller/{actor_index_controller => }/update_actor.rs (100%) rename src/index_controller/{actor_index_controller => }/update_handler.rs (100%) rename src/index_controller/{actor_index_controller => }/update_store.rs (100%) rename src/index_controller/{actor_index_controller => }/uuid_resolver.rs (100%) diff --git a/src/data/mod.rs b/src/data/mod.rs index 79572fcf7..76effde58 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use sha2::Digest; use crate::index_controller::{IndexMetadata, IndexSettings}; -use crate::index_controller::actor_index_controller::IndexController; +use crate::index_controller::IndexController; use crate::index::Settings; use crate::option::Opt; diff --git a/src/index_controller/actor_index_controller/index_actor.rs b/src/index_controller/index_actor.rs similarity index 100% rename from src/index_controller/actor_index_controller/index_actor.rs rename to src/index_controller/index_actor.rs diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 16f884137..54199c1d7 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,6 +1,17 @@ -pub mod actor_index_controller; mod updates; +mod index_actor; +mod update_actor; +mod uuid_resolver; +mod update_store; +mod update_handler; +use std::path::Path; + +use tokio::sync::{mpsc, oneshot}; +use futures::stream::StreamExt; +use actix_web::web::Payload; +use crate::index::{SearchResult, SearchQuery}; +use actix_web::web::Bytes; use chrono::{DateTime, Utc}; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -34,10 +45,117 @@ pub enum UpdateMeta { Facets(Facets), } - - #[derive(Clone, Debug)] pub struct IndexSettings { pub name: Option, pub primary_key: Option, } + + +pub struct IndexController { + uuid_resolver: uuid_resolver::UuidResolverHandle, + index_handle: index_actor::IndexActorHandle, + update_handle: update_actor::UpdateActorHandle, +} + +enum IndexControllerMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Shutdown, +} + +impl IndexController { + pub fn new(path: impl AsRef) -> Self { + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let index_actor = index_actor::IndexActorHandle::new(&path); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + Self { uuid_resolver, index_handle: index_actor, update_handle } + } + + pub async fn add_documents( + &self, + index: String, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + mut payload: Payload, + primary_key: Option, + ) -> anyhow::Result { + let uuid = self.uuid_resolver.get_or_create(index).await?; + let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let (sender, receiver) = mpsc::channel(10); + + // It is necessary to spawn a local task to senf the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + while let Some(bytes) = payload.next().await { + match bytes { + Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Err(e) => { + let error: Box = Box::new(e); + sender.send(Err(error)).await; }, + } + } + }); + + // This must be done *AFTER* spawning the task. + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) + } + + fn clear_documents(&self, index: String) -> anyhow::Result { + todo!() + } + + fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + todo!() + } + + fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { + todo!() + } + + pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { + let IndexSettings { name, primary_key } = index_settings; + let uuid = self.uuid_resolver.create(name.unwrap()).await?; + let index_meta = self.index_handle.create_index(uuid, primary_key).await?; + Ok(index_meta) + } + + fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { + todo!() + } + + pub fn index(&self, name: String) -> anyhow::Result>> { + todo!() + } + + fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + todo!() + } + + fn all_update_status(&self, index: String) -> anyhow::Result> { + todo!() + } + + pub fn list_indexes(&self) -> anyhow::Result> { + todo!() + } + + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { + todo!() + } + + pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let result = self.index_handle.search(uuid, query).await?; + Ok(result) + } +} diff --git a/src/index_controller/actor_index_controller/update_actor.rs b/src/index_controller/update_actor.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_actor.rs rename to src/index_controller/update_actor.rs diff --git a/src/index_controller/actor_index_controller/update_handler.rs b/src/index_controller/update_handler.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_handler.rs rename to src/index_controller/update_handler.rs diff --git a/src/index_controller/actor_index_controller/update_store.rs b/src/index_controller/update_store.rs similarity index 100% rename from src/index_controller/actor_index_controller/update_store.rs rename to src/index_controller/update_store.rs diff --git a/src/index_controller/actor_index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs similarity index 100% rename from src/index_controller/actor_index_controller/uuid_resolver.rs rename to src/index_controller/uuid_resolver.rs From 47138c763290fe5274985358c365eac7c622cb63 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:20:14 +0100 Subject: [PATCH 17/69] update settings --- src/data/updates.rs | 10 ++++------ src/index_controller/mod.rs | 10 ++++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 9f3e9b3fd..92bba693c 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -25,13 +25,11 @@ impl Data { pub async fn update_settings( &self, - _index: impl AsRef + Send + Sync + 'static, - _settings: Settings + index: String, + settings: Settings ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; - //Ok(update.into()) + let update = self.index_controller.update_settings(index, settings).await?; + Ok(update.into()) } pub async fn clear_documents( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 54199c1d7..dd2dbd7cf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -114,8 +114,14 @@ impl IndexController { todo!() } - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - todo!() + pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { + let uuid = self.uuid_resolver.get_or_create(index_uid).await?; + let meta = UpdateMeta::Settings(settings); + // Nothing so send, drop the sender right away, as not to block the update actor. + let (_, receiver) = mpsc::channel(1); + + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { From 17b84691f2d3c10354915ac889798baab9e28e93 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 12:38:55 +0100 Subject: [PATCH 18/69] list settings --- src/data/mod.rs | 31 ++--------------------------- src/index/mod.rs | 29 +++++++++++++++++++++++++++ src/index_controller/index_actor.rs | 27 +++++++++++++++++++++---- src/index_controller/mod.rs | 26 +++++++++++++----------- src/routes/settings/mod.rs | 4 ++-- 5 files changed, 70 insertions(+), 47 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 76effde58..1dec00766 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -78,35 +78,8 @@ impl Data { Ok(Data { inner }) } - pub fn settings>(&self, index_uid: S) -> anyhow::Result { - let index = self.index_controller - .index(index_uid.as_ref().to_string())? - .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; - - let txn = index.read_txn()?; - - let displayed_attributes = index - .displayed_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); - - let searchable_attributes = index - .searchable_fields(&txn)? - .map(|fields| fields.into_iter().map(String::from).collect()) - .unwrap_or_else(|| vec!["*".to_string()]); - - let faceted_attributes = index - .faceted_fields(&txn)? - .into_iter() - .map(|(k, v)| (k, v.to_string())) - .collect(); - - Ok(Settings { - displayed_attributes: Some(Some(displayed_attributes)), - searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(Some(faceted_attributes)), - criteria: None, - }) + pub async fn settings>(&self, index_uid: S) -> anyhow::Result { + self.index_controller.settings(index_uid.as_ref().to_string()).await } pub fn list_indexes(&self) -> anyhow::Result> { diff --git a/src/index/mod.rs b/src/index/mod.rs index f35e6b3dd..a68f983e9 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -17,3 +17,32 @@ impl Deref for Index { self.0.as_ref() } } + +impl Index { + pub fn settings(&self) -> anyhow::Result { + let txn = self.read_txn()?; + + let displayed_attributes = self + .displayed_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()) + .unwrap_or_else(|| vec!["*".to_string()]); + + let searchable_attributes = self + .searchable_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()) + .unwrap_or_else(|| vec!["*".to_string()]); + + let faceted_attributes = self + .faceted_fields(&txn)? + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(); + + Ok(Settings { + displayed_attributes: Some(Some(displayed_attributes)), + searchable_attributes: Some(Some(searchable_attributes)), + faceted_attributes: Some(Some(faceted_attributes)), + criteria: None, + }) + } +} diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 96e5010db..655a1e2d5 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -16,7 +16,7 @@ use super::update_handler::UpdateHandler; use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; use crate::index::UpdateResult as UResult; use crate::option::IndexerOpts; -use crate::index::{Index, SearchQuery, SearchResult}; +use crate::index::{Index, SearchQuery, SearchResult, Settings}; pub type Result = std::result::Result; type AsyncMap = Arc>>; @@ -26,6 +26,7 @@ enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, + Settings { uuid: Uuid, ret: oneshot::Sender> }, } struct IndexActor { @@ -75,6 +76,7 @@ impl IndexActor { IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + IndexMsg::Settings { ret, uuid } => self.handle_settings(uuid, ret).await, } }); @@ -100,9 +102,19 @@ impl IndexActor { let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - let result = tokio::task::spawn_blocking(move || update_handler.handle_update(meta, data, index)).await; - let result = result.unwrap(); - let _ = ret.send(result); + tokio::task::spawn_blocking(move || { + let result = update_handler.handle_update(meta, data, index); + let _ = ret.send(result); + }).await; + } + + async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.settings() + .map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }).await; } } @@ -141,6 +153,13 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn settings(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Settings { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dd2dbd7cf..f1bc922d6 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -7,14 +7,15 @@ mod update_handler; use std::path::Path; -use tokio::sync::{mpsc, oneshot}; -use futures::stream::StreamExt; -use actix_web::web::Payload; -use crate::index::{SearchResult, SearchQuery}; use actix_web::web::Bytes; +use actix_web::web::Payload; +use anyhow::Context; use chrono::{DateTime, Utc}; +use crate::index::{SearchResult, SearchQuery}; +use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -135,14 +136,6 @@ impl IndexController { todo!() } - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { - todo!() - } - - pub fn index(&self, name: String) -> anyhow::Result>> { - todo!() - } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { todo!() } @@ -155,6 +148,15 @@ impl IndexController { todo!() } + pub async fn settings(&self, index: String) -> anyhow::Result { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let settings = self.index_handle.settings(uuid).await?; + Ok(settings) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index 93e10674d..0f904f572 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -64,7 +64,7 @@ macro_rules! make_setting_route { data: actix_web::web::Data, index_uid: actix_web::web::Path, ) -> std::result::Result { - match data.settings(index_uid.as_ref()) { + match data.settings(index_uid.as_ref()).await { Ok(settings) => { let setting = settings.$attr; let json = serde_json::to_string(&setting).unwrap(); @@ -153,7 +153,7 @@ async fn get_all( data: web::Data, index_uid: web::Path, ) -> Result { - match data.settings(index_uid.as_ref()) { + match data.settings(index_uid.as_ref()).await { Ok(settings) => { let json = serde_json::to_string(&settings).unwrap(); Ok(HttpResponse::Ok().body(json)) From f3d65ec5e92f6a3307c6eae0a2eb488e2ade9d4c Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 14:20:19 +0100 Subject: [PATCH 19/69] implement retrieve documents --- src/data/search.rs | 53 ++------ src/index/mod.rs | 39 ++++++ src/index_controller/index_actor.rs | 187 ++++++++++++++++++++++------ src/index_controller/mod.rs | 17 ++- src/routes/document.rs | 23 ++-- 5 files changed, 225 insertions(+), 94 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index d6bf8438e..319f8b973 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -12,59 +12,22 @@ impl Data { self.index_controller.search(index.as_ref().to_string(), search_query).await } - pub async fn retrieve_documents( + pub async fn retrieve_documents( &self, - _index: String, - _offset: usize, - _limit: usize, - _attributes_to_retrieve: Option>, - ) -> anyhow::Result>> - where - S: AsRef + Send + Sync + 'static, - { - todo!() - //let index_controller = self.index_controller.clone(); - //let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - //let index = index_controller - //.index(index.clone())? - //.with_context(|| format!("Index {:?} doesn't exist", index))?; - - //let txn = index.read_txn()?; - - //let fields_ids_map = index.fields_ids_map(&txn)?; - - //let attributes_to_retrieve_ids = match attributes_to_retrieve { - //Some(attrs) => attrs - //.iter() - //.filter_map(|f| fields_ids_map.id(f.as_ref())) - //.collect::>(), - //None => fields_ids_map.iter().map(|(id, _)| id).collect(), - //}; - - //let iter = index.documents.range(&txn, &(..))?.skip(offset).take(limit); - - //let mut documents = Vec::new(); - - //for entry in iter { - //let (_id, obkv) = entry?; - //let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; - //documents.push(object); - //} - - //Ok(documents) - //}) - //.await?; - //documents + index: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result>> { + self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await } pub async fn retrieve_document( &self, _index: impl AsRef + Sync + Send + 'static, _document_id: impl AsRef + Sync + Send + 'static, - _attributes_to_retrieve: Option>, + _attributes_to_retrieve: Option>, ) -> anyhow::Result> - where - S: AsRef + Sync + Send + 'static, { todo!() //let index_controller = self.index_controller.clone(); diff --git a/src/index/mod.rs b/src/index/mod.rs index a68f983e9..8ae2ba6a2 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -4,9 +4,14 @@ mod updates; use std::sync::Arc; use std::ops::Deref; +use serde_json::{Value, Map}; +use milli::obkv_to_json; + pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Settings, Facets, UpdateResult}; +pub type Document = Map; + #[derive(Clone)] pub struct Index(pub Arc); @@ -45,4 +50,38 @@ impl Index { criteria: None, }) } + + pub fn retrieve_documents( + &self, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result>> + where + S: AsRef + Send + Sync + 'static, + { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); + + let mut documents = Vec::new(); + + for entry in iter { + let (_id, obkv) = entry?; + let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + documents.push(object); + } + + Ok(documents) + } } diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 655a1e2d5..8e59227ac 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,6 +1,6 @@ -use std::collections::{HashMap, hash_map::Entry}; -use std::fs::{File, create_dir_all}; -use std::path::{PathBuf, Path}; +use std::collections::{hash_map::Entry, HashMap}; +use std::fs::{create_dir_all, File}; +use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; @@ -8,25 +8,51 @@ use chrono::Utc; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use serde_json::{Map, Value}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; use super::update_handler::UpdateHandler; -use crate::index_controller::{IndexMetadata, UpdateMeta, updates::{Processed, Failed, Processing}}; use crate::index::UpdateResult as UResult; +use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{ + updates::{Failed, Processed, Processing}, + IndexMetadata, UpdateMeta, +}; use crate::option::IndexerOpts; -use crate::index::{Index, SearchQuery, SearchResult, Settings}; pub type Result = std::result::Result; type AsyncMap = Arc>>; type UpdateResult = std::result::Result, Failed>; enum IndexMsg { - CreateIndex { uuid: Uuid, primary_key: Option, ret: oneshot::Sender> }, - Update { meta: Processing, data: std::fs::File, ret: oneshot::Sender}, - Search { uuid: Uuid, query: SearchQuery, ret: oneshot::Sender> }, - Settings { uuid: Uuid, ret: oneshot::Sender> }, + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Update { + meta: Processing, + data: std::fs::File, + ret: oneshot::Sender, + }, + Search { + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + }, + Settings { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Documents { + uuid: Uuid, + attributes_to_retrieve: Option>, + offset: usize, + limit: usize, + ret: oneshot::Sender>>>, + }, } struct IndexActor { @@ -56,11 +82,20 @@ impl IndexActor { let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); let inbox = Some(inbox); - Self { inbox, store, update_handler } + Self { + inbox, + store, + update_handler, + } } async fn run(mut self) { - let mut inbox = self.inbox.take().expect("Index Actor must have a inbox at this point."); + use IndexMsg::*; + + let mut inbox = self + .inbox + .take() + .expect("Index Actor must have a inbox at this point."); let stream = stream! { loop { @@ -73,31 +108,59 @@ impl IndexActor { let fut = stream.for_each_concurrent(Some(10), |msg| async { match msg { - IndexMsg::CreateIndex { uuid, primary_key, ret } => self.handle_create_index(uuid, primary_key, ret).await, - IndexMsg::Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - IndexMsg::Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - IndexMsg::Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + CreateIndex { + uuid, + primary_key, + ret, + } => self.handle_create_index(uuid, primary_key, ret).await, + Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) + .await + } } }); fut.await; } - async fn handle_search(&self, uuid: Uuid, query: SearchQuery, ret: oneshot::Sender>) { + async fn handle_search( + &self, + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { let result = index.perform_search(query); ret.send(result) }); - } - async fn handle_create_index(&self, uuid: Uuid, primary_key: Option, ret: oneshot::Sender>) { + async fn handle_create_index( + &self, + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + ) { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); } - async fn handle_update(&self, meta: Processing, data: File, ret: oneshot::Sender) { + async fn handle_update( + &self, + meta: Processing, + data: File, + ret: oneshot::Sender, + ) { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let index = self.store.get_or_create(uuid).await.unwrap(); @@ -105,13 +168,30 @@ impl IndexActor { tokio::task::spawn_blocking(move || { let result = update_handler.handle_update(meta, data, index); let _ = ret.send(result); - }).await; + }) + .await; } async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.settings() + let result = index.settings().map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }) + .await; + } + + async fn handle_fetch_documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ret: oneshot::Sender>>, + ) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); }).await; @@ -133,9 +213,17 @@ impl IndexActorHandle { Self { sender } } - pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + pub async fn create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { ret, uuid, primary_key }; + let msg = IndexMsg::CreateIndex { + ret, + uuid, + primary_key, + }; let _ = self.sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } @@ -160,6 +248,25 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Documents { + uuid, + ret, + offset, + attributes_to_retrieve, + limit, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -190,29 +297,31 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) - }).await.expect("thread died"); + }) + .await + .expect("thread died"); - self.index_store.write().await.insert(meta.uuid.clone(), index?); + self.index_store + .write() + .await + .insert(meta.uuid.clone(), index?); Ok(meta) } async fn get_or_create(&self, uuid: Uuid) -> Result { match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => { - match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(_) => { - todo!() - } - Entry::Occupied(entry) => { - todo!() - } + Entry::Vacant(entry) => match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(_) => { + todo!() } - } + Entry::Occupied(entry) => { + todo!() + } + }, Entry::Occupied(entry) => Ok(entry.get().clone()), } } @@ -228,6 +337,10 @@ impl MapIndexStore { root.push("indexes/"); let meta_store = Arc::new(RwLock::new(HashMap::new())); let index_store = Arc::new(RwLock::new(HashMap::new())); - Self { meta_store, index_store, root } + Self { + meta_store, + index_store, + root, + } } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index f1bc922d6..47ce04b96 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -11,7 +11,6 @@ use actix_web::web::Bytes; use actix_web::web::Payload; use anyhow::Context; use chrono::{DateTime, Utc}; -use crate::index::{SearchResult, SearchQuery}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -19,6 +18,7 @@ use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; +use crate::index::{SearchResult, SearchQuery, Document}; use crate::index::{UpdateResult, Settings, Facets}; pub type UpdateStatus = updates::UpdateStatus; @@ -157,6 +157,21 @@ impl IndexController { Ok(settings) } + pub async fn documents( + &self, + index: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; + Ok(documents) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/document.rs b/src/routes/document.rs index 00d037359..af9efc701 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -55,17 +55,18 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - let index = path.index_uid.clone(); - let id = path.document_id.clone(); - match data.retrieve_document(index, id, None as Option>).await { - Ok(document) => { - let json = serde_json::to_string(&document).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } - } + todo!() + //let index = path.index_uid.clone(); + //let id = path.document_id.clone(); + //match data.retrieve_document(index, id, None as Option>).await { + //Ok(document) => { + //let json = serde_json::to_string(&document).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + //} + //} } #[delete( From 581dcd5735ed6374edeed3e27a2eb2472d01e991 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:09:00 +0100 Subject: [PATCH 20/69] implement retrieve one document --- src/data/search.rs | 49 +++-------------------------- src/index/mod.rs | 39 +++++++++++++++++++++++ src/index_controller/index_actor.rs | 42 +++++++++++++++++++++++-- src/index_controller/mod.rs | 14 +++++++++ src/routes/document.rs | 23 +++++++------- 5 files changed, 109 insertions(+), 58 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 319f8b973..753083b7e 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -22,52 +22,13 @@ impl Data { self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await } - pub async fn retrieve_document( + pub async fn retrieve_document( &self, - _index: impl AsRef + Sync + Send + 'static, - _document_id: impl AsRef + Sync + Send + 'static, - _attributes_to_retrieve: Option>, + index: impl AsRef + Sync + Send + 'static, + document_id: impl AsRef + Sync + Send + 'static, + attributes_to_retrieve: Option>, ) -> anyhow::Result> { - todo!() - //let index_controller = self.index_controller.clone(); - //let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { - //let index = index_controller - //.index(&index)? - //.with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - //let txn = index.read_txn()?; - - //let fields_ids_map = index.fields_ids_map(&txn)?; - - //let attributes_to_retrieve_ids = match attributes_to_retrieve { - //Some(attrs) => attrs - //.iter() - //.filter_map(|f| fields_ids_map.id(f.as_ref())) - //.collect::>(), - //None => fields_ids_map.iter().map(|(id, _)| id).collect(), - //}; - - //let internal_id = index - //.external_documents_ids(&txn)? - //.get(document_id.as_ref().as_bytes()) - //.with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; - - //let document = index - //.documents(&txn, std::iter::once(internal_id))? - //.into_iter() - //.next() - //.map(|(_, d)| d); - - //match document { - //Some(document) => Ok(obkv_to_json( - //&attributes_to_retrieve_ids, - //&fields_ids_map, - //document, - //)?), - //None => bail!("Document with id {} not found", document_id.as_ref()), - //} - //}) - //.await?; - //document + self.index_controller.document(index.as_ref().to_string(), document_id.as_ref().to_string(), attributes_to_retrieve).await } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 8ae2ba6a2..c50c2873c 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -4,6 +4,7 @@ mod updates; use std::sync::Arc; use std::ops::Deref; +use anyhow::{bail, Context}; use serde_json::{Value, Map}; use milli::obkv_to_json; @@ -84,4 +85,42 @@ impl Index { Ok(documents) } + + pub fn retrieve_document>( + &self, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> anyhow::Result> { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let internal_id = self + .external_documents_ids(&txn)? + .get(doc_id.as_bytes()) + .with_context(|| format!("Document with id {} not found", doc_id))?; + + let document = self + .documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); + + match document { + Some(document) => Ok(obkv_to_json( + &attributes_to_retrieve_ids, + &fields_ids_map, + document, + )?), + None => bail!("Document with id {} not found", doc_id), + } + } } diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 8e59227ac..59f000575 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -8,7 +8,6 @@ use chrono::Utc; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; -use serde_json::{Map, Value}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; @@ -51,8 +50,14 @@ enum IndexMsg { attributes_to_retrieve: Option>, offset: usize, limit: usize, - ret: oneshot::Sender>>>, + ret: oneshot::Sender>>, }, + Document { + uuid: Uuid, + attributes_to_retrieve: Option>, + doc_id: String, + ret: oneshot::Sender>, + } } struct IndexActor { @@ -126,6 +131,7 @@ impl IndexActor { self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) .await } + Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, } }); @@ -196,6 +202,21 @@ impl IndexActor { let _ = ret.send(result); }).await; } + + async fn handle_fetch_document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ret: oneshot::Sender>, + ) { + let index = self.store.get(uuid).await.unwrap().unwrap(); + tokio::task::spawn_blocking(move || { + let result = index.retrieve_document(doc_id, attributes_to_retrieve) + .map_err(|e| IndexError::Error(e)); + let _ = ret.send(result); + }).await; + } } #[derive(Clone)] @@ -267,6 +288,23 @@ impl IndexActorHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Document { + uuid, + ret, + doc_id, + attributes_to_retrieve, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 47ce04b96..dabf1b936 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -172,6 +172,20 @@ impl IndexController { Ok(documents) } + pub async fn document( + &self, + index: String, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> anyhow::Result { + let uuid = self.uuid_resolver + .resolve(index.clone()) + .await? + .with_context(|| format!("Index {:?} doesn't exist", index))?; + let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; + Ok(document) + } + fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } diff --git a/src/routes/document.rs b/src/routes/document.rs index af9efc701..00d037359 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -55,18 +55,17 @@ async fn get_document( data: web::Data, path: web::Path, ) -> Result { - todo!() - //let index = path.index_uid.clone(); - //let id = path.document_id.clone(); - //match data.retrieve_document(index, id, None as Option>).await { - //Ok(document) => { - //let json = serde_json::to_string(&document).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + let index = path.index_uid.clone(); + let id = path.document_id.clone(); + match data.retrieve_document(index, id, None as Option>).await { + Ok(document) => { + let json = serde_json::to_string(&document).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[delete( From 181eaf95f5db0e928308bb36a2777e3198adb070 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:10:58 +0100 Subject: [PATCH 21/69] restore update documents --- src/routes/document.rs | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/routes/document.rs b/src/routes/document.rs index 00d037359..965b669d9 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -199,26 +199,25 @@ async fn update_documents( params: web::Query, body: web::Payload, ) -> Result { - todo!() - //let addition_result = data - //.add_documents( - //path.into_inner().index_uid, - //IndexDocumentsMethod::UpdateDocuments, - //UpdateFormat::Json, - //body, - //params.primary_key.clone(), - //).await; + let addition_result = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::UpdateDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ).await; - //match addition_result { - //Ok(update) => { - //let value = serde_json::to_string(&update).unwrap(); - //let response = HttpResponse::Ok().body(value); - //Ok(response) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match addition_result { + Ok(update) => { + let value = serde_json::to_string(&update).unwrap(); + let response = HttpResponse::Ok().body(value); + Ok(response) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[post( From ae5581d37c6d8e1ed025fc3935179f49de7e8dfc Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 15:59:18 +0100 Subject: [PATCH 22/69] implement delete documents --- src/data/updates.rs | 10 ++++------ src/index_controller/mod.rs | 18 ++++++++++++++---- src/routes/document.rs | 27 +++++++++++++-------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 92bba693c..775734ac1 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -44,13 +44,11 @@ impl Data { pub async fn delete_documents( &self, - _index: impl AsRef + Sync + Send + 'static, - _document_ids: Vec, + index: impl AsRef + Sync + Send + 'static, + document_ids: Vec, ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; - //Ok(update.into()) + let update = self.index_controller.delete_documents(index.as_ref().to_string(), document_ids).await?; + Ok(update.into()) } pub async fn delete_index( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index dabf1b936..5f00c2540 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -7,8 +7,7 @@ mod update_handler; use std::path::Path; -use actix_web::web::Bytes; -use actix_web::web::Payload; +use actix_web::web::{Bytes, Payload}; use anyhow::Context; use chrono::{DateTime, Utc}; use futures::stream::StreamExt; @@ -111,8 +110,19 @@ impl IndexController { todo!() } - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - todo!() + pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let meta = UpdateMeta::DeleteDocuments; + let (sender, receiver) = mpsc::channel(10); + + tokio::task::spawn(async move { + let json = serde_json::to_vec(&document_ids).unwrap(); + let bytes = Bytes::from(json); + let _ = sender.send(Ok(bytes)).await; + }); + + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { diff --git a/src/routes/document.rs b/src/routes/document.rs index 965b669d9..a848ddcc4 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -229,21 +229,20 @@ async fn delete_documents( path: web::Path, body: web::Json>, ) -> Result { - todo!() - //let ids = body - //.iter() - //.map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) - //.collect(); + let ids = body + .iter() + .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + .collect(); - //match data.delete_documents(path.index_uid.clone(), ids).await { - //Ok(result) => { - //let json = serde_json::to_string(&result).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match data.delete_documents(path.index_uid.clone(), ids).await { + Ok(result) => { + let json = serde_json::to_string(&result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] From a955e04ab6c0b48821f01011d88409e0214a97c4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 16:04:12 +0100 Subject: [PATCH 23/69] implement clear documents --- src/data/updates.rs | 8 +++----- src/index_controller/mod.rs | 8 ++++++-- src/routes/document.rs | 19 +++++++++---------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 775734ac1..a712f4cc6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -34,12 +34,10 @@ impl Data { pub async fn clear_documents( &self, - _index: impl AsRef + Sync + Send + 'static, + index: impl AsRef + Sync + Send + 'static, ) -> anyhow::Result { - todo!() - //let index_controller = self.index_controller.clone(); - //let update = tokio::task::spawn_blocking(move || index_controller.clear_documents(index)).await??; - //Ok(update.into()) + let update = self.index_controller.clear_documents(index.as_ref().to_string()).await?; + Ok(update) } pub async fn delete_documents( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 5f00c2540..d790d1d58 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -106,8 +106,12 @@ impl IndexController { Ok(status) } - fn clear_documents(&self, index: String) -> anyhow::Result { - todo!() + pub async fn clear_documents(&self, index: String) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let meta = UpdateMeta::ClearDocuments; + let (_, receiver) = mpsc::channel(1); + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) } pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { diff --git a/src/routes/document.rs b/src/routes/document.rs index a848ddcc4..43d88e4b9 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -250,14 +250,13 @@ async fn clear_all_documents( data: web::Data, path: web::Path, ) -> Result { - todo!() - //match data.clear_documents(path.index_uid.clone()).await { - //Ok(update) => { - //let json = serde_json::to_string(&update).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - //} - //} + match data.clear_documents(path.index_uid.clone()).await { + Ok(update) => { + let json = serde_json::to_string(&update).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + } + } } From 6a0a9fec6b5e5bad8cdbaa0fcc52115536ed4869 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 17:25:02 +0100 Subject: [PATCH 24/69] async update store --- src/index_controller/update_actor.rs | 18 ++-- src/index_controller/update_store.rs | 138 +++++++++++++++++---------- 2 files changed, 99 insertions(+), 57 deletions(-) diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 384c52098..2f3c058bc 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -24,15 +24,15 @@ pub enum UpdateError { } enum UpdateMsg { - CreateIndex{ - uuid: Uuid, - ret: oneshot::Sender>, - }, Update { uuid: Uuid, meta: UpdateMeta, data: mpsc::Receiver>, ret: oneshot::Sender> + }, + ListUpdates { + uuid: Uuid, + ret: oneshot::Sender>>, } } @@ -58,12 +58,14 @@ where D: AsRef<[u8]> + Sized + 'static, } async fn run(mut self) { + use UpdateMsg::*; + info!("started update actor."); loop { match self.inbox.recv().await { - Some(UpdateMsg::Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, - Some(_) => {} + Some(Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, + Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, None => {} } } @@ -99,6 +101,10 @@ where D: AsRef<[u8]> + Sized + 'static, let _ = ret.send(result); }).await; } + + async fn handle_list_updates(&self, uuid: Uuid, ret: oneshot::Sender>>) { + todo!() + } } #[derive(Clone)] diff --git a/src/index_controller/update_store.rs b/src/index_controller/update_store.rs index ae4bfb8d8..02c15ed8c 100644 --- a/src/index_controller/update_store.rs +++ b/src/index_controller/update_store.rs @@ -1,12 +1,12 @@ +use std::fs::remove_file; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::fs::remove_file; -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; +use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; +use heed::{Database, Env, EnvOpenOptions}; +use serde::{Deserialize, Serialize}; use std::fs::File; +use tokio::sync::mpsc; use uuid::Uuid; use crate::index_controller::updates::*; @@ -22,17 +22,26 @@ pub struct UpdateStore { failed_meta: Database, SerdeJson>>, aborted_meta: Database, SerdeJson>>, processing: Arc>>>, - notification_sender: Sender<()>, + notification_sender: mpsc::Sender<()>, } pub trait HandleUpdate { - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed>; + fn handle_update( + &mut self, + meta: Processing, + content: File, + ) -> Result, Failed>; } impl HandleUpdate for F -where F: FnMut(Processing, File) -> Result, Failed> +where + F: FnMut(Processing, File) -> Result, Failed>, { - fn handle_update(&mut self, meta: Processing, content: File) -> Result, Failed> { + fn handle_update( + &mut self, + meta: Processing, + content: File, + ) -> Result, Failed> { self(meta, content) } } @@ -46,11 +55,11 @@ where pub fn open( mut options: EnvOpenOptions, path: P, - mut update_handler: U, + update_handler: U, ) -> heed::Result> where P: AsRef, - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Sync + Clone + Send + 'static, { options.max_dbs(5); @@ -62,7 +71,7 @@ where let failed_meta = env.create_database(Some("failed-meta"))?; let processing = Arc::new(RwLock::new(None)); - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + let (notification_sender, mut notification_receiver) = mpsc::channel(10); // Send a first notification to trigger the process. let _ = notification_sender.send(()); @@ -80,13 +89,19 @@ where // We need a weak reference so we can take ownership on the arc later when we // want to close the index. let update_store_weak = Arc::downgrade(&update_store); - std::thread::spawn(move || { + tokio::task::spawn(async move { // Block and wait for something to process. - 'outer: for _ in notification_receiver { + 'outer: while let Some(_) = notification_receiver.recv().await { loop { match update_store_weak.upgrade() { Some(update_store) => { - match update_store.process_pending_update(&mut update_handler) { + let handler = update_handler.clone(); + let res = tokio::task::spawn_blocking(move || { + update_store.process_pending_update(handler) + }) + .await + .unwrap(); + match res { Ok(Some(_)) => (), Ok(None) => break, Err(e) => eprintln!("error while processing update: {}", e), @@ -108,17 +123,20 @@ where /// Returns the new biggest id to use to store the new update. fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta + let last_pending = self + .pending_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); - let last_processed = self.processed_meta + let last_processed = self + .processed_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); - let last_aborted = self.aborted_meta + let last_aborted = self + .aborted_meta .remap_data_type::() .last(txn)? .map(|(k, _)| k.get()); @@ -154,21 +172,22 @@ where let meta = Pending::new(meta, update_id, index_uuid); self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; + self.pending + .put(&mut wtxn, &update_key, &content.as_ref().to_owned())?; wtxn.commit()?; - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } + self.notification_sender + .blocking_send(()) + .expect("Update store loop exited."); Ok(meta) } /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> + fn process_pending_update(&self, mut handler: U) -> heed::Result> where - U: HandleUpdate + Send + 'static, + U: HandleUpdate, { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; @@ -178,7 +197,8 @@ where // a reader while processing it, not a writer. match first_meta { Some((first_id, pending)) => { - let content_path = self.pending + let content_path = self + .pending .get(&rtxn, &first_id)? .expect("associated update content"); @@ -186,10 +206,7 @@ where // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); + self.processing.write().unwrap().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file); @@ -199,10 +216,7 @@ where // we must remove the content from the pending and processing stores and // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); + self.processing.write().unwrap().take(); self.pending_meta.delete(&mut wtxn, &first_id)?; remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; @@ -213,8 +227,8 @@ where wtxn.commit()?; Ok(Some(())) - }, - None => Ok(None) + } + None => Ok(None), } } @@ -241,7 +255,13 @@ where let failed_iter = self.failed_meta.iter(&rtxn)?; // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + (f)( + processing, + processed_iter, + aborted_iter, + pending_iter, + failed_iter, + ) } /// Returns the update associated meta or `None` if the update doesn't exist. @@ -340,22 +360,33 @@ mod tests { use std::time::{Duration, Instant}; impl HandleUpdate for F - where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { - self(meta, content) - } + where + F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static, + { + fn handle_update( + &mut self, + meta: Processing, + content: &[u8], + ) -> Result, Failed> { + self(meta, content) } + } #[test] fn simple() { let dir = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); + let update_store = UpdateStore::open( + options, + dir, + |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }, + ) + .unwrap(); let meta = String::from("kiki"); let update = update_store.register_update(meta, &[]).unwrap(); @@ -374,12 +405,17 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); + let update_store = UpdateStore::open( + options, + dir, + |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }, + ) + .unwrap(); let before_register = Instant::now(); From f090f42e7a250a7a94ef094a98d4fba19c2c2d34 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 19:18:01 +0100 Subject: [PATCH 25/69] multi index store create two channels for Index handler, one for writes and one for reads, so write are processed one at a time, while reads are processed in parallel. --- Cargo.lock | 4 +- src/index_controller/index_actor.rs | 129 ++++++++++++++++--------- src/index_controller/update_actor.rs | 138 +++++++++++++++++++-------- 3 files changed, 188 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f3b20e89..5f1134379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.3.0" @@ -428,7 +430,7 @@ checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "assert-json-diff" version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" dependencies = [ "serde", "serde_json", diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 59f000575..0e74d6665 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -11,6 +11,8 @@ use log::info; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use std::future::Future; +use futures::pin_mut; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; @@ -61,7 +63,8 @@ enum IndexMsg { } struct IndexActor { - inbox: Option>, + read_receiver: Option>, + write_receiver: Option>, update_handler: Arc, store: S, } @@ -82,60 +85,96 @@ trait IndexStore { } impl IndexActor { - fn new(inbox: mpsc::Receiver, store: S) -> Self { + fn new( + read_receiver: mpsc::Receiver, + write_receiver: mpsc::Receiver, + store: S + ) -> Self { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); let update_handler = Arc::new(update_handler); - let inbox = Some(inbox); + let read_receiver = Some(read_receiver); + let write_receiver = Some(write_receiver); Self { - inbox, + read_receiver, + write_receiver, store, update_handler, } } + /// `run` poll the write_receiver and read_receiver concurrently, but while messages send + /// through the read channel are processed concurrently, the messages sent through the write + /// channel are processed one at a time. async fn run(mut self) { - use IndexMsg::*; - - let mut inbox = self - .inbox + let mut read_receiver = self + .read_receiver .take() .expect("Index Actor must have a inbox at this point."); - let stream = stream! { + let read_stream = stream! { loop { - match inbox.recv().await { + match read_receiver.recv().await { Some(msg) => yield msg, None => break, } } }; - let fut = stream.for_each_concurrent(Some(10), |msg| async { - match msg { - CreateIndex { - uuid, - primary_key, - ret, - } => self.handle_create_index(uuid, primary_key, ret).await, - Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - Settings { ret, uuid } => self.handle_settings(uuid, ret).await, - Documents { - ret, - uuid, - attributes_to_retrieve, - offset, - limit, - } => { - self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) - .await - } - Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, - } - }); + let mut write_receiver = self + .write_receiver + .take() + .expect("Index Actor must have a inbox at this point."); - fut.await; + let write_stream = stream! { + loop { + match write_receiver.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + pin_mut!(write_stream); + pin_mut!(read_stream); + + let fut1 = read_stream.for_each_concurrent(Some(10), |msg| self.handle_message(msg)); + let fut2 = write_stream.for_each_concurrent(Some(1), |msg| self.handle_message(msg)); + + let fut1: Box + Unpin + Send> = Box::new(fut1); + let fut2: Box + Unpin + Send> = Box::new(fut2); + + //let futures = futures::stream::futures_unordered::FuturesUnordered::new(); + //futures.push(fut1); + //futures.push(fut2); + //futures.for_each(f) + tokio::join!(fut1, fut2); + + } + + async fn handle_message(&self, msg: IndexMsg) { + use IndexMsg::*; + match msg { + CreateIndex { + uuid, + primary_key, + ret, + } => self.handle_create_index(uuid, primary_key, ret).await, + Update { ret, meta, data } => self.handle_update(meta, data, ret).await, + Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, + Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) + .await + } + Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, + } } async fn handle_search( @@ -221,17 +260,19 @@ impl IndexActor { #[derive(Clone)] pub struct IndexActorHandle { - sender: mpsc::Sender, + read_sender: mpsc::Sender, + write_sender: mpsc::Sender, } impl IndexActorHandle { pub fn new(path: impl AsRef) -> Self { - let (sender, receiver) = mpsc::channel(100); + let (read_sender, read_receiver) = mpsc::channel(100); + let (write_sender, write_receiver) = mpsc::channel(100); let store = MapIndexStore::new(path); - let actor = IndexActor::new(receiver, store); + let actor = IndexActor::new(read_receiver, write_receiver, store); tokio::task::spawn(actor.run()); - Self { sender } + Self { read_sender, write_sender } } pub async fn create_index( @@ -245,28 +286,28 @@ impl IndexActorHandle { uuid, primary_key, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; receiver.await.expect("IndexActor has been killed") } pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Search { uuid, query, ret }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } pub async fn settings(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Settings { uuid, ret }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -285,7 +326,7 @@ impl IndexActorHandle { attributes_to_retrieve, limit, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -302,7 +343,7 @@ impl IndexActorHandle { doc_id, attributes_to_retrieve, }; - let _ = self.sender.send(msg).await; + let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } } diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 2f3c058bc..a9e2a412a 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,17 +1,18 @@ use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::collections::{HashMap, hash_map::Entry}; -use log::info; use super::index_actor::IndexActorHandle; +use log::info; use thiserror::Error; -use tokio::sync::{mpsc, oneshot}; -use uuid::Uuid; use tokio::fs::File; use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; -use crate::index_controller::{UpdateMeta, UpdateStatus}; use crate::index::UpdateResult; +use crate::index_controller::{UpdateMeta, UpdateStatus}; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -28,33 +29,42 @@ enum UpdateMsg { uuid: Uuid, meta: UpdateMeta, data: mpsc::Receiver>, - ret: oneshot::Sender> + ret: oneshot::Sender>, }, ListUpdates { uuid: Uuid, ret: oneshot::Sender>>, - } + }, } -struct UpdateActor { +struct UpdateActor { path: PathBuf, - store: Arc, + store: S, inbox: mpsc::Receiver>, - index_handle: IndexActorHandle, } -impl UpdateActor -where D: AsRef<[u8]> + Sized + 'static, +#[async_trait::async_trait] +trait UpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result>; +} + +impl UpdateActor +where + D: AsRef<[u8]> + Sized + 'static, + S: UpdateStoreStore, { fn new( - store: Arc, + store: S, inbox: mpsc::Receiver>, - index_handle: IndexActorHandle, path: impl AsRef, - ) -> Self { + ) -> Self { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path).unwrap(); - Self { store, inbox, index_handle, path } + Self { + store, + inbox, + path, + } } async fn run(mut self) { @@ -64,15 +74,26 @@ where D: AsRef<[u8]> + Sized + 'static, loop { match self.inbox.recv().await { - Some(Update { uuid, meta, data, ret }) => self.handle_update(uuid, meta, data, ret).await, + Some(Update { + uuid, + meta, + data, + ret, + }) => self.handle_update(uuid, meta, data, ret).await, Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, None => {} } } } - async fn handle_update(&self, uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, ret: oneshot::Sender>) { - let store = self.store.clone(); + async fn handle_update( + &self, + uuid: Uuid, + meta: UpdateMeta, + mut payload: mpsc::Receiver>, + ret: oneshot::Sender>, + ) { + let update_store = self.store.get_or_create(uuid).await.unwrap(); let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); let mut file = File::create(&path).await.unwrap(); @@ -84,7 +105,7 @@ where D: AsRef<[u8]> + Sized + 'static, } Err(e) => { ret.send(Err(UpdateError::Error(e))); - return + return; } } } @@ -94,15 +115,20 @@ where D: AsRef<[u8]> + Sized + 'static, let file = file.into_std().await; let result = tokio::task::spawn_blocking(move || { - let result = store + let result = update_store .register_update(meta, path, uuid) .map(|pending| UpdateStatus::Pending(pending)) .map_err(|e| UpdateError::Error(Box::new(e))); let _ = ret.send(result); - }).await; + }) + .await; } - async fn handle_list_updates(&self, uuid: Uuid, ret: oneshot::Sender>>) { + async fn handle_list_updates( + &self, + uuid: Uuid, + ret: oneshot::Sender>>, + ) { todo!() } } @@ -113,29 +139,26 @@ pub struct UpdateActorHandle { } impl UpdateActorHandle -where D: AsRef<[u8]> + Sized + 'static, +where + D: AsRef<[u8]> + Sized + 'static, { pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let store = MapUpdateStoreStore::new(index_handle, &path); + let actor = UpdateActor::new(store, receiver, path); - let path = path - .as_ref() - .to_owned() - .join("updates"); - - create_dir_all(&path).unwrap(); - let index_handle_clone = index_handle.clone(); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle_clone.update(meta, file)) - }).unwrap(); - let actor = UpdateActor::new(store, receiver, index_handle, path); tokio::task::spawn_local(actor.run()); + Self { sender } } - pub async fn update(&self, meta: UpdateMeta, data: mpsc::Receiver>, uuid: Uuid) -> Result { + pub async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::Update { uuid, @@ -143,7 +166,46 @@ where D: AsRef<[u8]> + Sized + 'static, meta, ret, }; - let _ = self.sender.send(msg).await; + let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } } + +struct MapUpdateStoreStore { + db: Arc>>>, + index_handle: IndexActorHandle, + path: PathBuf, +} + +impl MapUpdateStoreStore { + fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + let db = Arc::new(RwLock::new(HashMap::new())); + let path = path.as_ref().to_owned(); + Self { db, index_handle, path } + } +} + +#[async_trait::async_trait] +impl UpdateStoreStore for MapUpdateStoreStore { + async fn get_or_create(&self, uuid: Uuid) -> Result> { + match self.db.write().await.entry(uuid) { + Entry::Vacant(e) => { + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let path = self.path.clone().join(format!("updates-{}", e.key())); + create_dir_all(&path).unwrap(); + let index_handle = self.index_handle.clone(); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }).unwrap(); + let store = e.insert(store); + Ok(store.clone()) + } + Entry::Occupied(e) => { + Ok(e.get().clone()) + } + } + } +} + + From c2282ab5cb0d94dedb04978f2931c4078dbe3fee Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Mar 2021 19:30:13 +0100 Subject: [PATCH 26/69] non local udpate actor --- src/index_controller/update_actor.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index a9e2a412a..4d1ff78fb 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -140,7 +140,7 @@ pub struct UpdateActorHandle { impl UpdateActorHandle where - D: AsRef<[u8]> + Sized + 'static, + D: AsRef<[u8]> + Sized + 'static + Sync + Send, { pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let path = path.as_ref().to_owned().join("updates"); @@ -148,7 +148,7 @@ where let store = MapUpdateStoreStore::new(index_handle, &path); let actor = UpdateActor::new(store, receiver, path); - tokio::task::spawn_local(actor.run()); + tokio::task::spawn(actor.run()); Self { sender } } @@ -207,5 +207,3 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } } - - From a9c7b7374434c31b0f1977445f72dd4a2c104b6b Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 5 Mar 2021 18:34:04 +0100 Subject: [PATCH 27/69] implement list all updates --- src/data/updates.rs | 5 +- src/index_controller/mod.rs | 10 ++- src/index_controller/update_actor.rs | 103 ++++++++++++++++++--------- src/routes/index.rs | 2 +- 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index a712f4cc6..3e3837861 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -65,9 +65,8 @@ impl Data { //self.index_controller.update_status(index, uid) } - pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - todo!() - //self.index_controller.all_update_status(index) + pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { + self.index_controller.all_update_status(index.as_ref().to_string()).await } pub fn update_index( diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d790d1d58..8eb1684a2 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -107,7 +107,7 @@ impl IndexController { } pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?.unwrap(); let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; @@ -154,8 +154,12 @@ impl IndexController { todo!() } - fn all_update_status(&self, index: String) -> anyhow::Result> { - todo!() + pub async fn all_update_status(&self, index: String) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index).await? + .context("index not found")?; + let result = self.update_handle.get_all_updates_status(uuid).await?; + Ok(result) } pub fn list_indexes(&self) -> anyhow::Result> { diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 4d1ff78fb..b236df2a6 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,7 +1,7 @@ +use std::collections::{hash_map::Entry, HashMap}; use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::collections::{HashMap, hash_map::Entry}; use super::index_actor::IndexActorHandle; use log::info; @@ -10,6 +10,7 @@ use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use itertools::Itertools; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; @@ -22,6 +23,8 @@ type PayloadData = std::result::Result), + #[error("Index {0} doesn't exist.")] + UnexistingIndex(Uuid), } enum UpdateMsg { @@ -46,6 +49,7 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn get(&self, uuid: &Uuid) -> Result>>; } impl UpdateActor @@ -53,24 +57,16 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new( - store: S, - inbox: mpsc::Receiver>, - path: impl AsRef, - ) -> Self { + fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> Self { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path).unwrap(); - Self { - store, - inbox, - path, - } + Self { store, inbox, path } } async fn run(mut self) { use UpdateMsg::*; - info!("started update actor."); + info!("Started update actor."); loop { match self.inbox.recv().await { @@ -79,8 +75,12 @@ where meta, data, ret, - }) => self.handle_update(uuid, meta, data, ret).await, - Some(ListUpdates { uuid, ret }) => self.handle_list_updates(uuid, ret).await, + }) => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + Some(ListUpdates { uuid, ret }) => { + let _ = ret.send(self.handle_list_updates(uuid).await); + } , None => {} } } @@ -91,45 +91,68 @@ where uuid: Uuid, meta: UpdateMeta, mut payload: mpsc::Receiver>, - ret: oneshot::Sender>, - ) { - let update_store = self.store.get_or_create(uuid).await.unwrap(); + ) -> Result { + let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); - let mut file = File::create(&path).await.unwrap(); + let mut file = File::create(&path).await + .map_err(|e| UpdateError::Error(Box::new(e)))?; while let Some(bytes) = payload.recv().await { match bytes { Ok(bytes) => { - file.write_all(bytes.as_ref()).await; + file.write_all(bytes.as_ref()).await + .map_err(|e| UpdateError::Error(Box::new(e)))?; } Err(e) => { - ret.send(Err(UpdateError::Error(e))); - return; + return Err(UpdateError::Error(e)); } } } - file.flush().await; + file.flush().await + .map_err(|e| UpdateError::Error(Box::new(e)))?; let file = file.into_std().await; - let result = tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || { let result = update_store .register_update(meta, path, uuid) .map(|pending| UpdateStatus::Pending(pending)) .map_err(|e| UpdateError::Error(Box::new(e))); - let _ = ret.send(result); + result }) - .await; + .await + .map_err(|e| UpdateError::Error(Box::new(e)))? } async fn handle_list_updates( &self, uuid: Uuid, - ret: oneshot::Sender>>, - ) { - todo!() + ) -> Result> { + let store = self.store.get(&uuid).await?; + tokio::task::spawn_blocking(move || { + let result = match store { + Some(update_store) => { + let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { + Ok(processing + .map(UpdateStatus::from) + .into_iter() + .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) + .chain(aborted.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(processed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(failed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .sorted_by(|a, b| a.id().cmp(&b.id())) + .collect()) + }) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + Ok(updates) + } + None => Err(UpdateError::UnexistingIndex(uuid)), + }; + result + }).await + .map_err(|e| UpdateError::Error(Box::new(e)))? } } @@ -169,6 +192,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::ListUpdates { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -181,7 +211,11 @@ impl MapUpdateStoreStore { fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); - Self { db, index_handle, path } + Self { + db, + index_handle, + path, + } } } @@ -197,13 +231,16 @@ impl UpdateStoreStore for MapUpdateStoreStore { let index_handle = self.index_handle.clone(); let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) - }).unwrap(); + }) + .unwrap(); let store = e.insert(store); Ok(store.clone()) } - Entry::Occupied(e) => { - Ok(e.get().clone()) - } + Entry::Occupied(e) => Ok(e.get().clone()), } } + + async fn get(&self, uuid: &Uuid) -> Result>> { + Ok(self.db.read().await.get(uuid).cloned()) + } } diff --git a/src/routes/index.rs b/src/routes/index.rs index 813256517..a8618d9dd 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -154,7 +154,7 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid); + let result = data.get_updates_status(&path.index_uid).await; match result { Ok(metas) => { let json = serde_json::to_string(&metas).unwrap(); From 7d28f8cff0fda748663e53dc27373496ee064eba Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 10:51:52 +0100 Subject: [PATCH 28/69] implement get single udpate --- Cargo.lock | 1 + Cargo.toml | 2 +- src/data/updates.rs | 6 ++---- src/index_controller/mod.rs | 9 +++++++-- src/index_controller/update_actor.rs | 26 ++++++++++++++++++++++++++ src/routes/index.rs | 2 +- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f1134379..cbae20c98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,6 +1870,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", diff --git a/Cargo.toml b/Cargo.toml index aaf571ef3..9bc5e4d1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../milli/milli" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/src/data/updates.rs b/src/data/updates.rs index 3e3837861..dd74bc47f 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -59,10 +59,8 @@ impl Data { //Ok(()) } - #[inline] - pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - todo!() - //self.index_controller.update_status(index, uid) + pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { + self.index_controller.update_status(index.as_ref().to_string(), uid).await } pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 8eb1684a2..4606526c4 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -150,8 +150,13 @@ impl IndexController { todo!() } - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { - todo!() + pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + let uuid = self.uuid_resolver + .resolve(index) + .await? + .context("index not found")?; + let result = self.update_handle.update_status(uuid, id).await?; + Ok(result) } pub async fn all_update_status(&self, index: String) -> anyhow::Result> { diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index b236df2a6..83f29c380 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -38,6 +38,11 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>>, }, + GetUpdate { + uuid: Uuid, + ret: oneshot::Sender>>, + id: u64, + } } struct UpdateActor { @@ -81,6 +86,9 @@ where Some(ListUpdates { uuid, ret }) => { let _ = ret.send(self.handle_list_updates(uuid).await); } , + Some(GetUpdate { uuid, ret, id }) => { + let _ = ret.send(self.handle_get_update(uuid, id).await); + } None => {} } } @@ -154,6 +162,17 @@ where }).await .map_err(|e| UpdateError::Error(Box::new(e)))? } + + + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { + let store = self.store + .get(&uuid) + .await? + .ok_or(UpdateError::UnexistingIndex(uuid))?; + let result = store.meta(id) + .map_err(|e| UpdateError::Error(Box::new(e)))?; + Ok(result) + } } #[derive(Clone)] @@ -199,6 +218,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetUpdate { uuid, id, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { diff --git a/src/routes/index.rs b/src/routes/index.rs index a8618d9dd..c9200f085 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -133,7 +133,7 @@ async fn get_update_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id); + let result = data.get_update_status(&path.index_uid, path.update_id).await; match result { Ok(Some(meta)) => { let json = serde_json::to_string(&meta).unwrap(); From 86211b1ddd0c928bea2222d9341c25673102e1d1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 10:53:11 +0100 Subject: [PATCH 29/69] import routes modules in main --- src/main.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index c5e2d11e6..9e69c4091 100644 --- a/src/main.rs +++ b/src/main.rs @@ -83,18 +83,20 @@ async fn main() -> Result<(), MainError> { } async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { + use meilisearch::routes::*; + let http_server = HttpServer::new(move || { let app = App::new() .configure(|c| configure_data(c, &data)) - .configure(meilisearch::routes::document::services) - .configure(meilisearch::routes::index::services) - .configure(meilisearch::routes::search::services) - .configure(meilisearch::routes::settings::services) - .configure(meilisearch::routes::stop_words::services) - .configure(meilisearch::routes::synonym::services) - .configure(meilisearch::routes::health::services) - .configure(meilisearch::routes::stats::services) - .configure(meilisearch::routes::key::services); + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); //.configure(routes::dump::services); let app = if enable_frontend { app From d9254c43554755f8cc8e153f1d5b52dd029b2d58 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 12:57:56 +0100 Subject: [PATCH 30/69] implement index delete --- src/data/updates.rs | 8 +-- src/index_controller/index_actor.rs | 80 +++++++++++++++++++++++---- src/index_controller/mod.rs | 32 +++++++++-- src/index_controller/update_actor.rs | 51 +++++++++++++++-- src/index_controller/uuid_resolver.rs | 30 ++++++++-- 5 files changed, 171 insertions(+), 30 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index dd74bc47f..e9d1b51b7 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -51,12 +51,10 @@ impl Data { pub async fn delete_index( &self, - _index: impl AsRef + Send + Sync + 'static, + index: impl AsRef + Send + Sync + 'static, ) -> anyhow::Result<()> { - todo!() - //let index_controller = self.index_controller.clone(); - //tokio::task::spawn_blocking(move || { index_controller.delete_index(index) }).await??; - //Ok(()) + self.index_controller.delete_index(index.as_ref().to_owned()).await?; + Ok(()) } pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 0e74d6665..c77793a5b 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,19 +1,20 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::{create_dir_all, File}; +use std::fs::{create_dir_all, File, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; use chrono::Utc; +use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use std::future::Future; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use std::future::Future; -use futures::pin_mut; +use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; @@ -59,7 +60,11 @@ enum IndexMsg { attributes_to_retrieve: Option>, doc_id: String, ret: oneshot::Sender>, - } + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, } struct IndexActor { @@ -82,13 +87,14 @@ trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: &Uuid) -> Result>; } impl IndexActor { fn new( read_receiver: mpsc::Receiver, write_receiver: mpsc::Receiver, - store: S + store: S, ) -> Self { let options = IndexerOpts::default(); let update_handler = UpdateHandler::new(&options).unwrap(); @@ -149,7 +155,6 @@ impl IndexActor { //futures.push(fut2); //futures.for_each(f) tokio::join!(fut1, fut2); - } async fn handle_message(&self, msg: IndexMsg) { @@ -173,7 +178,18 @@ impl IndexActor { self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) .await } - Document { uuid, attributes_to_retrieve, doc_id, ret } => self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret).await, + Document { + uuid, + attributes_to_retrieve, + doc_id, + ret, + } => { + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret) + .await + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + }, } } @@ -236,10 +252,12 @@ impl IndexActor { ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.retrieve_documents(offset, limit, attributes_to_retrieve) + let result = index + .retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); - }).await; + }) + .await; } async fn handle_fetch_document( @@ -251,10 +269,29 @@ impl IndexActor { ) { let index = self.store.get(uuid).await.unwrap().unwrap(); tokio::task::spawn_blocking(move || { - let result = index.retrieve_document(doc_id, attributes_to_retrieve) + let result = index + .retrieve_document(doc_id, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)); let _ = ret.send(result); - }).await; + }) + .await; + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let index = self.store.delete(&uuid).await?; + + if let Some(index) = index { + tokio::task::spawn(async move { + let index = index.0; + let store = get_arc_ownership_blocking(index).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Index {} was closed.", uuid); + }); + }); + } + + Ok(()) } } @@ -272,7 +309,10 @@ impl IndexActorHandle { let store = MapIndexStore::new(path); let actor = IndexActor::new(read_receiver, write_receiver, store); tokio::task::spawn(actor.run()); - Self { read_sender, write_sender } + Self { + read_sender, + write_sender, + } } pub async fn create_index( @@ -346,6 +386,13 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Delete { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { @@ -408,6 +455,15 @@ impl IndexStore for MapIndexStore { async fn get(&self, uuid: Uuid) -> Result> { Ok(self.index_store.read().await.get(&uuid).cloned()) } + + async fn delete(&self, uuid: &Uuid) -> Result> { + let index = self.index_store.write().await.remove(&uuid); + if index.is_some() { + let db_path = self.root.join(format!("index-{}", uuid)); + remove_dir_all(db_path).unwrap(); + } + Ok(index) + } } impl MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 4606526c4..19fe62f4d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,11 +1,13 @@ -mod updates; mod index_actor; mod update_actor; -mod uuid_resolver; -mod update_store; mod update_handler; +mod update_store; +mod updates; +mod uuid_resolver; use std::path::Path; +use std::sync::Arc; +use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::Context; @@ -14,6 +16,7 @@ use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; use tokio::sync::{mpsc, oneshot}; +use tokio::time::sleep; use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; @@ -146,8 +149,14 @@ impl IndexController { Ok(index_meta) } - fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { - todo!() + pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + let uuid = self.uuid_resolver + .delete(index_uid) + .await? + .context("index not found")?; + self.update_handle.delete(uuid.clone()).await?; + self.index_handle.delete(uuid).await?; + Ok(()) } pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { @@ -219,3 +228,16 @@ impl IndexController { Ok(result) } } + +pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { + loop { + match Arc::try_unwrap(item) { + Ok(item) => return item, + Err(item_arc) => { + item = item_arc; + sleep(Duration::from_millis(100)).await; + continue; + } + } + } +} diff --git a/src/index_controller/update_actor.rs b/src/index_controller/update_actor.rs index 83f29c380..370912dcf 100644 --- a/src/index_controller/update_actor.rs +++ b/src/index_controller/update_actor.rs @@ -1,19 +1,20 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::create_dir_all; +use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use super::index_actor::IndexActorHandle; +use itertools::Itertools; use log::info; +use super::index_actor::IndexActorHandle; use thiserror::Error; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use itertools::Itertools; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; +use super::get_arc_ownership_blocking; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -42,7 +43,11 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>>, id: u64, - } + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, } struct UpdateActor { @@ -54,6 +59,7 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: &Uuid) -> Result>>; async fn get(&self, uuid: &Uuid) -> Result>>; } @@ -89,6 +95,9 @@ where Some(GetUpdate { uuid, ret, id }) => { let _ = ret.send(self.handle_get_update(uuid, id).await); } + Some(Delete { uuid, ret }) => { + let _ = ret.send(self.handle_delete(uuid).await); + } None => {} } } @@ -173,6 +182,24 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))?; Ok(result) } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let store = self.store + .delete(&uuid) + .await?; + + if let Some(store) = store { + tokio::task::spawn(async move { + let store = get_arc_ownership_blocking(store).await; + tokio::task::spawn_blocking(move || { + store.prepare_for_closing().wait(); + info!("Update store {} was closed.", uuid); + }); + }); + } + + Ok(()) + } } #[derive(Clone)] @@ -225,6 +252,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Delete { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -269,4 +303,13 @@ impl UpdateStoreStore for MapUpdateStoreStore { async fn get(&self, uuid: &Uuid) -> Result>> { Ok(self.db.read().await.get(uuid).cloned()) } + + async fn delete(&self, uuid: &Uuid) -> Result>> { + let store = self.db.write().await.remove(&uuid); + if store.is_some() { + let path = self.path.clone().join(format!("updates-{}", uuid)); + remove_dir_all(path).unwrap(); + } + Ok(store) + } } diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index 3143ae8fc..50740f30f 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -22,6 +22,10 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>, }, + Delete { + name: String, + ret: oneshot::Sender>>, + }, } struct UuidResolverActor { @@ -45,6 +49,7 @@ impl UuidResolverActor { Some(Create { name, ret }) => self.handle_create(name, ret).await, Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, + Some(Delete { name, ret }) => self.handle_delete(name, ret).await, // all senders have been dropped, need to quit. None => break, } @@ -64,7 +69,12 @@ impl UuidResolverActor { } async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.get_uuid(name).await; + let result = self.store.get_uuid(&name).await; + let _ = ret.send(result); + } + + async fn handle_delete(&self, name: String, ret: oneshot::Sender>>) { + let result = self.store.delete(&name).await; let _ = ret.send(result); } } @@ -103,6 +113,13 @@ impl UuidResolverHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + + pub async fn delete(&self, name: String) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Delete { name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } } #[derive(Clone, Debug, Error)] @@ -116,7 +133,8 @@ trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: String) -> Result>; + async fn get_uuid(&self, name: &str) -> Result>; + async fn delete(&self, name: &str) -> Result>; } struct MapUuidStore(Arc>>); @@ -140,7 +158,11 @@ impl UuidStore for MapUuidStore { } } - async fn get_uuid(&self, name: String) -> Result> { - Ok(self.0.read().await.get(&name).cloned()) + async fn get_uuid(&self, name: &str) -> Result> { + Ok(self.0.read().await.get(name).cloned()) + } + + async fn delete(&self, name: &str) -> Result> { + Ok(self.0.write().await.remove(name)) } } From 281a4459988cf9e25464be2ff72818fdee8ddfcc Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 20:12:20 +0100 Subject: [PATCH 31/69] implement list indexes --- src/data/mod.rs | 4 +- src/index_controller/index_actor.rs | 62 ++++++++++++++++++++------- src/index_controller/mod.rs | 32 +++++++++----- src/index_controller/uuid_resolver.rs | 24 +++++++++++ src/routes/index.rs | 2 +- 5 files changed, 95 insertions(+), 29 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 1dec00766..cdb2127a8 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -82,8 +82,8 @@ impl Data { self.index_controller.settings(index_uid.as_ref().to_string()).await } - pub fn list_indexes(&self) -> anyhow::Result> { - self.index_controller.list_indexes() + pub async fn list_indexes(&self) -> anyhow::Result> { + self.index_controller.list_indexes().await } pub fn index(&self, name: impl AsRef) -> anyhow::Result> { diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index c77793a5b..360f3a3c0 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -1,18 +1,19 @@ use std::collections::{hash_map::Entry, HashMap}; -use std::fs::{create_dir_all, File, remove_dir_all}; +use std::fs::{create_dir_all, remove_dir_all, File}; +use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; -use chrono::Utc; +use chrono::{Utc, DateTime}; use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; -use std::future::Future; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use serde::{Serialize, Deserialize}; use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; @@ -20,7 +21,7 @@ use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ updates::{Failed, Processed, Processing}, - IndexMetadata, UpdateMeta, + UpdateMeta, }; use crate::option::IndexerOpts; @@ -28,11 +29,20 @@ pub type Result = std::result::Result; type AsyncMap = Arc>>; type UpdateResult = std::result::Result, Failed>; +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMeta{ + uuid: Uuid, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + enum IndexMsg { CreateIndex { uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, + ret: oneshot::Sender>, }, Update { meta: Processing, @@ -65,6 +75,10 @@ enum IndexMsg { uuid: Uuid, ret: oneshot::Sender>, }, + GetMeta { + uuid: Uuid, + ret: oneshot::Sender>>, + }, } struct IndexActor { @@ -84,10 +98,11 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; + async fn get_meta(&self, uuid: &Uuid) -> Result>; } impl IndexActor { @@ -150,10 +165,6 @@ impl IndexActor { let fut1: Box + Unpin + Send> = Box::new(fut1); let fut2: Box + Unpin + Send> = Box::new(fut2); - //let futures = futures::stream::futures_unordered::FuturesUnordered::new(); - //futures.push(fut1); - //futures.push(fut2); - //futures.for_each(f) tokio::join!(fut1, fut2); } @@ -189,7 +200,10 @@ impl IndexActor { } Delete { uuid, ret } => { let _ = ret.send(self.handle_delete(uuid).await); - }, + } + GetMeta { uuid, ret } => { + let _ = ret.send(self.handle_get_meta(uuid).await); + } } } @@ -210,7 +224,7 @@ impl IndexActor { &self, uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, + ret: oneshot::Sender>, ) { let result = self.store.create_index(uuid, primary_key).await; let _ = ret.send(result); @@ -293,6 +307,11 @@ impl IndexActor { Ok(()) } + + async fn handle_get_meta(&self, uuid: Uuid) -> Result> { + let result = self.store.get_meta(&uuid).await?; + Ok(result) + } } #[derive(Clone)] @@ -319,7 +338,7 @@ impl IndexActorHandle { &self, uuid: Uuid, primary_key: Option, - ) -> Result { + ) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -393,20 +412,27 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn get_index_meta(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetMeta { uuid, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct MapIndexStore { root: PathBuf, - meta_store: AsyncMap, + meta_store: AsyncMap, index_store: AsyncMap, } #[async_trait::async_trait] impl IndexStore for MapIndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { - let meta = IndexMetadata { + let meta = IndexMeta{ uuid, created_at: Utc::now(), updated_at: Utc::now(), @@ -464,6 +490,10 @@ impl IndexStore for MapIndexStore { } Ok(index) } + + async fn get_meta(&self, uuid: &Uuid) -> Result> { + Ok(self.meta_store.read().await.get(uuid).cloned()) + } } impl MapIndexStore { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 19fe62f4d..09a979cbd 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -11,7 +11,6 @@ use std::time::Duration; use actix_web::web::{Bytes, Payload}; use anyhow::Context; -use chrono::{DateTime, Utc}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -28,10 +27,9 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - uuid: Uuid, - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, + name: String, + #[serde(flatten)] + meta: index_actor::IndexMeta, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -144,9 +142,12 @@ impl IndexController { pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { let IndexSettings { name, primary_key } = index_settings; - let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_handle.create_index(uuid, primary_key).await?; - Ok(index_meta) + let name = name.unwrap(); + let uuid = self.uuid_resolver.create(name.clone()).await?; + let meta = self.index_handle.create_index(uuid, primary_key).await?; + let meta = IndexMetadata { name, meta }; + + Ok(meta) } pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { @@ -176,8 +177,19 @@ impl IndexController { Ok(result) } - pub fn list_indexes(&self) -> anyhow::Result> { - todo!() + pub async fn list_indexes(&self) -> anyhow::Result> { + let uuids = self.uuid_resolver.list().await?; + + let mut ret = Vec::new(); + + for (name, uuid) in uuids { + if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { + let meta = IndexMetadata { name, meta }; + ret.push(meta); + } + } + + Ok(ret) } pub async fn settings(&self, index: String) -> anyhow::Result { diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index 50740f30f..e2539fdb2 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -26,6 +26,9 @@ enum UuidResolveMsg { name: String, ret: oneshot::Sender>>, }, + List { + ret: oneshot::Sender>>, + }, } struct UuidResolverActor { @@ -50,6 +53,9 @@ impl UuidResolverActor { Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, Some(Delete { name, ret }) => self.handle_delete(name, ret).await, + Some(List { ret }) => { + let _ = ret.send(self.handle_list().await); + } // all senders have been dropped, need to quit. None => break, } @@ -77,6 +83,11 @@ impl UuidResolverActor { let result = self.store.delete(&name).await; let _ = ret.send(result); } + + async fn handle_list(&self) -> Result> { + let result = self.store.list().await?; + Ok(result) + } } #[derive(Clone)] @@ -120,6 +131,13 @@ impl UuidResolverHandle { let _ = self.sender.send(msg).await; Ok(receiver.await.expect("Uuid resolver actor has been killed")?) } + + pub async fn list(&self) -> anyhow::Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::List { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + } } #[derive(Clone, Debug, Error)] @@ -135,6 +153,7 @@ trait UuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result; async fn get_uuid(&self, name: &str) -> Result>; async fn delete(&self, name: &str) -> Result>; + async fn list(&self) -> Result>; } struct MapUuidStore(Arc>>); @@ -165,4 +184,9 @@ impl UuidStore for MapUuidStore { async fn delete(&self, name: &str) -> Result> { Ok(self.0.write().await.remove(name)) } + + async fn list(&self) -> Result> { + let list = self.0.read().await.iter().map(|(name, uuid)| (name.to_owned(), uuid.clone())).collect(); + Ok(list) + } } diff --git a/src/routes/index.rs b/src/routes/index.rs index c9200f085..752493a3b 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -21,7 +21,7 @@ pub fn services(cfg: &mut web::ServiceConfig) { #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { - match data.list_indexes() { + match data.list_indexes().await { Ok(indexes) => { let json = serde_json::to_string(&indexes).unwrap(); Ok(HttpResponse::Ok().body(&json)) From ced32afd9fb102385bf4089e69db8c1382ed1574 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 6 Mar 2021 20:17:58 +0100 Subject: [PATCH 32/69] implement get single index --- src/data/mod.rs | 8 ++------ src/index_controller/mod.rs | 12 ++++++++++++ src/routes/index.rs | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index cdb2127a8..0f7722657 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -86,12 +86,8 @@ impl Data { self.index_controller.list_indexes().await } - pub fn index(&self, name: impl AsRef) -> anyhow::Result> { - todo!() - //Ok(self - //.list_indexes()? - //.into_iter() - //.find(|i| i.uid == name.as_ref())) + pub async fn index(&self, name: impl AsRef) -> anyhow::Result> { + self.index_controller.get_index(name.as_ref().to_string()).await } pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 09a979cbd..bcf663ec8 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -239,6 +239,18 @@ impl IndexController { let result = self.index_handle.search(uuid, query).await?; Ok(result) } + + pub async fn get_index(&self, name: String) -> anyhow::Result> { + let uuid = self.uuid_resolver.resolve(name.clone()).await?; + if let Some(uuid) = uuid { + let result = self.index_handle + .get_index_meta(uuid) + .await? + .map(|meta| IndexMetadata { name, meta }); + return Ok(result) + } + Ok(None) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/src/routes/index.rs b/src/routes/index.rs index 752493a3b..818c87303 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -37,7 +37,7 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - match data.index(&path.index_uid)? { + match data.index(&path.index_uid).await? { Some(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) From ac4d795eff4a625080f3da9dd48265ebc52c2e51 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 10:20:53 +0100 Subject: [PATCH 33/69] update created at when updating index --- src/index_controller/index_actor.rs | 42 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 360f3a3c0..2556e224a 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -94,11 +94,16 @@ pub enum IndexError { Error(#[from] anyhow::Error), #[error("index already exists")] IndexAlreadyExists, + #[error("Index doesn't exists")] + UnexistingIndex, } #[async_trait::async_trait] trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn update_index(&self, uuid: Uuid, f: F) -> Result + where F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static; async fn get_or_create(&self, uuid: Uuid) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; @@ -238,13 +243,16 @@ impl IndexActor { ) { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); - let index = self.store.get_or_create(uuid).await.unwrap(); let update_handler = self.update_handler.clone(); - tokio::task::spawn_blocking(move || { - let result = update_handler.handle_update(meta, data, index); - let _ = ret.send(result); - }) - .await; + let handle = self.store.update_index(uuid, |index| { + let handle = tokio::task::spawn_blocking(move || { + let result = update_handler.handle_update(meta, data, index); + let _ = ret.send(result); + }); + Ok(handle) + }); + + handle.await; } async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { @@ -432,10 +440,11 @@ impl IndexStore for MapIndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { + let now = Utc::now(); let meta = IndexMeta{ uuid, - created_at: Utc::now(), - updated_at: Utc::now(), + created_at: now.clone(), + updated_at: now, primary_key, }; entry.insert(meta).clone() @@ -494,6 +503,23 @@ impl IndexStore for MapIndexStore { async fn get_meta(&self, uuid: &Uuid) -> Result> { Ok(self.meta_store.read().await.get(uuid).cloned()) } + + async fn update_index(&self, uuid: Uuid, f: F) -> Result + where F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static, + { + let index = self.get_or_create(uuid.clone()).await?; + let mut meta = self.get_meta(&uuid).await? + .ok_or(IndexError::UnexistingIndex)?; + match f(index) { + Ok(r) => { + meta.updated_at = Utc::now(); + self.meta_store.write().await.insert(uuid, meta); + Ok(r) + } + Err(e) => Err(e) + } + } } impl MapIndexStore { From 2f93cce7aaf371adff969264291a313f08e47417 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 10:48:34 +0100 Subject: [PATCH 34/69] auto index creation --- src/index_controller/index_actor.rs | 49 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index 2556e224a..c85224bb4 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -104,7 +104,7 @@ trait IndexStore { async fn update_index(&self, uuid: Uuid, f: F) -> Result where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static; - async fn get_or_create(&self, uuid: Uuid) -> Result; + async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; async fn get_meta(&self, uuid: &Uuid) -> Result>; @@ -473,14 +473,49 @@ impl IndexStore for MapIndexStore { Ok(meta) } - async fn get_or_create(&self, uuid: Uuid) -> Result { + async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result { match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(_) => { - todo!() + Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { + Entry::Vacant(meta_entry) => { + let now = Utc::now(); + let meta = IndexMeta{ + uuid, + created_at: now.clone(), + updated_at: now, + primary_key, + }; + let meta = meta_entry.insert(meta); + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); + Ok(index) + }) + .await + .expect("thread died"); + + Ok(index_entry.insert(index?).clone()) } Entry::Occupied(entry) => { - todo!() + let meta = entry.get(); + let db_path = self.root.join(format!("index-{}", meta.uuid)); + + let index: Result = tokio::task::spawn_blocking(move || { + create_dir_all(&db_path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = Index(Arc::new(index)); + Ok(index) + }) + .await + .expect("thread died"); + + Ok(index_entry.insert(index?).clone()) } }, Entry::Occupied(entry) => Ok(entry.get().clone()), @@ -508,7 +543,7 @@ impl IndexStore for MapIndexStore { where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static, { - let index = self.get_or_create(uuid.clone()).await?; + let index = self.get_or_create(uuid.clone(), None).await?; let mut meta = self.get_meta(&uuid).await? .ok_or(IndexError::UnexistingIndex)?; match f(index) { From 06403a57081a63805baee44ade6232e2500df5e8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 15:53:16 +0100 Subject: [PATCH 35/69] clean index actor unwraps --- src/data/mod.rs | 4 +- src/index_controller/index_actor.rs | 188 +++++++++++++++------------ src/index_controller/mod.rs | 6 +- src/index_controller/update_store.rs | 12 +- 4 files changed, 113 insertions(+), 97 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 0f7722657..b011c18b3 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -59,10 +59,8 @@ impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - //let indexer_opts = options.indexer_options.clone(); - create_dir_all(&path)?; - let index_controller = IndexController::new(&path); + let index_controller = IndexController::new(&path)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/src/index_controller/index_actor.rs b/src/index_controller/index_actor.rs index c85224bb4..1ddc041a1 100644 --- a/src/index_controller/index_actor.rs +++ b/src/index_controller/index_actor.rs @@ -5,15 +5,15 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; -use chrono::{Utc, DateTime}; +use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; use heed::EnvOpenOptions; use log::info; +use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; -use serde::{Serialize, Deserialize}; use super::get_arc_ownership_blocking; use super::update_handler::UpdateHandler; @@ -31,7 +31,7 @@ type UpdateResult = std::result::Result, Failed, updated_at: DateTime, @@ -47,7 +47,7 @@ enum IndexMsg { Update { meta: Processing, data: std::fs::File, - ret: oneshot::Sender, + ret: oneshot::Sender>, }, Search { uuid: Uuid, @@ -102,8 +102,9 @@ pub enum IndexError { trait IndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; async fn update_index(&self, uuid: Uuid, f: F) -> Result - where F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static; + where + F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static; async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: &Uuid) -> Result>; @@ -115,18 +116,19 @@ impl IndexActor { read_receiver: mpsc::Receiver, write_receiver: mpsc::Receiver, store: S, - ) -> Self { + ) -> Result { let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options).unwrap(); + let update_handler = UpdateHandler::new(&options) + .map_err(|e| IndexError::Error(e.into()))?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); - Self { + Ok(Self { read_receiver, write_receiver, store, update_handler, - } + }) } /// `run` poll the write_receiver and read_receiver concurrently, but while messages send @@ -180,10 +182,18 @@ impl IndexActor { uuid, primary_key, ret, - } => self.handle_create_index(uuid, primary_key, ret).await, - Update { ret, meta, data } => self.handle_update(meta, data, ret).await, - Search { ret, query, uuid } => self.handle_search(uuid, query, ret).await, - Settings { ret, uuid } => self.handle_settings(uuid, ret).await, + } => { + let _ = ret.send(self.handle_create_index(uuid, primary_key).await); + } + Update { ret, meta, data } => { + let _ = ret.send(self.handle_update(meta, data).await); + } + Search { ret, query, uuid } => { + let _ = ret.send(self.handle_search(uuid, query).await); + } + Settings { ret, uuid } => { + let _ = ret.send(self.handle_settings(uuid).await); + } Documents { ret, uuid, @@ -191,8 +201,10 @@ impl IndexActor { offset, limit, } => { - self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve, ret) - .await + let _ = ret.send( + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve) + .await, + ); } Document { uuid, @@ -200,8 +212,10 @@ impl IndexActor { doc_id, ret, } => { - self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve, ret) - .await + let _ = ret.send( + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve) + .await, + ); } Delete { uuid, ret } => { let _ = ret.send(self.handle_delete(uuid).await); @@ -212,56 +226,51 @@ impl IndexActor { } } - async fn handle_search( - &self, - uuid: Uuid, - query: SearchQuery, - ret: oneshot::Sender>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); - tokio::task::spawn_blocking(move || { - let result = index.perform_search(query); - ret.send(result) - }); + async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + tokio::task::spawn_blocking(move || index.perform_search(query)).await? } async fn handle_create_index( &self, uuid: Uuid, primary_key: Option, - ret: oneshot::Sender>, - ) { - let result = self.store.create_index(uuid, primary_key).await; - let _ = ret.send(result); + ) -> Result { + self.store.create_index(uuid, primary_key).await } async fn handle_update( &self, meta: Processing, data: File, - ret: oneshot::Sender, - ) { + ) -> Result { info!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); - let handle = self.store.update_index(uuid, |index| { - let handle = tokio::task::spawn_blocking(move || { - let result = update_handler.handle_update(meta, data, index); - let _ = ret.send(result); - }); - Ok(handle) - }); + let handle = self + .store + .update_index(uuid, |index| { + let handle = tokio::task::spawn_blocking(move || { + update_handler.handle_update(meta, data, index) + }); + Ok(handle) + }) + .await?; - handle.await; + handle.await.map_err(|e| IndexError::Error(e.into())) } - async fn handle_settings(&self, uuid: Uuid, ret: oneshot::Sender>) { - let index = self.store.get(uuid).await.unwrap().unwrap(); - tokio::task::spawn_blocking(move || { - let result = index.settings().map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); - }) - .await; + async fn handle_settings(&self, uuid: Uuid) -> Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + tokio::task::spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_fetch_documents( @@ -270,16 +279,17 @@ impl IndexActor { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ret: oneshot::Sender>>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); + ) -> Result> { + let index = self.store.get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; tokio::task::spawn_blocking(move || { - let result = index + index .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); + .map_err(|e| IndexError::Error(e)) }) - .await; + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_fetch_document( @@ -287,16 +297,19 @@ impl IndexActor { uuid: Uuid, doc_id: String, attributes_to_retrieve: Option>, - ret: oneshot::Sender>, - ) { - let index = self.store.get(uuid).await.unwrap().unwrap(); + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; tokio::task::spawn_blocking(move || { - let result = index + index .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)); - let _ = ret.send(result); + .map_err(|e| IndexError::Error(e)) }) - .await; + .await + .map_err(|e| IndexError::Error(e.into()))? } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { @@ -329,24 +342,20 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new(path: impl AsRef) -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); let store = MapIndexStore::new(path); - let actor = IndexActor::new(read_receiver, write_receiver, store); + let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); - Self { + Ok(Self { read_sender, write_sender, - } + }) } - pub async fn create_index( - &self, - uuid: Uuid, - primary_key: Option, - ) -> Result { + pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::CreateIndex { ret, @@ -357,11 +366,15 @@ impl IndexActorHandle { receiver.await.expect("IndexActor has been killed") } - pub async fn update(&self, meta: Processing, data: std::fs::File) -> UpdateResult { + pub async fn update( + &self, + meta: Processing, + data: std::fs::File, + ) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::Update { ret, meta, data }; let _ = self.read_sender.send(msg).await; - receiver.await.expect("IndexActor has been killed") + Ok(receiver.await.expect("IndexActor has been killed")?) } pub async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { @@ -441,7 +454,7 @@ impl IndexStore for MapIndexStore { let meta = match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(entry) => { let now = Utc::now(); - let meta = IndexMeta{ + let meta = IndexMeta { uuid, created_at: now.clone(), updated_at: now, @@ -478,7 +491,7 @@ impl IndexStore for MapIndexStore { Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { Entry::Vacant(meta_entry) => { let now = Utc::now(); - let meta = IndexMeta{ + let meta = IndexMeta { uuid, created_at: now.clone(), updated_at: now, @@ -491,12 +504,13 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) }) .await - .expect("thread died"); + .expect("thread died"); Ok(index_entry.insert(index?).clone()) } @@ -508,12 +522,13 @@ impl IndexStore for MapIndexStore { create_dir_all(&db_path).expect("can't create db"); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &db_path) + .map_err(|e| IndexError::Error(e))?; let index = Index(Arc::new(index)); Ok(index) }) .await - .expect("thread died"); + .expect("thread died"); Ok(index_entry.insert(index?).clone()) } @@ -540,11 +555,14 @@ impl IndexStore for MapIndexStore { } async fn update_index(&self, uuid: Uuid, f: F) -> Result - where F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static, + where + F: FnOnce(Index) -> Result + Send + Sync + 'static, + R: Sync + Send + 'static, { let index = self.get_or_create(uuid.clone(), None).await?; - let mut meta = self.get_meta(&uuid).await? + let mut meta = self + .get_meta(&uuid) + .await? .ok_or(IndexError::UnexistingIndex)?; match f(index) { Ok(r) => { @@ -552,7 +570,7 @@ impl IndexStore for MapIndexStore { self.meta_store.write().await.insert(uuid, meta); Ok(r) } - Err(e) => Err(e) + Err(e) => Err(e), } } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index bcf663ec8..0bae7f42d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -69,11 +69,11 @@ enum IndexControllerMsg { } impl IndexController { - pub fn new(path: impl AsRef) -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(&path); + let index_actor = index_actor::IndexActorHandle::new(&path)?; let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } + Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } pub async fn add_documents( diff --git a/src/index_controller/update_store.rs b/src/index_controller/update_store.rs index 02c15ed8c..b8f174a34 100644 --- a/src/index_controller/update_store.rs +++ b/src/index_controller/update_store.rs @@ -30,18 +30,18 @@ pub trait HandleUpdate { &mut self, meta: Processing, content: File, - ) -> Result, Failed>; + ) -> anyhow::Result, Failed>>; } impl HandleUpdate for F where - F: FnMut(Processing, File) -> Result, Failed>, + F: FnMut(Processing, File) -> anyhow::Result, Failed>>, { fn handle_update( &mut self, meta: Processing, content: File, - ) -> Result, Failed> { + ) -> anyhow::Result, Failed>> { self(meta, content) } } @@ -100,7 +100,7 @@ where update_store.process_pending_update(handler) }) .await - .unwrap(); + .expect("Fatal error processing update."); match res { Ok(Some(_)) => (), Ok(None) => break, @@ -185,7 +185,7 @@ where /// Executes the user provided function on the next pending update (the one with the lowest id). /// This is asynchronous as it let the user process the update with a read-only txn and /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, mut handler: U) -> heed::Result> + fn process_pending_update(&self, mut handler: U) -> anyhow::Result> where U: HandleUpdate, { @@ -209,7 +209,7 @@ where self.processing.write().unwrap().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. - let result = handler.handle_update(processing, file); + let result = handler.handle_update(processing, file)?; drop(rtxn); // Once the pending update have been successfully processed From 4e1597bd1dbbc9c1b10a85124f9a748be6031e26 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 8 Mar 2021 16:27:29 +0100 Subject: [PATCH 36/69] clean Uuid resolver actor --- src/index_controller/mod.rs | 38 ++++------- src/index_controller/uuid_resolver.rs | 93 +++++++++++++++++---------- 2 files changed, 74 insertions(+), 57 deletions(-) diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0bae7f42d..a824852bf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -10,7 +10,6 @@ use std::sync::Arc; use std::time::Duration; use actix_web::web::{Bytes, Payload}; -use anyhow::Context; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; @@ -108,7 +107,7 @@ impl IndexController { } pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?.unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; @@ -116,7 +115,7 @@ impl IndexController { } pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(index).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -153,8 +152,7 @@ impl IndexController { pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { let uuid = self.uuid_resolver .delete(index_uid) - .await? - .context("index not found")?; + .await?; self.update_handle.delete(uuid.clone()).await?; self.index_handle.delete(uuid).await?; Ok(()) @@ -163,16 +161,14 @@ impl IndexController { pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { let uuid = self.uuid_resolver .resolve(index) - .await? - .context("index not found")?; + .await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } pub async fn all_update_status(&self, index: String) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index).await? - .context("index not found")?; + .resolve(index).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -195,8 +191,7 @@ impl IndexController { pub async fn settings(&self, index: String) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) } @@ -210,8 +205,7 @@ impl IndexController { ) -> anyhow::Result> { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) } @@ -224,8 +218,7 @@ impl IndexController { ) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(index.clone()) - .await? - .with_context(|| format!("Index {:?} doesn't exist", index))?; + .await?; let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } @@ -235,21 +228,18 @@ impl IndexController { } pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); + let uuid = self.uuid_resolver.resolve(name).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } pub async fn get_index(&self, name: String) -> anyhow::Result> { let uuid = self.uuid_resolver.resolve(name.clone()).await?; - if let Some(uuid) = uuid { - let result = self.index_handle - .get_index_meta(uuid) - .await? - .map(|meta| IndexMetadata { name, meta }); - return Ok(result) - } - Ok(None) + let result = self.index_handle + .get_index_meta(uuid) + .await? + .map(|meta| IndexMetadata { name, meta }); + Ok(result) } } diff --git a/src/index_controller/uuid_resolver.rs b/src/index_controller/uuid_resolver.rs index e2539fdb2..8d33edef4 100644 --- a/src/index_controller/uuid_resolver.rs +++ b/src/index_controller/uuid_resolver.rs @@ -1,10 +1,10 @@ -use thiserror::Error; -use tokio::sync::{RwLock, mpsc, oneshot}; -use uuid::Uuid; +use log::{info, warn}; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::Arc; -use std::collections::hash_map::Entry; -use log::{info, warn}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot, RwLock}; +use uuid::Uuid; pub type Result = std::result::Result; @@ -12,7 +12,7 @@ pub type Result = std::result::Result; enum UuidResolveMsg { Resolve { name: String, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, GetOrCreate { name: String, @@ -24,7 +24,7 @@ enum UuidResolveMsg { }, Delete { name: String, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, List { ret: oneshot::Sender>>, @@ -46,13 +46,20 @@ impl UuidResolverActor { info!("uuid resolver started"); - // TODO: benchmark and use buffered streams to improve throughput. loop { match self.inbox.recv().await { - Some(Create { name, ret }) => self.handle_create(name, ret).await, - Some(GetOrCreate { name, ret }) => self.handle_get_or_create(name, ret).await, - Some(Resolve { name, ret }) => self.handle_resolve(name, ret).await, - Some(Delete { name, ret }) => self.handle_delete(name, ret).await, + Some(Create { name, ret }) => { + let _ = ret.send(self.handle_create(name).await); + } + Some(GetOrCreate { name, ret }) => { + let _ = ret.send(self.handle_get_or_create(name).await); + } + Some(Resolve { name, ret }) => { + let _ = ret.send(self.handle_resolve(name).await); + } + Some(Delete { name, ret }) => { + let _ = ret.send(self.handle_delete(name).await); + } Some(List { ret }) => { let _ = ret.send(self.handle_list().await); } @@ -64,24 +71,26 @@ impl UuidResolverActor { warn!("exiting uuid resolver loop"); } - async fn handle_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name, true).await; - let _ = ret.send(result); + async fn handle_create(&self, name: String) -> Result { + self.store.create_uuid(name, true).await } - async fn handle_get_or_create(&self, name: String, ret: oneshot::Sender>) { - let result = self.store.create_uuid(name, false).await; - let _ = ret.send(result); + async fn handle_get_or_create(&self, name: String) -> Result { + self.store.create_uuid(name, false).await } - async fn handle_resolve(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.get_uuid(&name).await; - let _ = ret.send(result); + async fn handle_resolve(&self, name: String) -> Result { + self.store + .get_uuid(&name) + .await? + .ok_or(UuidError::UnexistingIndex(name)) } - async fn handle_delete(&self, name: String, ret: oneshot::Sender>>) { - let result = self.store.delete(&name).await; - let _ = ret.send(result); + async fn handle_delete(&self, name: String) -> Result { + self.store + .delete(&name) + .await? + .ok_or(UuidError::UnexistingIndex(name)) } async fn handle_list(&self) -> Result> { @@ -104,39 +113,49 @@ impl UuidResolverHandle { Self { sender } } - pub async fn resolve(&self, name: String) -> anyhow::Result> { + pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Resolve { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn get_or_create(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::GetOrCreate { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Create { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } - pub async fn delete(&self, name: String) -> anyhow::Result> { + pub async fn delete(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::Delete { name, ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } pub async fn list(&self) -> anyhow::Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::List { ret }; let _ = self.sender.send(msg).await; - Ok(receiver.await.expect("Uuid resolver actor has been killed")?) + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) } } @@ -144,6 +163,8 @@ impl UuidResolverHandle { pub enum UuidError { #[error("Name already exist.")] NameAlreadyExist, + #[error("Index \"{0}\" doesn't exist.")] + UnexistingIndex(String), } #[async_trait::async_trait] @@ -168,7 +189,7 @@ impl UuidStore for MapUuidStore { } else { Ok(entry.get().clone()) } - }, + } Entry::Vacant(entry) => { let uuid = Uuid::new_v4(); let uuid = entry.insert(uuid); @@ -186,7 +207,13 @@ impl UuidStore for MapUuidStore { } async fn list(&self) -> Result> { - let list = self.0.read().await.iter().map(|(name, uuid)| (name.to_owned(), uuid.clone())).collect(); + let list = self + .0 + .read() + .await + .iter() + .map(|(name, uuid)| (name.to_owned(), uuid.clone())) + .collect(); Ok(list) } } From 562da9dd3fb06caf5eb4b0f2049f7926d7150119 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 11:56:51 +0100 Subject: [PATCH 37/69] fix test compilation --- Cargo.lock | 71 +++++------ Cargo.toml | 10 +- src/helpers/mod.rs | 4 +- src/index_controller/update_store.rs | 169 +++++++++++++-------------- src/lib.rs | 53 +++++++++ src/main.rs | 54 +-------- tests/common/index.rs | 4 +- tests/common/service.rs | 14 +-- 8 files changed, 180 insertions(+), 199 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbae20c98..0bdc739d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.0-beta.3" +version = "3.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a12706e793a92377f85cec219514b72625b3b89f9b4912d8bfb53ab6a615bf0" +checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" dependencies = [ "actix-codec 0.4.0-beta.1", "actix-rt 2.1.0", @@ -127,22 +127,21 @@ dependencies = [ "brotli2", "bytes 1.0.1", "bytestring", + "cfg-if 1.0.0", "cookie", "derive_more", "encoding_rs", "flate2", - "futures-channel", "futures-core", "futures-util", - "h2 0.3.0", + "h2 0.3.1", "http", "httparse", - "indexmap", "itoa", "language-tags", - "lazy_static", "log", "mime", + "once_cell", "percent-encoding", "pin-project 1.0.2", "rand 0.8.3", @@ -151,9 +150,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha-1 0.9.2", - "slab", "smallvec", "time 0.2.23", + "tokio 1.2.0", ] [[package]] @@ -324,12 +323,12 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.0-beta.3" +version = "4.0.0-beta.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9683dc8c3037ea524e0fec6032d34e1cb1ee72c4eb8689f428a60c2a544ea3" +checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" dependencies = [ "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.3", + "actix-http 3.0.0-beta.4", "actix-macros 0.2.0", "actix-router", "actix-rt 2.1.0", @@ -362,9 +361,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "0.5.0-beta.1" +version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8313dc4cbcae1785a7f14c3dfb7dfeb25fe96a03b20e5c38fe026786def5aa70" +checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2", "quote", @@ -500,12 +499,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "awc" -version = "3.0.0-beta.2" +version = "3.0.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7225ad81fbad09ef56ccc61e0688abe8494a68722c5d0df5e2fc8b724a200b" +checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" dependencies = [ "actix-codec 0.4.0-beta.1", - "actix-http 3.0.0-beta.3", + "actix-http 3.0.0-beta.4", "actix-rt 2.1.0", "actix-service 2.0.0-beta.4", "base64 0.13.0", @@ -513,9 +512,11 @@ dependencies = [ "cfg-if 1.0.0", "derive_more", "futures-core", + "itoa", "log", "mime", "percent-encoding", + "pin-project-lite 0.2.0", "rand 0.8.3", "rustls 0.19.0", "serde", @@ -1310,9 +1311,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" dependencies = [ "bytes 1.0.1", "fnv", @@ -1325,7 +1326,6 @@ dependencies = [ "tokio 1.2.0", "tokio-util 0.6.3", "tracing", - "tracing-futures", ] [[package]] @@ -1761,11 +1761,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "meilisearch" +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + +[[package]] +name = "meilisearch-http" version = "0.17.0" dependencies = [ "actix-cors", - "actix-service 1.0.6", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", "actix-web", "anyhow", "assert-json-diff", @@ -1813,19 +1822,11 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 0.2.24", "tokio 1.2.0", "uuid", "vergen", ] -[[package]] -name = "meilisearch-error" -version = "0.19.0" -dependencies = [ - "actix-http 2.2.0", -] - [[package]] name = "meilisearch-tokenizer" version = "0.1.1" @@ -3236,7 +3237,6 @@ dependencies = [ "pin-project-lite 0.1.11", "signal-hook-registry", "slab", - "tokio-macros 0.2.6", "winapi 0.3.9", ] @@ -3256,21 +3256,10 @@ dependencies = [ "parking_lot", "pin-project-lite 0.2.0", "signal-hook-registry", - "tokio-macros 1.1.0", + "tokio-macros", "winapi 0.3.9", ] -[[package]] -name = "tokio-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-macros" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9bc5e4d1d..792ebd763 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Quentin de Quelen ", "Clément Renault HandleUpdate for F - where - F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static, - { - fn handle_update( - &mut self, - meta: Processing, - content: &[u8], - ) -> Result, Failed> { - self(meta, content) - } - } + //#[test] + //fn simple() { + //let dir = tempfile::tempdir().unwrap(); + //let mut options = EnvOpenOptions::new(); + //options.map_size(4096 * 100); + //let update_store = UpdateStore::open( + //options, + //dir, + //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + //let new_meta = meta.meta().to_string() + " processed"; + //let processed = meta.process(new_meta); + //Ok(processed) + //}, + //) + //.unwrap(); - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open( - options, - dir, - |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }, - ) - .unwrap(); + //let meta = String::from("kiki"); + //let update = update_store.register_update(meta, &[]).unwrap(); + //thread::sleep(Duration::from_millis(100)); + //let meta = update_store.meta(update.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "kiki processed"); + //} else { + //panic!() + //} + //} - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } + //#[test] + //#[ignore] + //fn long_running_update() { + //let dir = tempfile::tempdir().unwrap(); + //let mut options = EnvOpenOptions::new(); + //options.map_size(4096 * 100); + //let update_store = UpdateStore::open( + //options, + //dir, + //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + //thread::sleep(Duration::from_millis(400)); + //let new_meta = meta.meta().to_string() + "processed"; + //let processed = meta.process(new_meta); + //Ok(processed) + //}, + //) + //.unwrap(); - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let update_store = UpdateStore::open( - options, - dir, - |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }, - ) - .unwrap(); + //let before_register = Instant::now(); - let before_register = Instant::now(); + //let meta = String::from("kiki"); + //let update_kiki = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //let meta = String::from("coco"); + //let update_coco = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //let meta = String::from("cucu"); + //let update_cucu = update_store.register_update(meta, &[]).unwrap(); + //assert!(before_register.elapsed() < Duration::from_millis(200)); - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); + //thread::sleep(Duration::from_millis(400 * 3 + 100)); - thread::sleep(Duration::from_millis(400 * 3 + 100)); + //let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "kiki processed"); + //} else { + //panic!() + //} - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } + //let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "coco processed"); + //} else { + //panic!() + //} - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} + //let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + //if let UpdateStatus::Processed(Processed { success, .. }) = meta { + //assert_eq!(success, "cucu processed"); + //} else { + //panic!() + //} + //} +//} diff --git a/src/lib.rs b/src/lib.rs index 32c8e4266..bd7379d7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,3 +13,56 @@ mod index_controller; pub use option::Opt; pub use self::data::Data; + +#[macro_export] +macro_rules! create_app { + ($data:expr, $enable_frontend:expr) => { + { + use actix_cors::Cors; + use actix_web::App; + use actix_web::middleware::TrailingSlash; + use actix_web::{web, middleware}; + use meilisearch_http::error::payload_error_handler; + use meilisearch_http::routes::*; + + let app = App::new() + .data($data.clone()) + .app_data( + web::JsonConfig::default() + .limit($data.http_payload_size_limit()) + .content_type(|_mime| true) // Accept all mime types + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()) + ) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); + //.configure(routes::dump::services); + let app = if $enable_frontend { + app + .service(load_html) + .service(load_css) + } else { + app + }; + app.wrap( + Cors::default() + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .max_age(86_400) // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + } + }; +} diff --git a/src/main.rs b/src/main.rs index 9e69c4091..36959c5b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,9 @@ use std::env; -use actix_cors::Cors; -use actix_web::{middleware, HttpServer, web, web::ServiceConfig}; +use actix_web::HttpServer; use main_error::MainError; -use meilisearch::{Data, Opt}; +use meilisearch_http::{Data, Opt, create_app}; use structopt::StructOpt; -use actix_web::App; -use meilisearch::error::payload_error_handler; -use actix_web::middleware::TrailingSlash; //mod analytics; @@ -83,38 +79,8 @@ async fn main() -> Result<(), MainError> { } async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { - use meilisearch::routes::*; - let http_server = HttpServer::new(move || { - let app = App::new() - .configure(|c| configure_data(c, &data)) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(stop_words::services) - .configure(synonym::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); - let app = if enable_frontend { - app - .service(meilisearch::routes::load_html) - .service(meilisearch::routes::load_css) - } else { - app - }; - app.wrap( - Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .max_age(86_400) // 24h - ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - }) + let http_server = HttpServer::new(move || create_app!(&data, enable_frontend)) // Disable signals allows the server to terminate immediately when a user enter CTRL-C .disable_signals(); @@ -129,20 +95,6 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box Ok(()) } -fn configure_data(config: &mut ServiceConfig, data: &Data) { - config - .data(data.clone()) - .app_data( - web::JsonConfig::default() - .limit(data.http_payload_size_limit()) - .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ); -} pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" diff --git a/tests/common/index.rs b/tests/common/index.rs index 710b96a9f..58a8de200 100644 --- a/tests/common/index.rs +++ b/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::delay_for; +use tokio::time::sleep; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/tests/common/service.rs b/tests/common/service.rs index 8e797c00d..feff49cad 100644 --- a/tests/common/service.rs +++ b/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::create_app; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .header("content-type", "application/json") + .insert_header(("content-type", "application/json")) .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; From 8061a04661017b8b0764ecd361c51920af824f01 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 13:38:30 +0100 Subject: [PATCH 38/69] add test assets --- tests/assets/test_set.json | 1613 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1613 insertions(+) create mode 100644 tests/assets/test_set.json diff --git a/tests/assets/test_set.json b/tests/assets/test_set.json new file mode 100644 index 000000000..63534c896 --- /dev/null +++ b/tests/assets/test_set.json @@ -0,0 +1,1613 @@ +[ + { + "id": 0, + "isActive": false, + "balance": "$2,668.55", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "email": "lucashess@chorizon.com", + "phone": "+1 (998) 478-2597", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered": "2016-06-21T09:30:25 -02:00", + "latitude": -44.174957, + "longitude": -145.725388, + "tags": [ + "bug", + "bug" + ] + }, + { + "id": 1, + "isActive": true, + "balance": "$1,706.13", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Cherry Orr", + "gender": "female", + "email": "cherryorr@chorizon.com", + "phone": "+1 (995) 479-3174", + "address": "442 Beverly Road, Ventress, New Mexico, 3361", + "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", + "registered": "2020-03-18T11:12:21 -01:00", + "latitude": -24.356932, + "longitude": 27.184808, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 2, + "isActive": true, + "balance": "$2,467.47", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Patricia Goff", + "gender": "female", + "email": "patriciagoff@chorizon.com", + "phone": "+1 (864) 463-2277", + "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", + "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", + "registered": "2014-10-28T12:59:30 -01:00", + "latitude": -64.008555, + "longitude": 11.867098, + "tags": [ + "good first issue" + ] + }, + { + "id": 3, + "isActive": true, + "balance": "$3,344.40", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Adeline Flynn", + "gender": "female", + "email": "adelineflynn@chorizon.com", + "phone": "+1 (994) 600-2840", + "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", + "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", + "registered": "2014-03-27T06:24:45 -01:00", + "latitude": -74.485173, + "longitude": -11.059859, + "tags": [ + "bug", + "good first issue", + "wontfix", + "new issue" + ] + }, + { + "id": 4, + "isActive": false, + "balance": "$2,575.78", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "Green", + "name": "Mariana Pacheco", + "gender": "female", + "email": "marianapacheco@chorizon.com", + "phone": "+1 (820) 414-2223", + "address": "664 Rapelye Street, Faywood, California, 7320", + "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", + "registered": "2015-09-02T03:23:35 -02:00", + "latitude": 75.763501, + "longitude": -78.777124, + "tags": [ + "new issue" + ] + }, + { + "id": 5, + "isActive": true, + "balance": "$3,793.09", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Warren Watson", + "gender": "male", + "email": "warrenwatson@chorizon.com", + "phone": "+1 (807) 583-2427", + "address": "671 Prince Street, Faxon, Connecticut, 4275", + "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", + "registered": "2017-06-04T06:02:17 -02:00", + "latitude": 29.979223, + "longitude": 25.358943, + "tags": [ + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 6, + "isActive": true, + "balance": "$2,919.70", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "blue", + "name": "Shelia Berry", + "gender": "female", + "email": "sheliaberry@chorizon.com", + "phone": "+1 (853) 511-2651", + "address": "437 Forrest Street, Coventry, Illinois, 2056", + "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", + "registered": "2018-07-11T02:45:01 -02:00", + "latitude": 54.815991, + "longitude": -118.690609, + "tags": [ + "good first issue", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 7, + "isActive": true, + "balance": "$1,349.50", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Chrystal Boyd", + "gender": "female", + "email": "chrystalboyd@chorizon.com", + "phone": "+1 (936) 563-2802", + "address": "670 Croton Loop, Sussex, Florida, 4692", + "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", + "registered": "2016-11-01T07:36:04 -01:00", + "latitude": -24.711933, + "longitude": 147.246705, + "tags": [] + }, + { + "id": 8, + "isActive": false, + "balance": "$3,999.56", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Martin Porter", + "gender": "male", + "email": "martinporter@chorizon.com", + "phone": "+1 (895) 580-2304", + "address": "577 Regent Place, Aguila, Guam, 6554", + "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", + "registered": "2014-09-20T02:08:30 -02:00", + "latitude": -88.344273, + "longitude": 37.964466, + "tags": [] + }, + { + "id": 9, + "isActive": true, + "balance": "$3,729.71", + "picture": "http://placehold.it/32x32", + "age": 26, + "color": "blue", + "name": "Kelli Mendez", + "gender": "female", + "email": "kellimendez@chorizon.com", + "phone": "+1 (936) 401-2236", + "address": "242 Caton Place, Grazierville, Alabama, 3968", + "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", + "registered": "2018-05-04T10:35:30 -02:00", + "latitude": 49.37551, + "longitude": 41.872323, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 10, + "isActive": false, + "balance": "$1,127.47", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "blue", + "name": "Maddox Johns", + "gender": "male", + "email": "maddoxjohns@chorizon.com", + "phone": "+1 (892) 470-2357", + "address": "756 Beard Street, Avalon, Louisiana, 114", + "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", + "registered": "2016-04-22T06:41:25 -02:00", + "latitude": 66.640229, + "longitude": -17.222666, + "tags": [ + "new issue", + "good first issue", + "good first issue", + "new issue" + ] + }, + { + "id": 11, + "isActive": true, + "balance": "$1,351.43", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Evans Wagner", + "gender": "male", + "email": "evanswagner@chorizon.com", + "phone": "+1 (889) 496-2332", + "address": "118 Monaco Place, Lutsen, Delaware, 6209", + "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", + "registered": "2016-10-27T01:26:31 -02:00", + "latitude": -77.673222, + "longitude": -142.657214, + "tags": [ + "good first issue", + "good first issue" + ] + }, + { + "id": 12, + "isActive": false, + "balance": "$3,394.96", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Aida Kirby", + "gender": "female", + "email": "aidakirby@chorizon.com", + "phone": "+1 (942) 532-2325", + "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", + "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", + "registered": "2018-06-18T04:39:57 -02:00", + "latitude": -58.062041, + "longitude": 34.999254, + "tags": [ + "new issue", + "wontfix", + "bug", + "new issue" + ] + }, + { + "id": 13, + "isActive": true, + "balance": "$2,812.62", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Nelda Burris", + "gender": "female", + "email": "neldaburris@chorizon.com", + "phone": "+1 (813) 600-2576", + "address": "160 Opal Court, Fowlerville, Tennessee, 2170", + "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", + "registered": "2015-08-15T12:39:53 -02:00", + "latitude": 66.6871, + "longitude": 179.549488, + "tags": [ + "wontfix" + ] + }, + { + "id": 14, + "isActive": true, + "balance": "$1,718.33", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Jennifer Hart", + "gender": "female", + "email": "jenniferhart@chorizon.com", + "phone": "+1 (850) 537-2513", + "address": "124 Veranda Place, Nash, Utah, 985", + "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", + "registered": "2016-09-04T11:46:59 -02:00", + "latitude": -66.827751, + "longitude": 99.220079, + "tags": [ + "wontfix", + "bug", + "new issue", + "new issue" + ] + }, + { + "id": 15, + "isActive": false, + "balance": "$2,698.16", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "blue", + "name": "Aurelia Contreras", + "gender": "female", + "email": "aureliacontreras@chorizon.com", + "phone": "+1 (932) 442-3103", + "address": "655 Dwight Street, Grapeview, Palau, 8356", + "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", + "registered": "2014-09-11T10:43:15 -02:00", + "latitude": -71.328973, + "longitude": 133.404895, + "tags": [ + "wontfix", + "bug", + "good first issue" + ] + }, + { + "id": 16, + "isActive": true, + "balance": "$3,303.25", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Estella Bass", + "gender": "female", + "email": "estellabass@chorizon.com", + "phone": "+1 (825) 436-2909", + "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", + "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", + "registered": "2017-11-23T09:32:09 -01:00", + "latitude": 81.17014, + "longitude": -145.262693, + "tags": [ + "new issue" + ] + }, + { + "id": 17, + "isActive": false, + "balance": "$3,579.20", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "brown", + "name": "Ortega Brennan", + "gender": "male", + "email": "ortegabrennan@chorizon.com", + "phone": "+1 (906) 526-2287", + "address": "440 Berry Street, Rivera, Maine, 1849", + "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", + "registered": "2016-03-31T02:17:13 -02:00", + "latitude": -68.407524, + "longitude": -113.642067, + "tags": [ + "new issue", + "wontfix" + ] + }, + { + "id": 18, + "isActive": false, + "balance": "$1,484.92", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Leonard Tillman", + "gender": "male", + "email": "leonardtillman@chorizon.com", + "phone": "+1 (864) 541-3456", + "address": "985 Provost Street, Charco, New Hampshire, 8632", + "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", + "registered": "2018-05-06T08:21:27 -02:00", + "latitude": -8.581801, + "longitude": -61.910062, + "tags": [ + "wontfix", + "new issue", + "bug", + "bug" + ] + }, + { + "id": 19, + "isActive": true, + "balance": "$3,572.55", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Dale Payne", + "gender": "male", + "email": "dalepayne@chorizon.com", + "phone": "+1 (814) 469-3499", + "address": "536 Dare Court, Ironton, Arkansas, 8605", + "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", + "registered": "2019-10-11T01:01:33 -02:00", + "latitude": -18.280968, + "longitude": -126.091797, + "tags": [ + "bug", + "wontfix", + "wontfix", + "wontfix" + ] + }, + { + "id": 20, + "isActive": true, + "balance": "$1,986.48", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Florence Long", + "gender": "female", + "email": "florencelong@chorizon.com", + "phone": "+1 (972) 557-3858", + "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", + "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", + "registered": "2016-05-02T09:18:59 -02:00", + "latitude": -27.110866, + "longitude": -45.09445, + "tags": [] + }, + { + "id": 21, + "isActive": true, + "balance": "$1,440.09", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "blue", + "name": "Levy Whitley", + "gender": "male", + "email": "levywhitley@chorizon.com", + "phone": "+1 (911) 458-2411", + "address": "187 Thomas Street, Hachita, North Carolina, 2989", + "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", + "registered": "2014-04-30T07:31:38 -02:00", + "latitude": -6.537315, + "longitude": 171.813536, + "tags": [ + "bug" + ] + }, + { + "id": 22, + "isActive": false, + "balance": "$2,938.57", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Bernard Mcfarland", + "gender": "male", + "email": "bernardmcfarland@chorizon.com", + "phone": "+1 (979) 442-3386", + "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", + "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", + "registered": "2017-08-10T10:07:59 -02:00", + "latitude": 63.766795, + "longitude": 68.177069, + "tags": [] + }, + { + "id": 23, + "isActive": true, + "balance": "$1,678.49", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "brown", + "name": "Blanca Mcclain", + "gender": "female", + "email": "blancamcclain@chorizon.com", + "phone": "+1 (976) 439-2772", + "address": "176 Crooke Avenue, Valle, Virginia, 5373", + "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", + "registered": "2015-10-12T11:57:28 -02:00", + "latitude": -8.944564, + "longitude": -150.711709, + "tags": [ + "bug", + "wontfix", + "good first issue" + ] + }, + { + "id": 24, + "isActive": true, + "balance": "$2,276.87", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Espinoza Ford", + "gender": "male", + "email": "espinozaford@chorizon.com", + "phone": "+1 (945) 429-3975", + "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", + "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", + "registered": "2014-03-26T02:16:08 -01:00", + "latitude": -37.137666, + "longitude": -51.811757, + "tags": [ + "wontfix", + "bug" + ] + }, + { + "id": 25, + "isActive": true, + "balance": "$3,973.43", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "Green", + "name": "Sykes Conley", + "gender": "male", + "email": "sykesconley@chorizon.com", + "phone": "+1 (851) 401-3916", + "address": "345 Grand Street, Woodlands, Missouri, 4461", + "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", + "registered": "2015-09-12T06:03:56 -02:00", + "latitude": 67.282955, + "longitude": -64.341323, + "tags": [ + "wontfix" + ] + }, + { + "id": 26, + "isActive": false, + "balance": "$1,431.50", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Barlow Duran", + "gender": "male", + "email": "barlowduran@chorizon.com", + "phone": "+1 (995) 436-2562", + "address": "481 Everett Avenue, Allison, Nebraska, 3065", + "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", + "registered": "2017-06-29T04:28:43 -02:00", + "latitude": -38.70606, + "longitude": 55.02816, + "tags": [ + "new issue" + ] + }, + { + "id": 27, + "isActive": true, + "balance": "$3,478.27", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "blue", + "name": "Schwartz Morgan", + "gender": "male", + "email": "schwartzmorgan@chorizon.com", + "phone": "+1 (861) 507-2067", + "address": "451 Lincoln Road, Fairlee, Washington, 2717", + "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", + "registered": "2016-05-10T08:34:54 -02:00", + "latitude": -75.886403, + "longitude": 93.044471, + "tags": [ + "bug", + "bug", + "wontfix", + "wontfix" + ] + }, + { + "id": 28, + "isActive": true, + "balance": "$2,825.59", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Kristy Leon", + "gender": "female", + "email": "kristyleon@chorizon.com", + "phone": "+1 (948) 465-2563", + "address": "594 Macon Street, Floris, South Dakota, 3565", + "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", + "registered": "2014-12-14T04:10:29 -01:00", + "latitude": -50.01615, + "longitude": -68.908804, + "tags": [ + "wontfix", + "good first issue" + ] + }, + { + "id": 29, + "isActive": false, + "balance": "$3,028.03", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "blue", + "name": "Ashley Pittman", + "gender": "male", + "email": "ashleypittman@chorizon.com", + "phone": "+1 (928) 507-3523", + "address": "646 Adelphi Street, Clara, Colorado, 6056", + "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", + "registered": "2016-01-07T10:40:48 -01:00", + "latitude": -58.766037, + "longitude": -124.828485, + "tags": [ + "wontfix" + ] + }, + { + "id": 30, + "isActive": true, + "balance": "$2,021.11", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Stacy Espinoza", + "gender": "female", + "email": "stacyespinoza@chorizon.com", + "phone": "+1 (999) 487-3253", + "address": "931 Alabama Avenue, Bangor, Alaska, 8215", + "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", + "registered": "2014-07-16T06:15:53 -02:00", + "latitude": 41.560197, + "longitude": 177.697, + "tags": [ + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 31, + "isActive": false, + "balance": "$3,609.82", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "blue", + "name": "Vilma Garza", + "gender": "female", + "email": "vilmagarza@chorizon.com", + "phone": "+1 (944) 585-2021", + "address": "565 Tech Place, Sedley, Puerto Rico, 858", + "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", + "registered": "2017-06-30T07:43:52 -02:00", + "latitude": -12.574889, + "longitude": -54.771186, + "tags": [ + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 32, + "isActive": false, + "balance": "$2,882.34", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "brown", + "name": "June Dunlap", + "gender": "female", + "email": "junedunlap@chorizon.com", + "phone": "+1 (997) 504-2937", + "address": "353 Cozine Avenue, Goodville, Indiana, 1438", + "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", + "registered": "2016-08-23T08:54:11 -02:00", + "latitude": -27.883363, + "longitude": -163.919683, + "tags": [ + "new issue", + "new issue", + "bug", + "wontfix" + ] + }, + { + "id": 33, + "isActive": true, + "balance": "$3,556.54", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Cecilia Greer", + "gender": "female", + "email": "ceciliagreer@chorizon.com", + "phone": "+1 (977) 573-3498", + "address": "696 Withers Street, Lydia, Oklahoma, 3220", + "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", + "registered": "2017-01-13T11:30:12 -01:00", + "latitude": 60.467215, + "longitude": 84.684575, + "tags": [ + "wontfix", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 34, + "isActive": true, + "balance": "$1,413.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "brown", + "name": "Mckay Schroeder", + "gender": "male", + "email": "mckayschroeder@chorizon.com", + "phone": "+1 (816) 480-3657", + "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", + "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", + "registered": "2016-02-08T04:50:15 -01:00", + "latitude": -72.413287, + "longitude": -159.254371, + "tags": [ + "good first issue" + ] + }, + { + "id": 35, + "isActive": true, + "balance": "$2,306.53", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Sawyer Mccormick", + "gender": "male", + "email": "sawyermccormick@chorizon.com", + "phone": "+1 (829) 569-3012", + "address": "749 Apollo Street, Eastvale, Texas, 7373", + "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", + "registered": "2019-11-30T11:53:23 -01:00", + "latitude": -48.978194, + "longitude": 110.950191, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 36, + "isActive": false, + "balance": "$1,844.54", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "brown", + "name": "Barbra Valenzuela", + "gender": "female", + "email": "barbravalenzuela@chorizon.com", + "phone": "+1 (992) 512-2649", + "address": "617 Schenck Court, Reinerton, Michigan, 2908", + "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", + "registered": "2019-03-29T01:59:31 -01:00", + "latitude": 45.193723, + "longitude": -12.486778, + "tags": [ + "new issue", + "new issue", + "wontfix", + "wontfix" + ] + }, + { + "id": 37, + "isActive": false, + "balance": "$3,469.82", + "picture": "http://placehold.it/32x32", + "age": 39, + "color": "brown", + "name": "Opal Weiss", + "gender": "female", + "email": "opalweiss@chorizon.com", + "phone": "+1 (809) 400-3079", + "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", + "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", + "registered": "2019-09-04T07:22:28 -02:00", + "latitude": 72.50376, + "longitude": 61.656435, + "tags": [ + "bug", + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 38, + "isActive": true, + "balance": "$1,992.38", + "picture": "http://placehold.it/32x32", + "age": 40, + "color": "Green", + "name": "Christina Short", + "gender": "female", + "email": "christinashort@chorizon.com", + "phone": "+1 (884) 589-2705", + "address": "594 Willmohr Street, Dexter, Montana, 660", + "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", + "registered": "2014-01-21T09:31:56 -01:00", + "latitude": -42.762739, + "longitude": 77.052349, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 39, + "isActive": false, + "balance": "$1,722.85", + "picture": "http://placehold.it/32x32", + "age": 29, + "color": "brown", + "name": "Golden Horton", + "gender": "male", + "email": "goldenhorton@chorizon.com", + "phone": "+1 (903) 426-2489", + "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", + "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", + "registered": "2015-08-19T02:56:41 -02:00", + "latitude": 69.922534, + "longitude": 9.881433, + "tags": [ + "bug" + ] + }, + { + "id": 40, + "isActive": false, + "balance": "$1,656.54", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Stafford Emerson", + "gender": "male", + "email": "staffordemerson@chorizon.com", + "phone": "+1 (992) 455-2573", + "address": "523 Thornton Street, Conway, Vermont, 6331", + "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", + "registered": "2019-02-16T04:07:08 -01:00", + "latitude": -29.143111, + "longitude": -57.207703, + "tags": [ + "wontfix", + "good first issue", + "good first issue" + ] + }, + { + "id": 41, + "isActive": false, + "balance": "$1,861.56", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "brown", + "name": "Salinas Gamble", + "gender": "male", + "email": "salinasgamble@chorizon.com", + "phone": "+1 (901) 525-2373", + "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", + "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", + "registered": "2017-08-21T05:47:53 -02:00", + "latitude": -22.593819, + "longitude": -63.613004, + "tags": [ + "good first issue", + "bug", + "bug", + "wontfix" + ] + }, + { + "id": 42, + "isActive": true, + "balance": "$3,179.74", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "brown", + "name": "Graciela Russell", + "gender": "female", + "email": "gracielarussell@chorizon.com", + "phone": "+1 (893) 464-3951", + "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", + "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", + "registered": "2015-05-18T09:52:56 -02:00", + "latitude": -14.634444, + "longitude": 12.931783, + "tags": [ + "wontfix", + "bug", + "wontfix" + ] + }, + { + "id": 43, + "isActive": true, + "balance": "$1,777.38", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "blue", + "name": "Arnold Bender", + "gender": "male", + "email": "arnoldbender@chorizon.com", + "phone": "+1 (945) 581-3808", + "address": "781 Lorraine Street, Gallina, American Samoa, 1832", + "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", + "registered": "2018-12-23T02:26:30 -01:00", + "latitude": 41.208579, + "longitude": 51.948925, + "tags": [ + "bug", + "good first issue", + "good first issue", + "wontfix" + ] + }, + { + "id": 44, + "isActive": true, + "balance": "$2,893.45", + "picture": "http://placehold.it/32x32", + "age": 22, + "color": "Green", + "name": "Joni Spears", + "gender": "female", + "email": "jonispears@chorizon.com", + "phone": "+1 (916) 565-2124", + "address": "307 Harwood Place, Canterwood, Maryland, 2047", + "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", + "registered": "2015-03-01T12:38:28 -01:00", + "latitude": 8.19071, + "longitude": 146.323808, + "tags": [ + "wontfix", + "new issue", + "good first issue", + "good first issue" + ] + }, + { + "id": 45, + "isActive": true, + "balance": "$2,830.36", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "brown", + "name": "Irene Bennett", + "gender": "female", + "email": "irenebennett@chorizon.com", + "phone": "+1 (904) 431-2211", + "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", + "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", + "registered": "2018-04-17T05:18:51 -02:00", + "latitude": -36.435177, + "longitude": -127.552573, + "tags": [ + "bug", + "wontfix" + ] + }, + { + "id": 46, + "isActive": true, + "balance": "$1,348.04", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "Green", + "name": "Lawson Curtis", + "gender": "male", + "email": "lawsoncurtis@chorizon.com", + "phone": "+1 (896) 532-2172", + "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", + "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", + "registered": "2016-08-23T01:41:09 -02:00", + "latitude": -48.783539, + "longitude": 20.492944, + "tags": [] + }, + { + "id": 47, + "isActive": true, + "balance": "$1,132.41", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goff May", + "gender": "male", + "email": "goffmay@chorizon.com", + "phone": "+1 (859) 453-3415", + "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", + "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", + "registered": "2014-10-25T07:32:30 -02:00", + "latitude": 13.079225, + "longitude": 76.215086, + "tags": [ + "bug" + ] + }, + { + "id": 48, + "isActive": true, + "balance": "$1,201.87", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Goodman Becker", + "gender": "male", + "email": "goodmanbecker@chorizon.com", + "phone": "+1 (825) 470-3437", + "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", + "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", + "registered": "2019-09-05T04:49:03 -02:00", + "latitude": -23.792094, + "longitude": -13.621221, + "tags": [ + "bug", + "bug", + "wontfix", + "new issue" + ] + }, + { + "id": 49, + "isActive": true, + "balance": "$1,476.39", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Maureen Dale", + "gender": "female", + "email": "maureendale@chorizon.com", + "phone": "+1 (984) 538-3684", + "address": "817 Newton Street, Bannock, Wyoming, 1468", + "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", + "registered": "2018-04-26T06:04:40 -02:00", + "latitude": -64.196802, + "longitude": -117.396238, + "tags": [ + "wontfix" + ] + }, + { + "id": 50, + "isActive": true, + "balance": "$1,947.08", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "Green", + "name": "Guerra Mcintyre", + "gender": "male", + "email": "guerramcintyre@chorizon.com", + "phone": "+1 (951) 536-2043", + "address": "423 Lombardy Street, Stewart, West Virginia, 908", + "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", + "registered": "2015-07-16T05:11:42 -02:00", + "latitude": 79.733743, + "longitude": -20.602356, + "tags": [ + "bug", + "good first issue", + "good first issue" + ] + }, + { + "id": 51, + "isActive": true, + "balance": "$2,960.90", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Key Cervantes", + "gender": "male", + "email": "keycervantes@chorizon.com", + "phone": "+1 (931) 474-3865", + "address": "410 Barbey Street, Vernon, Oregon, 2328", + "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", + "registered": "2019-12-15T12:13:35 -01:00", + "latitude": 47.627647, + "longitude": 117.049918, + "tags": [ + "new issue" + ] + }, + { + "id": 52, + "isActive": false, + "balance": "$1,884.02", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "blue", + "name": "Karen Nelson", + "gender": "female", + "email": "karennelson@chorizon.com", + "phone": "+1 (993) 528-3607", + "address": "930 Frank Court, Dunbar, New York, 8810", + "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", + "registered": "2014-06-23T09:21:44 -02:00", + "latitude": -59.059033, + "longitude": 76.565373, + "tags": [ + "new issue", + "bug" + ] + }, + { + "id": 53, + "isActive": true, + "balance": "$3,559.55", + "picture": "http://placehold.it/32x32", + "age": 32, + "color": "brown", + "name": "Caitlin Burnett", + "gender": "female", + "email": "caitlinburnett@chorizon.com", + "phone": "+1 (945) 480-2796", + "address": "516 Senator Street, Emory, Iowa, 4145", + "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", + "registered": "2019-01-09T02:26:31 -01:00", + "latitude": -82.774237, + "longitude": 42.316194, + "tags": [ + "bug", + "good first issue" + ] + }, + { + "id": 54, + "isActive": true, + "balance": "$2,113.29", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "Green", + "name": "Richards Walls", + "gender": "male", + "email": "richardswalls@chorizon.com", + "phone": "+1 (865) 517-2982", + "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", + "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", + "registered": "2014-09-25T06:51:22 -02:00", + "latitude": 80.09202, + "longitude": 87.49759, + "tags": [ + "wontfix", + "wontfix", + "bug" + ] + }, + { + "id": 55, + "isActive": true, + "balance": "$1,977.66", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "brown", + "name": "Combs Stanley", + "gender": "male", + "email": "combsstanley@chorizon.com", + "phone": "+1 (827) 419-2053", + "address": "153 Beverley Road, Siglerville, South Carolina, 3666", + "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", + "registered": "2019-08-22T07:53:15 -02:00", + "latitude": 78.386181, + "longitude": 143.661058, + "tags": [] + }, + { + "id": 56, + "isActive": false, + "balance": "$3,886.12", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Tucker Barry", + "gender": "male", + "email": "tuckerbarry@chorizon.com", + "phone": "+1 (808) 544-3433", + "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", + "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", + "registered": "2016-08-29T07:28:00 -02:00", + "latitude": 71.701551, + "longitude": 9.903068, + "tags": [] + }, + { + "id": 57, + "isActive": false, + "balance": "$1,844.56", + "picture": "http://placehold.it/32x32", + "age": 20, + "color": "Green", + "name": "Kaitlin Conner", + "gender": "female", + "email": "kaitlinconner@chorizon.com", + "phone": "+1 (862) 467-2666", + "address": "501 Knight Court, Joppa, Rhode Island, 274", + "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", + "registered": "2019-05-30T06:38:24 -02:00", + "latitude": 15.613464, + "longitude": 171.965629, + "tags": [] + }, + { + "id": 58, + "isActive": true, + "balance": "$2,876.10", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Mamie Fischer", + "gender": "female", + "email": "mamiefischer@chorizon.com", + "phone": "+1 (948) 545-3901", + "address": "599 Hunterfly Place, Haena, Georgia, 6005", + "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", + "registered": "2019-05-27T05:07:10 -02:00", + "latitude": 70.915079, + "longitude": -48.813584, + "tags": [ + "bug", + "wontfix", + "wontfix", + "good first issue" + ] + }, + { + "id": 59, + "isActive": true, + "balance": "$1,921.58", + "picture": "http://placehold.it/32x32", + "age": 31, + "color": "Green", + "name": "Harper Carson", + "gender": "male", + "email": "harpercarson@chorizon.com", + "phone": "+1 (912) 430-3243", + "address": "883 Dennett Place, Knowlton, New Mexico, 9219", + "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", + "registered": "2019-12-07T07:33:15 -01:00", + "latitude": -60.812605, + "longitude": -27.129016, + "tags": [ + "bug", + "new issue" + ] + }, + { + "id": 60, + "isActive": true, + "balance": "$1,770.93", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "brown", + "name": "Jody Herrera", + "gender": "female", + "email": "jodyherrera@chorizon.com", + "phone": "+1 (890) 583-3222", + "address": "261 Jay Street, Strykersville, Ohio, 9248", + "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", + "registered": "2016-05-21T01:00:02 -02:00", + "latitude": -36.846586, + "longitude": 131.156223, + "tags": [] + }, + { + "id": 61, + "isActive": false, + "balance": "$2,813.41", + "picture": "http://placehold.it/32x32", + "age": 37, + "color": "Green", + "name": "Charles Castillo", + "gender": "male", + "email": "charlescastillo@chorizon.com", + "phone": "+1 (934) 467-2108", + "address": "675 Morton Street, Rew, Pennsylvania, 137", + "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", + "registered": "2019-06-10T02:54:22 -02:00", + "latitude": -16.423202, + "longitude": -146.293752, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 62, + "isActive": true, + "balance": "$3,341.35", + "picture": "http://placehold.it/32x32", + "age": 33, + "color": "blue", + "name": "Estelle Ramirez", + "gender": "female", + "email": "estelleramirez@chorizon.com", + "phone": "+1 (816) 459-2073", + "address": "636 Nolans Lane, Camptown, California, 7794", + "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", + "registered": "2015-02-14T01:05:50 -01:00", + "latitude": -46.591249, + "longitude": -83.385587, + "tags": [ + "good first issue", + "bug" + ] + }, + { + "id": 63, + "isActive": true, + "balance": "$2,478.30", + "picture": "http://placehold.it/32x32", + "age": 21, + "color": "blue", + "name": "Knowles Hebert", + "gender": "male", + "email": "knowleshebert@chorizon.com", + "phone": "+1 (819) 409-2308", + "address": "361 Kathleen Court, Gratton, Connecticut, 7254", + "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", + "registered": "2016-03-08T08:34:52 -01:00", + "latitude": 71.042482, + "longitude": 152.460406, + "tags": [ + "good first issue", + "wontfix" + ] + }, + { + "id": 64, + "isActive": false, + "balance": "$2,559.09", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Thelma Mckenzie", + "gender": "female", + "email": "thelmamckenzie@chorizon.com", + "phone": "+1 (941) 596-2777", + "address": "202 Leonard Street, Riverton, Illinois, 8577", + "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", + "registered": "2020-04-14T12:43:06 -02:00", + "latitude": 16.026129, + "longitude": 105.464476, + "tags": [] + }, + { + "id": 65, + "isActive": true, + "balance": "$1,025.08", + "picture": "http://placehold.it/32x32", + "age": 34, + "color": "blue", + "name": "Carole Rowland", + "gender": "female", + "email": "carolerowland@chorizon.com", + "phone": "+1 (862) 558-3448", + "address": "941 Melba Court, Bluetown, Florida, 9555", + "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", + "registered": "2014-12-01T05:55:35 -01:00", + "latitude": -0.191998, + "longitude": 43.389652, + "tags": [ + "wontfix" + ] + }, + { + "id": 66, + "isActive": true, + "balance": "$1,061.49", + "picture": "http://placehold.it/32x32", + "age": 35, + "color": "brown", + "name": "Higgins Aguilar", + "gender": "male", + "email": "higginsaguilar@chorizon.com", + "phone": "+1 (911) 540-3791", + "address": "132 Sackman Street, Layhill, Guam, 8729", + "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", + "registered": "2015-04-05T02:10:07 -02:00", + "latitude": 74.702813, + "longitude": 151.314972, + "tags": [ + "bug" + ] + }, + { + "id": 67, + "isActive": true, + "balance": "$3,510.14", + "picture": "http://placehold.it/32x32", + "age": 28, + "color": "brown", + "name": "Ilene Gillespie", + "gender": "female", + "email": "ilenegillespie@chorizon.com", + "phone": "+1 (937) 575-2676", + "address": "835 Lake Street, Naomi, Alabama, 4131", + "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", + "registered": "2015-06-28T09:41:45 -02:00", + "latitude": 71.573342, + "longitude": -95.295989, + "tags": [ + "wontfix", + "wontfix" + ] + }, + { + "id": 68, + "isActive": false, + "balance": "$1,539.98", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Angelina Dyer", + "gender": "female", + "email": "angelinadyer@chorizon.com", + "phone": "+1 (948) 574-3949", + "address": "575 Division Place, Gorham, Louisiana, 3458", + "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", + "registered": "2014-07-08T06:34:36 -02:00", + "latitude": -85.649593, + "longitude": 66.126018, + "tags": [ + "good first issue" + ] + }, + { + "id": 69, + "isActive": true, + "balance": "$3,367.69", + "picture": "http://placehold.it/32x32", + "age": 30, + "color": "brown", + "name": "Marks Burt", + "gender": "male", + "email": "marksburt@chorizon.com", + "phone": "+1 (895) 497-3138", + "address": "819 Village Road, Wadsworth, Delaware, 6099", + "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", + "registered": "2014-08-31T06:12:18 -02:00", + "latitude": 26.854112, + "longitude": -143.313948, + "tags": [ + "good first issue" + ] + }, + { + "id": 70, + "isActive": false, + "balance": "$3,755.72", + "picture": "http://placehold.it/32x32", + "age": 23, + "color": "blue", + "name": "Glass Perkins", + "gender": "male", + "email": "glassperkins@chorizon.com", + "phone": "+1 (923) 486-3725", + "address": "899 Roosevelt Court, Belleview, Idaho, 1737", + "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", + "registered": "2015-05-22T05:44:33 -02:00", + "latitude": 54.27147, + "longitude": -65.065604, + "tags": [ + "wontfix" + ] + }, + { + "id": 71, + "isActive": true, + "balance": "$3,381.63", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "Green", + "name": "Candace Sawyer", + "gender": "female", + "email": "candacesawyer@chorizon.com", + "phone": "+1 (830) 404-2636", + "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", + "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", + "registered": "2014-04-04T08:45:00 -02:00", + "latitude": 6.484262, + "longitude": -37.054928, + "tags": [ + "new issue", + "new issue" + ] + }, + { + "id": 72, + "isActive": true, + "balance": "$1,640.98", + "picture": "http://placehold.it/32x32", + "age": 27, + "color": "Green", + "name": "Hendricks Martinez", + "gender": "male", + "email": "hendricksmartinez@chorizon.com", + "phone": "+1 (857) 566-3245", + "address": "636 Agate Court, Newry, Utah, 3304", + "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", + "registered": "2018-06-15T10:36:11 -02:00", + "latitude": 86.746034, + "longitude": 10.347893, + "tags": [ + "new issue" + ] + }, + { + "id": 73, + "isActive": false, + "balance": "$1,239.74", + "picture": "http://placehold.it/32x32", + "age": 38, + "color": "blue", + "name": "Eleanor Shepherd", + "gender": "female", + "email": "eleanorshepherd@chorizon.com", + "phone": "+1 (894) 567-2617", + "address": "670 Lafayette Walk, Darlington, Palau, 8803", + "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", + "registered": "2020-02-29T12:15:28 -01:00", + "latitude": 35.749621, + "longitude": -94.40842, + "tags": [ + "good first issue", + "new issue", + "new issue", + "bug" + ] + }, + { + "id": 74, + "isActive": true, + "balance": "$1,180.90", + "picture": "http://placehold.it/32x32", + "age": 36, + "color": "Green", + "name": "Stark Wong", + "gender": "male", + "email": "starkwong@chorizon.com", + "phone": "+1 (805) 575-3055", + "address": "522 Bond Street, Bawcomville, Wisconsin, 324", + "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", + "registered": "2020-01-25T10:47:48 -01:00", + "latitude": -80.452139, + "longitude": 160.72546, + "tags": [ + "wontfix" + ] + }, + { + "id": 75, + "isActive": false, + "balance": "$1,913.42", + "picture": "http://placehold.it/32x32", + "age": 24, + "color": "Green", + "name": "Emma Jacobs", + "gender": "female", + "email": "emmajacobs@chorizon.com", + "phone": "+1 (899) 554-3847", + "address": "173 Tapscott Street, Esmont, Maine, 7450", + "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", + "registered": "2019-03-29T06:24:13 -01:00", + "latitude": -35.53722, + "longitude": 155.703874, + "tags": [] + }, + { + "id": 76, + "isActive": false, + "balance": "$1,274.29", + "picture": "http://placehold.it/32x32", + "age": 25, + "color": "Green", + "name": "Clarice Gardner", + "gender": "female", + "email": "claricegardner@chorizon.com", + "phone": "+1 (810) 407-3258", + "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", + "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", + "registered": "2014-10-20T10:13:32 -02:00", + "latitude": 17.11935, + "longitude": 65.38197, + "tags": [ + "new issue", + "wontfix" + ] + } +] From 5ecf514d28a2e9258debee05046941e1f9dca5cc Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 13:46:49 +0100 Subject: [PATCH 39/69] restructure project --- .gitignore | 8 +- Cargo.lock | 496 +-- Cargo.toml | 88 +- Dockerfile | 29 + meilisearch-error/Cargo.toml | 8 + meilisearch-error/src/lib.rs | 185 + meilisearch-http/Cargo.lock | 3804 +++++++++++++++++ meilisearch-http/Cargo.toml | 82 + build.rs => meilisearch-http/build.rs | 0 .../public}/bulma.min.css | 0 .../public}/interface.html | 0 {src => meilisearch-http/src}/analytics.rs | 0 {src => meilisearch-http/src}/data/mod.rs | 0 {src => meilisearch-http/src}/data/search.rs | 0 {src => meilisearch-http/src}/data/updates.rs | 0 {src => meilisearch-http/src}/dump.rs | 0 {src => meilisearch-http/src}/error.rs | 0 .../src}/helpers/authentication.rs | 0 .../src}/helpers/compression.rs | 0 {src => meilisearch-http/src}/helpers/mod.rs | 0 .../src}/helpers/normalize_path.rs | 0 {src => meilisearch-http/src}/index/mod.rs | 0 {src => meilisearch-http/src}/index/search.rs | 0 .../src}/index/updates.rs | 0 .../actor_index_controller/mod.rs | 0 .../src}/index_controller/index_actor.rs | 0 .../local_index_controller/index_store.rs | 0 .../local_index_controller/mod.rs | 0 .../src}/index_controller/mod.rs | 0 .../src}/index_controller/update_actor.rs | 0 .../src}/index_controller/update_handler.rs | 0 .../src}/index_controller/update_store.rs | 0 .../src}/index_controller/updates.rs | 0 .../src}/index_controller/uuid_resolver.rs | 0 {src => meilisearch-http/src}/lib.rs | 0 {src => meilisearch-http/src}/main.rs | 0 {src => meilisearch-http/src}/option.rs | 0 .../src}/routes/document.rs | 0 {src => meilisearch-http/src}/routes/dump.rs | 0 .../src}/routes/health.rs | 0 {src => meilisearch-http/src}/routes/index.rs | 0 {src => meilisearch-http/src}/routes/key.rs | 0 {src => meilisearch-http/src}/routes/mod.rs | 0 .../src}/routes/search.rs | 0 .../settings/attributes_for_faceting.rs | 0 .../routes/settings/displayed_attributes.rs | 0 .../routes/settings/distinct_attributes.rs | 0 .../src}/routes/settings/mod.rs | 0 .../src}/routes/settings/ranking_rules.rs | 0 .../routes/settings/searchable_attributes.rs | 0 .../src}/routes/settings/stop_words.rs | 0 .../src}/routes/settings/synonyms.rs | 0 {src => meilisearch-http/src}/routes/stats.rs | 0 .../src}/routes/stop_words.rs | 0 .../src}/routes/synonym.rs | 0 {src => meilisearch-http/src}/snapshot.rs | 0 .../tests}/assets/test_set.json | 0 .../tests}/common/index.rs | 0 .../tests}/common/mod.rs | 0 .../tests}/common/server.rs | 0 .../tests}/common/service.rs | 0 .../tests}/documents/add_documents.rs | 0 .../tests}/documents/delete_documents.rs | 0 .../tests}/documents/get_documents.rs | 0 .../tests}/documents/mod.rs | 0 .../tests}/index/create_index.rs | 0 .../tests}/index/delete_index.rs | 0 .../tests}/index/get_index.rs | 0 .../tests}/index/mod.rs | 0 .../tests}/index/update_index.rs | 0 .../tests}/integration.rs | 0 .../tests}/search/mod.rs | 0 .../tests}/settings/get_settings.rs | 0 .../tests}/settings/mod.rs | 0 .../tests}/updates/mod.rs | 0 75 files changed, 4377 insertions(+), 323 deletions(-) create mode 100644 Dockerfile create mode 100644 meilisearch-error/Cargo.toml create mode 100644 meilisearch-error/src/lib.rs create mode 100644 meilisearch-http/Cargo.lock create mode 100644 meilisearch-http/Cargo.toml rename build.rs => meilisearch-http/build.rs (100%) rename {public => meilisearch-http/public}/bulma.min.css (100%) rename {public => meilisearch-http/public}/interface.html (100%) rename {src => meilisearch-http/src}/analytics.rs (100%) rename {src => meilisearch-http/src}/data/mod.rs (100%) rename {src => meilisearch-http/src}/data/search.rs (100%) rename {src => meilisearch-http/src}/data/updates.rs (100%) rename {src => meilisearch-http/src}/dump.rs (100%) rename {src => meilisearch-http/src}/error.rs (100%) rename {src => meilisearch-http/src}/helpers/authentication.rs (100%) rename {src => meilisearch-http/src}/helpers/compression.rs (100%) rename {src => meilisearch-http/src}/helpers/mod.rs (100%) rename {src => meilisearch-http/src}/helpers/normalize_path.rs (100%) rename {src => meilisearch-http/src}/index/mod.rs (100%) rename {src => meilisearch-http/src}/index/search.rs (100%) rename {src => meilisearch-http/src}/index/updates.rs (100%) rename {src => meilisearch-http/src}/index_controller/actor_index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/index_actor.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/index_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/local_index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/mod.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_actor.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_handler.rs (100%) rename {src => meilisearch-http/src}/index_controller/update_store.rs (100%) rename {src => meilisearch-http/src}/index_controller/updates.rs (100%) rename {src => meilisearch-http/src}/index_controller/uuid_resolver.rs (100%) rename {src => meilisearch-http/src}/lib.rs (100%) rename {src => meilisearch-http/src}/main.rs (100%) rename {src => meilisearch-http/src}/option.rs (100%) rename {src => meilisearch-http/src}/routes/document.rs (100%) rename {src => meilisearch-http/src}/routes/dump.rs (100%) rename {src => meilisearch-http/src}/routes/health.rs (100%) rename {src => meilisearch-http/src}/routes/index.rs (100%) rename {src => meilisearch-http/src}/routes/key.rs (100%) rename {src => meilisearch-http/src}/routes/mod.rs (100%) rename {src => meilisearch-http/src}/routes/search.rs (100%) rename {src => meilisearch-http/src}/routes/settings/attributes_for_faceting.rs (100%) rename {src => meilisearch-http/src}/routes/settings/displayed_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/distinct_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/mod.rs (100%) rename {src => meilisearch-http/src}/routes/settings/ranking_rules.rs (100%) rename {src => meilisearch-http/src}/routes/settings/searchable_attributes.rs (100%) rename {src => meilisearch-http/src}/routes/settings/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/settings/synonyms.rs (100%) rename {src => meilisearch-http/src}/routes/stats.rs (100%) rename {src => meilisearch-http/src}/routes/stop_words.rs (100%) rename {src => meilisearch-http/src}/routes/synonym.rs (100%) rename {src => meilisearch-http/src}/snapshot.rs (100%) rename {tests => meilisearch-http/tests}/assets/test_set.json (100%) rename {tests => meilisearch-http/tests}/common/index.rs (100%) rename {tests => meilisearch-http/tests}/common/mod.rs (100%) rename {tests => meilisearch-http/tests}/common/server.rs (100%) rename {tests => meilisearch-http/tests}/common/service.rs (100%) rename {tests => meilisearch-http/tests}/documents/add_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/delete_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/get_documents.rs (100%) rename {tests => meilisearch-http/tests}/documents/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/create_index.rs (100%) rename {tests => meilisearch-http/tests}/index/delete_index.rs (100%) rename {tests => meilisearch-http/tests}/index/get_index.rs (100%) rename {tests => meilisearch-http/tests}/index/mod.rs (100%) rename {tests => meilisearch-http/tests}/index/update_index.rs (100%) rename {tests => meilisearch-http/tests}/integration.rs (100%) rename {tests => meilisearch-http/tests}/search/mod.rs (100%) rename {tests => meilisearch-http/tests}/settings/get_settings.rs (100%) rename {tests => meilisearch-http/tests}/settings/mod.rs (100%) rename {tests => meilisearch-http/tests}/updates/mod.rs (100%) diff --git a/.gitignore b/.gitignore index 1d71f78ca..e1f56a99c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -# the milli project is a library /target +meilisearch-core/target +**/*.csv +**/*.json_lines +**/*.rs.bk +/*.mdb +/query-history.txt +/data.ms diff --git a/Cargo.lock b/Cargo.lock index 0bdc739d5..7f8724ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "actix-codec" version = "0.3.0" @@ -14,7 +12,7 @@ dependencies = [ "futures-sink", "log", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-util 0.3.1", ] @@ -29,9 +27,9 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", - "tokio-util 0.6.3", + "pin-project-lite 0.2.6", + "tokio 1.3.0", + "tokio-util 0.6.4", ] [[package]] @@ -99,15 +97,15 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project 1.0.2", + "pin-project 1.0.5", "rand 0.7.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "slab", - "time 0.2.23", + "time 0.2.25", ] [[package]] @@ -121,7 +119,7 @@ dependencies = [ "actix-service 2.0.0-beta.4", "actix-tls", "actix-utils 3.0.0-beta.2", - "ahash 0.7.0", + "ahash 0.7.2", "base64 0.13.0", "bitflags", "brotli2", @@ -143,16 +141,16 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "pin-project 1.0.2", + "pin-project 1.0.5", "rand 0.8.3", "regex", "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.2", + "sha-1 0.9.4", "smallvec", - "time 0.2.23", - "tokio 1.2.0", + "time 0.2.25", + "tokio 1.3.0", ] [[package]] @@ -200,7 +198,7 @@ dependencies = [ "futures-channel", "futures-util", "smallvec", - "tokio 0.2.24", + "tokio 0.2.25", ] [[package]] @@ -211,7 +209,7 @@ checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" dependencies = [ "actix-macros 0.2.0", "futures-core", - "tokio 1.2.0", + "tokio 1.3.0", ] [[package]] @@ -229,7 +227,7 @@ dependencies = [ "mio 0.7.9", "num_cpus", "slab", - "tokio 1.2.0", + "tokio 1.3.0", ] [[package]] @@ -249,7 +247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" dependencies = [ "futures-core", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", ] [[package]] @@ -282,7 +280,7 @@ dependencies = [ "http", "log", "tokio-rustls 0.22.0", - "tokio-util 0.6.3", + "tokio-util 0.6.4", "webpki-roots 0.21.0", ] @@ -318,7 +316,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", ] [[package]] @@ -337,7 +335,7 @@ dependencies = [ "actix-tls", "actix-utils 3.0.0-beta.2", "actix-web-codegen", - "ahash 0.7.0", + "ahash 0.7.2", "awc", "bytes 1.0.1", "derive_more", @@ -347,7 +345,7 @@ dependencies = [ "futures-util", "log", "mime", - "pin-project 1.0.2", + "pin-project 1.0.5", "regex", "rustls 0.19.0", "serde", @@ -355,7 +353,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.2.23", + "time 0.2.25", "url", ] @@ -372,18 +370,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" @@ -393,9 +391,9 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -437,15 +435,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.2.6", + "tokio 0.2.25", ] [[package]] @@ -471,9 +469,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2", "quote", @@ -516,7 +514,7 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "rand 0.8.3", "rustls 0.19.0", "serde", @@ -526,9 +524,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -590,7 +588,7 @@ dependencies = [ "block-padding", "byte-tools", "byteorder", - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -645,9 +643,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byte-tools" @@ -672,9 +670,9 @@ checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -705,9 +703,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" dependencies = [ "jobserver", ] @@ -773,18 +771,18 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" [[package]] name = "cookie" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "percent-encoding", - "time 0.2.23", + "time 0.2.25", "version_check", ] @@ -822,7 +820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] @@ -833,18 +831,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "memoffset", "scopeguard", @@ -871,9 +868,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -882,9 +879,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -945,7 +942,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -971,9 +968,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] @@ -1005,12 +1002,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.0.1", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -1046,13 +1043,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "winapi 0.3.9", ] @@ -1076,9 +1073,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", @@ -1120,9 +1117,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" dependencies = [ "futures-channel", "futures-core", @@ -1135,9 +1132,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" dependencies = [ "futures-core", "futures-sink", @@ -1145,15 +1142,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" dependencies = [ "futures-core", "futures-task", @@ -1162,15 +1159,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -1180,24 +1177,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" -dependencies = [ - "once_cell", -] +checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" dependencies = [ "futures-channel", "futures-core", @@ -1206,7 +1200,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite 0.2.6", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1224,9 +1218,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" dependencies = [ "typenum", ] @@ -1243,11 +1237,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1303,7 +1297,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-util 0.3.1", "tracing", "tracing-futures", @@ -1323,8 +1317,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.2.0", - "tokio-util 0.6.3", + "tokio 1.3.0", + "tokio-util 0.6.4", "tracing", ] @@ -1393,9 +1387,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -1434,9 +1428,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" [[package]] name = "httpdate" @@ -1461,15 +1455,15 @@ dependencies = [ [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.9" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ "bytes 0.5.6", "futures-channel", @@ -1481,9 +1475,9 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", + "pin-project 1.0.5", "socket2", - "tokio 0.2.24", + "tokio 0.2.25", "tower-service", "tracing", "want", @@ -1500,16 +1494,16 @@ dependencies = [ "hyper", "log", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "webpki", ] [[package]] name = "idna" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" dependencies = [ "matches", "unicode-bidi", @@ -1532,9 +1526,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown 0.9.1", @@ -1597,9 +1591,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jemalloc-sys" @@ -1648,9 +1642,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" dependencies = [ "wasm-bindgen", ] @@ -1688,9 +1682,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" [[package]] name = "linked-hash-map" @@ -1787,7 +1781,7 @@ dependencies = [ "crossbeam-channel", "dashmap", "either", - "env_logger 0.8.2", + "env_logger 0.8.3", "flate2", "fst", "futures", @@ -1822,7 +1816,7 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "tokio 1.2.0", + "tokio 1.3.0", "uuid", "vergen", ] @@ -1830,7 +1824,7 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#31ba3ff4a15501f12b7d37ac64ddce7c35a9757c" dependencies = [ "character_converter", "cow-utils", @@ -1926,9 +1920,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", @@ -2062,9 +2056,9 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" [[package]] name = "obkv" @@ -2074,9 +2068,9 @@ checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.5.2" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "opaque-debug" @@ -2122,14 +2116,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi 0.3.9", ] @@ -2240,11 +2234,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal 1.0.5", ] [[package]] @@ -2260,9 +2254,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2", "quote", @@ -2271,15 +2265,15 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2331,9 +2325,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" @@ -2352,9 +2346,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] @@ -2378,7 +2372,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -2439,7 +2433,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -2507,7 +2501,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.3", "lazy_static", "num_cpus", ] @@ -2521,12 +2515,6 @@ dependencies = [ "rand_core 0.3.1", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.5" @@ -2594,12 +2582,12 @@ dependencies = [ "mime", "mime_guess", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", - "tokio 0.2.24", + "tokio 0.2.25", "tokio-rustls 0.14.1", "url", "wasm-bindgen", @@ -2627,9 +2615,9 @@ checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" [[package]] name = "ring" -version = "0.16.19" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", @@ -2663,7 +2651,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -2720,7 +2717,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2729,6 +2735,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sentry" version = "0.18.1" @@ -2747,7 +2762,7 @@ dependencies = [ "rand 0.7.3", "regex", "reqwest", - "rustc_version", + "rustc_version 0.2.3", "sentry-types", "uname", "url", @@ -2770,18 +2785,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2", "quote", @@ -2802,9 +2817,9 @@ dependencies = [ [[package]] name = "serde_url_params" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" dependencies = [ "serde", "url", @@ -2836,9 +2851,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2855,9 +2870,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -2921,9 +2936,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "snap" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" +checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" [[package]] name = "socket2" @@ -2944,9 +2959,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8" dependencies = [ "version_check", ] @@ -2958,7 +2973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -3032,9 +3047,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" dependencies = [ "proc-macro2", "quote", @@ -3064,13 +3079,12 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "c0bcfbd6a598361fda270d82469fff3d65089dc33e175c9a131f7b4cd395f228" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -3093,7 +3107,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall", "remove_dir_all", "winapi 0.3.9", ] @@ -3138,11 +3152,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -3167,9 +3181,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" dependencies = [ "const_fn", "libc", @@ -3205,9 +3219,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" dependencies = [ "tinyvec_macros", ] @@ -3220,9 +3234,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", "fnv", @@ -3234,7 +3248,7 @@ dependencies = [ "mio 0.6.23", "mio-uds", "num_cpus", - "pin-project-lite 0.1.11", + "pin-project-lite 0.1.12", "signal-hook-registry", "slab", "winapi 0.3.9", @@ -3242,9 +3256,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" dependencies = [ "autocfg", "bytes 1.0.1", @@ -3254,7 +3268,7 @@ dependencies = [ "num_cpus", "once_cell", "parking_lot", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "signal-hook-registry", "tokio-macros", "winapi 0.3.9", @@ -3279,7 +3293,7 @@ checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls 0.18.1", - "tokio 0.2.24", + "tokio 0.2.25", "webpki", ] @@ -3290,7 +3304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls 0.19.0", - "tokio 1.2.0", + "tokio 1.3.0", "webpki", ] @@ -3304,39 +3318,39 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.1.11", - "tokio 0.2.24", + "pin-project-lite 0.1.12", + "tokio 0.2.25", ] [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29" dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.0", - "tokio 1.2.0", + "pin-project-lite 0.2.6", + "tokio 1.3.0", ] [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.6", "tracing-core", ] @@ -3351,22 +3365,22 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project 1.0.5", "tracing", ] [[package]] name = "trust-dns-proto" -version = "0.19.6" +version = "0.19.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +checksum = "1cad71a0c0d68ab9941d2fb6e82f8fb2e86d9945b94e1661dd0aaea2b88215a9" dependencies = [ "async-trait", - "backtrace", + "cfg-if 1.0.0", "enum-as-inner", "futures", "idna", @@ -3375,17 +3389,16 @@ dependencies = [ "rand 0.7.3", "smallvec", "thiserror", - "tokio 0.2.24", + "tokio 0.2.25", "url", ] [[package]] name = "trust-dns-resolver" -version = "0.19.6" +version = "0.19.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" dependencies = [ - "backtrace", "cfg-if 0.1.10", "futures", "ipconfig", @@ -3395,7 +3408,7 @@ dependencies = [ "resolv-conf", "smallvec", "thiserror", - "tokio 0.2.24", + "tokio 0.2.25", "trust-dns-proto", ] @@ -3446,9 +3459,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] @@ -3479,9 +3492,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", "idna", @@ -3514,12 +3527,13 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", + "rustc_version 0.3.3", ] [[package]] @@ -3552,9 +3566,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3564,9 +3578,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" dependencies = [ "bumpalo", "lazy_static", @@ -3579,9 +3593,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3591,9 +3605,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3601,9 +3615,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ "proc-macro2", "quote", @@ -3614,15 +3628,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 792ebd763..a1dca038e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,82 +1,8 @@ -[package] -authors = ["Quentin de Quelen ", "Clément Renault "] -description = "MeiliSearch HTTP server" -edition = "2018" -license = "MIT" -name = "meilisearch-http" -version = "0.17.0" -[[bin]] -name = "meilisearch" -path = "src/main.rs" +[workspace] +members = [ + "meilisearch-http", + "meilisearch-error", +] -[build-dependencies] -vergen = "3.1.0" - -[dependencies] -actix-cors = { path = "../actix-extras/actix-cors" } -actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } -actix-service = "2.0.0-beta.4" -actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } -#actix-web = { version = "3", features = ["rustls"] } -anyhow = "1.0.36" -async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } -byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -bytes = "0.6.0" -chrono = { version = "0.4.19", features = ["serde"] } -crossbeam-channel = "0.5.0" -env_logger = "0.8.2" -flate2 = "1.0.19" -fst = "0.4.5" -futures = "0.3.7" -futures-util = "0.3.8" -grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" -http = "0.2.1" -indexmap = { version = "1.3.2", features = ["serde-1"] } -log = "0.4.8" -main_error = "0.1.0" -meilisearch-error = { path = "../MeiliSearch/meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } -memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } -mime = "0.3.16" -once_cell = "1.5.2" -rand = "0.7.3" -rayon = "1.5.0" -regex = "1.4.2" -rustls = "0.19" -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0.59", features = ["preserve_order"] } -sha2 = "0.9.1" -siphasher = "0.3.2" -slice-group-by = "0.2.6" -structopt = "0.3.20" -tar = "0.4.29" -tempfile = "3.1.0" -tokio = { version = "1", features = ["full"] } -dashmap = "4.0.2" -uuid = "0.8.2" -itertools = "0.10.0" -either = "1.6.1" -async-trait = "0.1.42" -thiserror = "1.0.24" -async-stream = "0.3.0" - -[dependencies.sentry] -default-features = false -features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] -optional = true -version = "0.18.1" - - -[dev-dependencies] -serde_url_params = "0.2.0" -tempdir = "0.3.7" -assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } -actix-rt = "2.1.0" - -[features] -default = ["sentry"] - -[target.'cfg(unix)'.dependencies] -jemallocator = "0.3.2" +[profile.release] +debug = true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..9898d02db --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Compile +FROM alpine:3.10 AS compiler + +RUN apk update --quiet +RUN apk add curl +RUN apk add build-base + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +WORKDIR /meilisearch + +COPY . . + +ENV RUSTFLAGS="-C target-feature=-crt-static" + +RUN $HOME/.cargo/bin/cargo build --release + +# Run +FROM alpine:3.10 + +RUN apk add -q --no-cache libgcc tini + +COPY --from=compiler /meilisearch/target/release/meilisearch . + +ENV MEILI_HTTP_ADDR 0.0.0.0:7700 +EXPOSE 7700/tcp + +ENTRYPOINT ["tini", "--"] +CMD ./meilisearch diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml new file mode 100644 index 000000000..d5a474ea5 --- /dev/null +++ b/meilisearch-error/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "meilisearch-error" +version = "0.19.0" +authors = ["marin "] +edition = "2018" + +[dependencies] +actix-http = "2.2.0" diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs new file mode 100644 index 000000000..d0e00e9be --- /dev/null +++ b/meilisearch-error/src/lib.rs @@ -0,0 +1,185 @@ +use std::fmt; + +use actix_http::http::StatusCode; + +pub trait ErrorCode: std::error::Error { + fn error_code(&self) -> Code; + + /// returns the HTTP status code ascociated with the error + fn http_status(&self) -> StatusCode { + self.error_code().http() + } + + /// returns the doc url ascociated with the error + fn error_url(&self) -> String { + self.error_code().url() + } + + /// returns error name, used as error code + fn error_name(&self) -> String { + self.error_code().name() + } + + /// return the error type + fn error_type(&self) -> String { + self.error_code().type_() + } +} + +#[allow(clippy::enum_variant_names)] +enum ErrorType { + InternalError, + InvalidRequestError, + AuthenticationError, +} + +impl fmt::Display for ErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use ErrorType::*; + + match self { + InternalError => write!(f, "internal_error"), + InvalidRequestError => write!(f, "invalid_request_error"), + AuthenticationError => write!(f, "authentication_error"), + } + } +} + +pub enum Code { + // index related error + CreateIndex, + IndexAlreadyExists, + IndexNotFound, + InvalidIndexUid, + OpenIndex, + + // invalid state error + InvalidState, + MissingPrimaryKey, + PrimaryKeyAlreadyPresent, + + MaxFieldsLimitExceeded, + MissingDocumentId, + + Facet, + Filter, + + BadParameter, + BadRequest, + DocumentNotFound, + Internal, + InvalidToken, + MissingAuthorizationHeader, + NotFound, + PayloadTooLarge, + RetrieveDocument, + SearchDocuments, + UnsupportedMediaType, + + DumpAlreadyInProgress, + DumpProcessFailed, +} + +impl Code { + + /// ascociate a `Code` variant to the actual ErrCode + fn err_code(&self) -> ErrCode { + use Code::*; + + match self { + // index related errors + // create index is thrown on internal error while creating an index. + CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST), + IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST), + // thrown when requesting an unexisting index + IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), + InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), + OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + + // invalid state error + InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), + // thrown when no primary key has been set + MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), + // error thrown when trying to set an already existing primary key + PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + + // invalid document + MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), + + // error related to facets + Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST), + // error related to filters + Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST), + + BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST), + BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST), + DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), + Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), + InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), + MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), + PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), + RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), + UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + + // error related to dump + DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), + DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + } + } + + /// return the HTTP status code ascociated with the `Code` + fn http(&self) -> StatusCode { + self.err_code().status_code + } + + /// return error name, used as error code + fn name(&self) -> String { + self.err_code().error_name.to_string() + } + + /// return the error type + fn type_(&self) -> String { + self.err_code().error_type.to_string() + } + + /// return the doc url ascociated with the error + fn url(&self) -> String { + format!("https://docs.meilisearch.com/errors#{}", self.name()) + } +} + +/// Internal structure providing a convenient way to create error codes +struct ErrCode { + status_code: StatusCode, + error_type: ErrorType, + error_name: &'static str, +} + +impl ErrCode { + fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::AuthenticationError, + } + } + + fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InternalError, + } + } + + fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode { + ErrCode { + status_code, + error_name, + error_type: ErrorType::InvalidRequestError, + } + } +} diff --git a/meilisearch-http/Cargo.lock b/meilisearch-http/Cargo.lock new file mode 100644 index 000000000..0bdc739d5 --- /dev/null +++ b/meilisearch-http/Cargo.lock @@ -0,0 +1,3804 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project 0.4.27", + "tokio 0.2.24", + "tokio-util 0.3.1", +] + +[[package]] +name = "actix-codec" +version = "0.4.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e" +dependencies = [ + "bitflags", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", + "tokio-util 0.6.3", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-utils 2.0.0", + "derive_more", + "either", + "futures-util", + "http", + "log", + "trust-dns-proto", + "trust-dns-resolver", +] + +[[package]] +name = "actix-cors" +version = "0.5.4" +dependencies = [ + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "tinyvec", +] + +[[package]] +name = "actix-http" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +dependencies = [ + "actix-codec 0.3.0", + "actix-connect", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "actix-threadpool", + "actix-utils 2.0.0", + "base64 0.13.0", + "bitflags", + "bytes 0.5.6", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2 0.2.7", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "slab", + "time 0.2.23", +] + +[[package]] +name = "actix-http" +version = "3.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a01f9e0681608afa887d4269a0857ac4226f09ba5ceda25939e8391c9da610a" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "ahash 0.7.0", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 1.0.1", + "bytestring", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "futures-util", + "h2 0.3.1", + "http", + "httparse", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project 1.0.2", + "rand 0.8.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1 0.9.2", + "smallvec", + "time 0.2.23", + "tokio 1.2.0", +] + +[[package]] +name = "actix-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros 0.1.3", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio 0.2.24", +] + +[[package]] +name = "actix-rt" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4e57bc1a3915e71526d128baf4323700bd1580bc676239e2298a4c5b001f18" +dependencies = [ + "actix-macros 0.2.0", + "futures-core", + "tokio 1.2.0", +] + +[[package]] +name = "actix-server" +version = "2.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99198727204a48f82559c18e4b0ba3197b97d5f4576a32bdbef371f3b4599c1" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "futures-core", + "log", + "mio 0.7.9", + "num_cpus", + "slab", + "tokio 1.2.0", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project 0.4.27", +] + +[[package]] +name = "actix-service" +version = "2.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca9756f4d32984ac454ae3155a276f6be69b424197bd3f0ca3c87cde72f41d63" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "3.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-utils 3.0.0-beta.2", + "derive_more", + "futures-core", + "http", + "log", + "tokio-rustls 0.22.0", + "tokio-util 0.6.3", + "webpki-roots 0.21.0", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec 0.3.0", + "actix-rt 1.1.1", + "actix-service 1.0.6", + "bitflags", + "bytes 0.5.6", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project 0.4.27", + "slab", +] + +[[package]] +name = "actix-utils" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458795e09a29bc5557604f9ff6f32236fd0ee457d631672e4ec8f6a0103bb292" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", +] + +[[package]] +name = "actix-web" +version = "4.0.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d95e50c9e32e8456220b5804867de76e97a86ab8c38b51c9edcccc0f0fddca7" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.4", + "actix-macros 0.2.0", + "actix-router", + "actix-rt 2.1.0", + "actix-server", + "actix-service 2.0.0-beta.4", + "actix-tls", + "actix-utils 3.0.0-beta.2", + "actix-web-codegen", + "ahash 0.7.0", + "awc", + "bytes 1.0.1", + "derive_more", + "either", + "encoding_rs", + "futures-core", + "futures-util", + "log", + "mime", + "pin-project 1.0.2", + "regex", + "rustls 0.19.0", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.2.23", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.5.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + +[[package]] +name = "ahash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "assert-json-diff" +version = "1.0.1" +source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite 0.1.11", + "tokio 0.2.24", +] + +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "3.0.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09aecd8728f6491a62b27454ea4b36fb7e50faf32928b0369b644e402c651f4e" +dependencies = [ + "actix-codec 0.4.0-beta.1", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "base64 0.13.0", + "bytes 1.0.1", + "cfg-if 1.0.0", + "derive_more", + "futures-core", + "itoa", + "log", + "mime", + "percent-encoding", + "pin-project-lite 0.2.0", + "rand 0.8.3", + "rustls 0.19.0", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "backtrace" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +dependencies = [ + "byteorder", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "bstr" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byte-unit" +version = "4.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "bytemuck" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +dependencies = [ + "bytes 1.0.1", +] + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cedarwood" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "character_converter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" +dependencies = [ + "bincode", +] + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "cookie" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +dependencies = [ + "percent-encoding", + "time 0.2.23", + "version_check", +] + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "cow-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.1", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils 0.8.1", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + +[[package]] +name = "debugid" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deunicode" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +dependencies = [ + "atty", + "humantime 2.0.1", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.1.57", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "fst" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" + +[[package]] +name = "futures-executor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" + +[[package]] +name = "futures-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" + +[[package]] +name = "futures-task" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.2", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "grenad" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" +dependencies = [ + "byteorder", + "flate2", + "log", + "nix", + "snap", + "tempfile", + "zstd", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 0.2.24", + "tokio-util 0.3.1", + "tracing", + "tracing-futures", +] + +[[package]] +name = "h2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.2.0", + "tokio-util 0.6.3", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +dependencies = [ + "ahash 0.3.8", + "autocfg", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heed" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +dependencies = [ + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", + "zerocopy", +] + +[[package]] +name = "heed-traits" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" + +[[package]] +name = "heed-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +dependencies = [ + "bincode", + "heed-traits", + "serde", + "serde_json", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +dependencies = [ + "bytes 1.0.1", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "human_format" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "humantime" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.2", + "socket2", + "tokio 0.2.24", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +dependencies = [ + "bytes 0.5.6", + "futures-util", + "hyper", + "log", + "rustls 0.18.1", + "tokio 0.2.24", + "tokio-rustls 0.14.1", + "webpki", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "im" +version = "14.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" +dependencies = [ + "bitmaps", + "rand_core 0.5.1", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown 0.9.1", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "jieba-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +dependencies = [ + "cedarwood", + "fxhash", + "hashbrown 0.9.1", + "lazy_static", + "phf", + "phf_codegen", + "regex", +] + +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "levenshtein_automata" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +dependencies = [ + "fst", +] + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "main_error" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb63bb1e282e0b6aba0addb1f0e87cb5181ea68142b2dfd21ba108f8e8088a64" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "meilisearch-error" +version = "0.19.0" +dependencies = [ + "actix-http 2.2.0", +] + +[[package]] +name = "meilisearch-http" +version = "0.17.0" +dependencies = [ + "actix-cors", + "actix-http 3.0.0-beta.4", + "actix-rt 2.1.0", + "actix-service 2.0.0-beta.4", + "actix-web", + "anyhow", + "assert-json-diff", + "async-compression", + "async-stream", + "async-trait", + "byte-unit", + "bytes 0.6.0", + "chrono", + "crossbeam-channel", + "dashmap", + "either", + "env_logger 0.8.2", + "flate2", + "fst", + "futures", + "futures-util", + "grenad", + "heed", + "http", + "indexmap", + "itertools 0.10.0", + "jemallocator", + "log", + "main_error", + "meilisearch-error", + "meilisearch-tokenizer", + "memmap", + "milli", + "mime", + "once_cell", + "rand 0.7.3", + "rayon", + "regex", + "rustls 0.19.0", + "sentry", + "serde", + "serde_json", + "serde_url_params", + "sha2", + "siphasher", + "slice-group-by", + "structopt", + "tar", + "tempdir", + "tempfile", + "thiserror", + "tokio 1.2.0", + "uuid", + "vergen", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.1" +source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#147b6154b1b34cb8f5da2df6a416b7da191bc850" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "milli" +version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" +dependencies = [ + "anyhow", + "bstr", + "byteorder", + "crossbeam-channel", + "csv", + "either", + "flate2", + "fst", + "fxhash", + "grenad", + "heed", + "human_format", + "itertools 0.10.0", + "levenshtein_automata", + "linked-hash-map", + "log", + "meilisearch-tokenizer", + "memmap", + "num-traits", + "obkv", + "once_cell", + "ordered-float", + "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", + "pest_derive", + "rayon", + "regex", + "roaring", + "serde", + "serde_json", + "smallstr", + "smallvec", + "tempfile", + "uuid", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +dependencies = [ + "libc", + "log", + "miow 0.3.6", + "ntapi", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" + +[[package]] +name = "obkv" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ordered-float" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest" +version = "2.1.3" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha-1 0.8.2", +] + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +dependencies = [ + "pin-project-internal 1.0.2", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.15", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.1", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite 0.2.0", + "rustls 0.18.1", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 0.2.24", + "tokio-rustls 0.14.1", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.20.0", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "retain_mut" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" + +[[package]] +name = "ring" +version = "0.16.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "roaring" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6744a4a918e91359ad1d356a91e2e943a86d9fb9ae77f715d617032ea2af88f" +dependencies = [ + "bytemuck", + "byteorder", + "retain_mut", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "sentry" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +dependencies = [ + "backtrace", + "env_logger 0.7.1", + "failure", + "hostname", + "httpdate", + "im", + "lazy_static", + "libc", + "log", + "rand 0.7.3", + "regex", + "reqwest", + "rustc_version", + "sentry-types", + "uname", + "url", +] + +[[package]] +name = "sentry-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +dependencies = [ + "chrono", + "debugid", + "failure", + "serde", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_url_params" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +dependencies = [ + "serde", + "url", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" + +[[package]] +name = "sized-chunks" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "slice-group-by" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" + +[[package]] +name = "smallstr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" +dependencies = [ + "serde", + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "snap" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d3306e84bf86710d6cd8b4c9c3b721d5454cc91a603180f8f8cd06cfd317b4" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +dependencies = [ + "filetime", + "libc", + "redox_syscall 0.1.57", + "xattr", +] + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall 0.2.5", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio 0.6.23", + "mio-uds", + "num_cpus", + "pin-project-lite 0.1.11", + "signal-hook-registry", + "slab", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes 1.0.1", + "libc", + "memchr", + "mio 0.7.9", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.0", + "signal-hook-registry", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls 0.18.1", + "tokio 0.2.24", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.0", + "tokio 1.2.0", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.11", + "tokio 0.2.24", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.0", + "tokio 1.2.0", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.0", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand 0.7.3", + "smallvec", + "thiserror", + "tokio 0.2.24", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" +dependencies = [ + "backtrace", + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio 0.2.24", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "vergen" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +dependencies = [ + "bitflags", + "chrono", +] + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki", +] + +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + +[[package]] +name = "whatlang" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" +dependencies = [ + "hashbrown 0.7.2", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "zstd" +version = "0.5.4+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "2.0.6+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.18+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +dependencies = [ + "cc", + "glob", + "itertools 0.9.0", + "libc", +] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml new file mode 100644 index 000000000..0f3d3cb38 --- /dev/null +++ b/meilisearch-http/Cargo.toml @@ -0,0 +1,82 @@ +[package] +authors = ["Quentin de Quelen ", "Clément Renault "] +description = "MeiliSearch HTTP server" +edition = "2018" +license = "MIT" +name = "meilisearch-http" +version = "0.17.0" +[[bin]] +name = "meilisearch" +path = "src/main.rs" + +[build-dependencies] +vergen = "3.1.0" + +[dependencies] +actix-cors = { path = "../../actix-extras/actix-cors" } +actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } +actix-service = "2.0.0-beta.4" +actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } +#actix-web = { version = "3", features = ["rustls"] } +anyhow = "1.0.36" +async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } +byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +bytes = "0.6.0" +chrono = { version = "0.4.19", features = ["serde"] } +crossbeam-channel = "0.5.0" +env_logger = "0.8.2" +flate2 = "1.0.19" +fst = "0.4.5" +futures = "0.3.7" +futures-util = "0.3.8" +grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } +heed = "0.10.6" +http = "0.2.1" +indexmap = { version = "1.3.2", features = ["serde-1"] } +log = "0.4.8" +main_error = "0.1.0" +meilisearch-error = { path = "../meilisearch-error" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } +memmap = "0.7.0" +milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +mime = "0.3.16" +once_cell = "1.5.2" +rand = "0.7.3" +rayon = "1.5.0" +regex = "1.4.2" +rustls = "0.19" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.59", features = ["preserve_order"] } +sha2 = "0.9.1" +siphasher = "0.3.2" +slice-group-by = "0.2.6" +structopt = "0.3.20" +tar = "0.4.29" +tempfile = "3.1.0" +tokio = { version = "1", features = ["full"] } +dashmap = "4.0.2" +uuid = "0.8.2" +itertools = "0.10.0" +either = "1.6.1" +async-trait = "0.1.42" +thiserror = "1.0.24" +async-stream = "0.3.0" + +[dependencies.sentry] +default-features = false +features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] +optional = true +version = "0.18.1" + + +[dev-dependencies] +serde_url_params = "0.2.0" +tempdir = "0.3.7" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } +actix-rt = "2.1.0" + +[features] +default = ["sentry"] + +[target.'cfg(unix)'.dependencies] +jemallocator = "0.3.2" diff --git a/build.rs b/meilisearch-http/build.rs similarity index 100% rename from build.rs rename to meilisearch-http/build.rs diff --git a/public/bulma.min.css b/meilisearch-http/public/bulma.min.css similarity index 100% rename from public/bulma.min.css rename to meilisearch-http/public/bulma.min.css diff --git a/public/interface.html b/meilisearch-http/public/interface.html similarity index 100% rename from public/interface.html rename to meilisearch-http/public/interface.html diff --git a/src/analytics.rs b/meilisearch-http/src/analytics.rs similarity index 100% rename from src/analytics.rs rename to meilisearch-http/src/analytics.rs diff --git a/src/data/mod.rs b/meilisearch-http/src/data/mod.rs similarity index 100% rename from src/data/mod.rs rename to meilisearch-http/src/data/mod.rs diff --git a/src/data/search.rs b/meilisearch-http/src/data/search.rs similarity index 100% rename from src/data/search.rs rename to meilisearch-http/src/data/search.rs diff --git a/src/data/updates.rs b/meilisearch-http/src/data/updates.rs similarity index 100% rename from src/data/updates.rs rename to meilisearch-http/src/data/updates.rs diff --git a/src/dump.rs b/meilisearch-http/src/dump.rs similarity index 100% rename from src/dump.rs rename to meilisearch-http/src/dump.rs diff --git a/src/error.rs b/meilisearch-http/src/error.rs similarity index 100% rename from src/error.rs rename to meilisearch-http/src/error.rs diff --git a/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs similarity index 100% rename from src/helpers/authentication.rs rename to meilisearch-http/src/helpers/authentication.rs diff --git a/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs similarity index 100% rename from src/helpers/compression.rs rename to meilisearch-http/src/helpers/compression.rs diff --git a/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs similarity index 100% rename from src/helpers/mod.rs rename to meilisearch-http/src/helpers/mod.rs diff --git a/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs similarity index 100% rename from src/helpers/normalize_path.rs rename to meilisearch-http/src/helpers/normalize_path.rs diff --git a/src/index/mod.rs b/meilisearch-http/src/index/mod.rs similarity index 100% rename from src/index/mod.rs rename to meilisearch-http/src/index/mod.rs diff --git a/src/index/search.rs b/meilisearch-http/src/index/search.rs similarity index 100% rename from src/index/search.rs rename to meilisearch-http/src/index/search.rs diff --git a/src/index/updates.rs b/meilisearch-http/src/index/updates.rs similarity index 100% rename from src/index/updates.rs rename to meilisearch-http/src/index/updates.rs diff --git a/src/index_controller/actor_index_controller/mod.rs b/meilisearch-http/src/index_controller/actor_index_controller/mod.rs similarity index 100% rename from src/index_controller/actor_index_controller/mod.rs rename to meilisearch-http/src/index_controller/actor_index_controller/mod.rs diff --git a/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs similarity index 100% rename from src/index_controller/index_actor.rs rename to meilisearch-http/src/index_controller/index_actor.rs diff --git a/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs similarity index 100% rename from src/index_controller/local_index_controller/index_store.rs rename to meilisearch-http/src/index_controller/local_index_controller/index_store.rs diff --git a/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs similarity index 100% rename from src/index_controller/local_index_controller/mod.rs rename to meilisearch-http/src/index_controller/local_index_controller/mod.rs diff --git a/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs similarity index 100% rename from src/index_controller/mod.rs rename to meilisearch-http/src/index_controller/mod.rs diff --git a/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs similarity index 100% rename from src/index_controller/update_actor.rs rename to meilisearch-http/src/index_controller/update_actor.rs diff --git a/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs similarity index 100% rename from src/index_controller/update_handler.rs rename to meilisearch-http/src/index_controller/update_handler.rs diff --git a/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs similarity index 100% rename from src/index_controller/update_store.rs rename to meilisearch-http/src/index_controller/update_store.rs diff --git a/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs similarity index 100% rename from src/index_controller/updates.rs rename to meilisearch-http/src/index_controller/updates.rs diff --git a/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs similarity index 100% rename from src/index_controller/uuid_resolver.rs rename to meilisearch-http/src/index_controller/uuid_resolver.rs diff --git a/src/lib.rs b/meilisearch-http/src/lib.rs similarity index 100% rename from src/lib.rs rename to meilisearch-http/src/lib.rs diff --git a/src/main.rs b/meilisearch-http/src/main.rs similarity index 100% rename from src/main.rs rename to meilisearch-http/src/main.rs diff --git a/src/option.rs b/meilisearch-http/src/option.rs similarity index 100% rename from src/option.rs rename to meilisearch-http/src/option.rs diff --git a/src/routes/document.rs b/meilisearch-http/src/routes/document.rs similarity index 100% rename from src/routes/document.rs rename to meilisearch-http/src/routes/document.rs diff --git a/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs similarity index 100% rename from src/routes/dump.rs rename to meilisearch-http/src/routes/dump.rs diff --git a/src/routes/health.rs b/meilisearch-http/src/routes/health.rs similarity index 100% rename from src/routes/health.rs rename to meilisearch-http/src/routes/health.rs diff --git a/src/routes/index.rs b/meilisearch-http/src/routes/index.rs similarity index 100% rename from src/routes/index.rs rename to meilisearch-http/src/routes/index.rs diff --git a/src/routes/key.rs b/meilisearch-http/src/routes/key.rs similarity index 100% rename from src/routes/key.rs rename to meilisearch-http/src/routes/key.rs diff --git a/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs similarity index 100% rename from src/routes/mod.rs rename to meilisearch-http/src/routes/mod.rs diff --git a/src/routes/search.rs b/meilisearch-http/src/routes/search.rs similarity index 100% rename from src/routes/search.rs rename to meilisearch-http/src/routes/search.rs diff --git a/src/routes/settings/attributes_for_faceting.rs b/meilisearch-http/src/routes/settings/attributes_for_faceting.rs similarity index 100% rename from src/routes/settings/attributes_for_faceting.rs rename to meilisearch-http/src/routes/settings/attributes_for_faceting.rs diff --git a/src/routes/settings/displayed_attributes.rs b/meilisearch-http/src/routes/settings/displayed_attributes.rs similarity index 100% rename from src/routes/settings/displayed_attributes.rs rename to meilisearch-http/src/routes/settings/displayed_attributes.rs diff --git a/src/routes/settings/distinct_attributes.rs b/meilisearch-http/src/routes/settings/distinct_attributes.rs similarity index 100% rename from src/routes/settings/distinct_attributes.rs rename to meilisearch-http/src/routes/settings/distinct_attributes.rs diff --git a/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs similarity index 100% rename from src/routes/settings/mod.rs rename to meilisearch-http/src/routes/settings/mod.rs diff --git a/src/routes/settings/ranking_rules.rs b/meilisearch-http/src/routes/settings/ranking_rules.rs similarity index 100% rename from src/routes/settings/ranking_rules.rs rename to meilisearch-http/src/routes/settings/ranking_rules.rs diff --git a/src/routes/settings/searchable_attributes.rs b/meilisearch-http/src/routes/settings/searchable_attributes.rs similarity index 100% rename from src/routes/settings/searchable_attributes.rs rename to meilisearch-http/src/routes/settings/searchable_attributes.rs diff --git a/src/routes/settings/stop_words.rs b/meilisearch-http/src/routes/settings/stop_words.rs similarity index 100% rename from src/routes/settings/stop_words.rs rename to meilisearch-http/src/routes/settings/stop_words.rs diff --git a/src/routes/settings/synonyms.rs b/meilisearch-http/src/routes/settings/synonyms.rs similarity index 100% rename from src/routes/settings/synonyms.rs rename to meilisearch-http/src/routes/settings/synonyms.rs diff --git a/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs similarity index 100% rename from src/routes/stats.rs rename to meilisearch-http/src/routes/stats.rs diff --git a/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs similarity index 100% rename from src/routes/stop_words.rs rename to meilisearch-http/src/routes/stop_words.rs diff --git a/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs similarity index 100% rename from src/routes/synonym.rs rename to meilisearch-http/src/routes/synonym.rs diff --git a/src/snapshot.rs b/meilisearch-http/src/snapshot.rs similarity index 100% rename from src/snapshot.rs rename to meilisearch-http/src/snapshot.rs diff --git a/tests/assets/test_set.json b/meilisearch-http/tests/assets/test_set.json similarity index 100% rename from tests/assets/test_set.json rename to meilisearch-http/tests/assets/test_set.json diff --git a/tests/common/index.rs b/meilisearch-http/tests/common/index.rs similarity index 100% rename from tests/common/index.rs rename to meilisearch-http/tests/common/index.rs diff --git a/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs similarity index 100% rename from tests/common/mod.rs rename to meilisearch-http/tests/common/mod.rs diff --git a/tests/common/server.rs b/meilisearch-http/tests/common/server.rs similarity index 100% rename from tests/common/server.rs rename to meilisearch-http/tests/common/server.rs diff --git a/tests/common/service.rs b/meilisearch-http/tests/common/service.rs similarity index 100% rename from tests/common/service.rs rename to meilisearch-http/tests/common/service.rs diff --git a/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs similarity index 100% rename from tests/documents/add_documents.rs rename to meilisearch-http/tests/documents/add_documents.rs diff --git a/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs similarity index 100% rename from tests/documents/delete_documents.rs rename to meilisearch-http/tests/documents/delete_documents.rs diff --git a/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs similarity index 100% rename from tests/documents/get_documents.rs rename to meilisearch-http/tests/documents/get_documents.rs diff --git a/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs similarity index 100% rename from tests/documents/mod.rs rename to meilisearch-http/tests/documents/mod.rs diff --git a/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs similarity index 100% rename from tests/index/create_index.rs rename to meilisearch-http/tests/index/create_index.rs diff --git a/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs similarity index 100% rename from tests/index/delete_index.rs rename to meilisearch-http/tests/index/delete_index.rs diff --git a/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs similarity index 100% rename from tests/index/get_index.rs rename to meilisearch-http/tests/index/get_index.rs diff --git a/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs similarity index 100% rename from tests/index/mod.rs rename to meilisearch-http/tests/index/mod.rs diff --git a/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs similarity index 100% rename from tests/index/update_index.rs rename to meilisearch-http/tests/index/update_index.rs diff --git a/tests/integration.rs b/meilisearch-http/tests/integration.rs similarity index 100% rename from tests/integration.rs rename to meilisearch-http/tests/integration.rs diff --git a/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs similarity index 100% rename from tests/search/mod.rs rename to meilisearch-http/tests/search/mod.rs diff --git a/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs similarity index 100% rename from tests/settings/get_settings.rs rename to meilisearch-http/tests/settings/get_settings.rs diff --git a/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs similarity index 100% rename from tests/settings/mod.rs rename to meilisearch-http/tests/settings/mod.rs diff --git a/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs similarity index 100% rename from tests/updates/mod.rs rename to meilisearch-http/tests/updates/mod.rs From 0cd8869349b5cd96242b112b887e3d7a863af8d9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 14:43:10 +0100 Subject: [PATCH 40/69] update relevant changes from master --- .dockerignore | 5 ++ .github/workflows/create_artifacts.yml | 38 +++++++++ .github/workflows/publish_to_docker.yml | 20 +++++ .github/workflows/rust.yml | 6 +- LICENSE | 21 +++++ .../tests/assets/dumps/v1/metadata.json | 12 +++ .../assets/dumps/v1/test/documents.jsonl | 77 +++++++++++++++++++ .../tests/assets/dumps/v1/test/settings.json | 59 ++++++++++++++ .../tests/assets/dumps/v1/test/updates.jsonl | 2 + meilisearch-http/tests/common/index.rs | 4 +- meilisearch-http/tests/common/server.rs | 7 +- meilisearch-http/tests/common/service.rs | 14 ++-- .../tests/documents/add_documents.rs | 16 ++++ meilisearch-http/tests/index/create_index.rs | 4 +- .../tests/settings/get_settings.rs | 15 +++- meilisearch-http/tests/updates/mod.rs | 2 - 16 files changed, 276 insertions(+), 26 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/create_artifacts.yml create mode 100644 .github/workflows/publish_to_docker.yml create mode 100644 LICENSE create mode 100644 meilisearch-http/tests/assets/dumps/v1/metadata.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/settings.json create mode 100644 meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..364510117 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +Dockerfile +.dockerignore +.git +.gitignore diff --git a/.github/workflows/create_artifacts.yml b/.github/workflows/create_artifacts.yml new file mode 100644 index 000000000..94378ba83 --- /dev/null +++ b/.github/workflows/create_artifacts.yml @@ -0,0 +1,38 @@ +name: Create artifacts + +on: + push: + tags: + - v*-alpha.* + +jobs: + nightly: + name: Build Nighlty ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + artifact_name: meilisearch + asset_name: meilisearch-alpha-linux-amd64 + - os: macos-latest + artifact_name: meilisearch + asset_name: meilisearch-alpha-macos-amd64 + - os: windows-latest + artifact_name: meilisearch.exe + asset_name: meilisearch-alpha-windows-amd64.exe + steps: + - uses: hecrj/setup-rust-action@master + with: + rust-version: stable + - uses: actions/checkout@v1 + - name: Build + run: cargo build --release --locked + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v1-release + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/${{ matrix.artifact_name }} + asset_name: ${{ matrix.asset_name }} + tag: ${{ github.ref }} diff --git a/.github/workflows/publish_to_docker.yml b/.github/workflows/publish_to_docker.yml new file mode 100644 index 000000000..d724f7253 --- /dev/null +++ b/.github/workflows/publish_to_docker.yml @@ -0,0 +1,20 @@ +name: Publish to dockerhub + +on: + push: + tags: + - v*-alpha.* + +jobs: + publish: + name: Publishing to dockerhub + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Publish to Registry + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: getmeili/meilisearch + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + tag_names: true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3c13d1be2..ea4b41a6a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,10 +1,6 @@ name: Rust -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: push env: CARGO_TERM_COLOR: always diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..87471adb1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Meili + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/meilisearch-http/tests/assets/dumps/v1/metadata.json b/meilisearch-http/tests/assets/dumps/v1/metadata.json new file mode 100644 index 000000000..6fe302324 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/metadata.json @@ -0,0 +1,12 @@ +{ + "indices": [{ + "uid": "test", + "primaryKey": "id" + }, { + "uid": "test2", + "primaryKey": "test2_id" + } + ], + "dbVersion": "0.13.0", + "dumpVersion": "1" +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl new file mode 100644 index 000000000..7af80f342 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl @@ -0,0 +1,77 @@ +{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} +{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} +{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} +{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} +{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} +{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} +{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} +{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} +{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} +{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} +{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} +{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} +{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} +{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} +{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} +{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} +{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} +{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} +{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} +{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} +{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} +{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} +{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} +{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} +{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} +{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} +{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} +{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} +{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} +{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} +{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} +{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} +{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} +{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} +{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} +{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} +{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} +{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} +{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} +{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} +{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} +{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} +{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} +{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} +{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} +{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} +{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} +{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} +{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} +{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} +{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} +{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} +{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} +{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} +{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} +{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} +{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} +{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} +{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} +{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} +{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} +{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} +{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} +{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} +{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} +{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} +{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} +{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} +{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} +{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} +{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} +{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} +{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} +{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} +{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} +{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} +{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/meilisearch-http/tests/assets/dumps/v1/test/settings.json b/meilisearch-http/tests/assets/dumps/v1/test/settings.json new file mode 100644 index 000000000..918cfab53 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/settings.json @@ -0,0 +1,59 @@ +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": "email", + "searchableAttributes": [ + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "displayedAttributes": [ + "id", + "isActive", + "balance", + "picture", + "age", + "color", + "name", + "gender", + "email", + "phone", + "address", + "about", + "registered", + "latitude", + "longitude", + "tags" + ], + "stopWords": [ + "in", + "ad" + ], + "synonyms": { + "wolverine": ["xmen", "logan"], + "logan": ["wolverine", "xmen"] + }, + "attributesForFaceting": [ + "gender", + "color", + "tags", + "isActive" + ] +} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl new file mode 100644 index 000000000..0dcffdce0 --- /dev/null +++ b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl @@ -0,0 +1,2 @@ +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 58a8de200..710b96a9f 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::sleep; +use tokio::time::delay_for; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - sleep(Duration::from_secs(1)).await; + delay_for(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index d7d76445c..943284736 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -1,7 +1,8 @@ -use tempdir::TempDir; +use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use serde_json::Value; -use actix_web::http::StatusCode; +use tempdir::TempDir; +use urlencoding::encode; use meilisearch_http::data::Data; use meilisearch_http::option::{Opt, IndexerOpts}; @@ -60,7 +61,7 @@ impl Server { /// Returns a view to an index. There is no guarantee that the index exists. pub fn index<'a>(&'a self, uid: impl AsRef) -> Index<'a> { Index { - uid: uid.as_ref().to_string(), + uid: encode(uid.as_ref()), service: &self.service, } } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index feff49cad..8e797c00d 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::create_app; +use meilisearch_http::helpers::NormalizePath; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .insert_header(("content-type", "application/json")) + .header("content-type", "application/json") .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(create_app!(&self.0, true)).await; + test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 70d6aab68..63724af18 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -45,6 +45,22 @@ async fn add_documents_no_index_creation() { assert_eq!(response["primaryKey"], "id"); } +#[actix_rt::test] +async fn document_add_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn document_update_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.update_documents(json!([]), None).await; + assert_eq!(code, 400); +} + #[actix_rt::test] async fn document_addition_with_primary_key() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 3ff452c33..c26941b91 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -47,12 +47,10 @@ async fn create_existing_index() { assert_eq!(code, 400); } -// test fails (issue #46) #[actix_rt::test] -#[ignore] async fn create_with_invalid_index_uid() { let server = Server::new().await; - let index = server.index("test test"); + let index = server.index("test test#!"); let (_, code) = index.create(None).await; assert_eq!(code, 400); } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index bae044acb..e5fbbde35 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -16,10 +16,11 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 3); + assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["facetedAttributes"], json!({})); + assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); } #[actix_rt::test] @@ -51,8 +52,6 @@ async fn test_partial_update() { } #[actix_rt::test] -#[ignore] -// need fix #54 async fn delete_settings_unexisting_index() { let server = Server::new().await; let index = server.index("test"); @@ -90,6 +89,15 @@ async fn update_setting_unexisting_index() { assert_eq!(code, 200); } +#[actix_rt::test] +async fn update_setting_unexisting_index_invalid_uid() { + let server = Server::new().await; + let index = server.index("test##! "); + let (_response, code) = index.update_settings(json!({})).await; + println!("response: {}", _response); + assert_eq!(code, 400); +} + macro_rules! test_setting_routes { ($($setting:ident), *) => { $( @@ -123,7 +131,6 @@ macro_rules! test_setting_routes { } #[actix_rt::test] - #[ignore] async fn delete_unexisting_index() { let server = Server::new().await; let url = format!("/indexes/test/settings/{}", diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 3fff2d911..03b307daf 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -50,9 +50,7 @@ async fn list_no_updates() { assert!(response.as_array().unwrap().is_empty()); } -// TODO: fix #32 #[actix_rt::test] -#[ignore] async fn list_updates() { let server = Server::new().await; let index = server.index("test"); From a56e8c1a0c3d46b9787235f6f81045f682a9cf60 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 14:47:04 +0100 Subject: [PATCH 41/69] fix tests --- Cargo.lock | 7 +++++++ meilisearch-http/Cargo.toml | 1 + meilisearch-http/tests/common/index.rs | 4 ++-- meilisearch-http/tests/common/service.rs | 14 +++++++------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f8724ca1..84c9b6854 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ dependencies = [ "tempfile", "thiserror", "tokio 1.3.0", + "urlencoding", "uuid", "vergen", ] @@ -3503,6 +3504,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" + [[package]] name = "utf8-width" version = "0.1.4" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0f3d3cb38..b09c52b87 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -74,6 +74,7 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } actix-rt = "2.1.0" +urlencoding = "1.1.1" [features] default = ["sentry"] diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 710b96a9f..58a8de200 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -2,7 +2,7 @@ use std::time::Duration; use actix_web::http::StatusCode; use serde_json::{json, Value}; -use tokio::time::delay_for; +use tokio::time::sleep; use super::service::Service; @@ -79,7 +79,7 @@ impl Index<'_> { return response; } - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } panic!("Timeout waiting for update id"); } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index 8e797c00d..feff49cad 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -2,14 +2,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; +use meilisearch_http::create_app; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -26,12 +26,12 @@ impl Service { /// Send a test post request from a text body, with a `content-type:application/json` header. pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) - .header("content-type", "application/json") + .insert_header(("content-type", "application/json")) .to_request(); let res = test::call_service(&mut app, req).await; let status_code = res.status(); @@ -43,7 +43,7 @@ impl Service { pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -56,7 +56,7 @@ impl Service { pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -72,7 +72,7 @@ impl Service { pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { let mut app = - test::init_service(meilisearch_http::create_app(&self.0, true).wrap(NormalizePath)).await; + test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; From 53cf500e36f0e477a18133f5335d3870ff2b2799 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 10 Mar 2021 18:04:20 +0100 Subject: [PATCH 42/69] uuid resolver hard state --- meilisearch-http/src/data/updates.rs | 3 - .../local_index_controller/mod.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/uuid_resolver.rs | 193 ++++++++++++++---- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/tests/common/index.rs | 6 +- meilisearch-http/tests/updates/mod.rs | 2 +- 7 files changed, 161 insertions(+), 49 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index e9d1b51b7..c6e30ea02 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,7 +1,4 @@ -//use async_compression::tokio_02::write::GzipEncoder; -//use futures_util::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -//use tokio::io::AsyncWriteExt; use actix_web::web::Payload; use crate::index_controller::{UpdateStatus, IndexMetadata}; diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs index 14efe42c7..8ac600f5f 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/mod.rs @@ -148,7 +148,7 @@ impl IndexController for LocalIndexController { fn update_index(&self, uid: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { if index_settings.name.is_some() { - bail!("can't udpate an index name.") + bail!("can't update an index name.") } let (primary_key, meta) = match index_settings.primary_key { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index a824852bf..85469728b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -69,7 +69,7 @@ enum IndexControllerMsg { impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); + let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path)?; let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 8d33edef4..a369790f3 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -1,9 +1,9 @@ +use std::{fs::create_dir_all, path::Path}; + +use heed::{Database, Env, EnvOpenOptions, types::{ByteSlice, Str}}; use log::{info, warn}; -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::sync::Arc; use thiserror::Error; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; pub type Result = std::result::Result; @@ -81,14 +81,14 @@ impl UuidResolverActor { async fn handle_resolve(&self, name: String) -> Result { self.store - .get_uuid(&name) + .get_uuid(name.clone()) .await? .ok_or(UuidError::UnexistingIndex(name)) } async fn handle_delete(&self, name: String) -> Result { self.store - .delete(&name) + .delete(name.clone()) .await? .ok_or(UuidError::UnexistingIndex(name)) } @@ -105,12 +105,12 @@ pub struct UuidResolverHandle { } impl UuidResolverHandle { - pub fn new() -> Self { + pub fn new(path: impl AsRef) -> anyhow::Result { let (sender, reveiver) = mpsc::channel(100); - let store = MapUuidStore(Arc::new(RwLock::new(HashMap::new()))); + let store = HeedUuidStore::new(path)?; let actor = UuidResolverActor::new(reveiver, store); tokio::spawn(actor.run()); - Self { sender } + Ok(Self { sender }) } pub async fn resolve(&self, name: String) -> anyhow::Result { @@ -159,12 +159,18 @@ impl UuidResolverHandle { } } -#[derive(Clone, Debug, Error)] +#[derive(Debug, Error)] pub enum UuidError { #[error("Name already exist.")] NameAlreadyExist, #[error("Index \"{0}\" doesn't exist.")] UnexistingIndex(String), + #[error("Error performing task: {0}")] + TokioTask(#[from] tokio::task::JoinError), + #[error("Database error: {0}")] + Heed(#[from] heed::Error), + #[error("Uuid error: {0}")] + Uuid(#[from] uuid::Error), } #[async_trait::async_trait] @@ -172,48 +178,157 @@ trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: &str) -> Result>; - async fn delete(&self, name: &str) -> Result>; + async fn get_uuid(&self, name: String) -> Result>; + async fn delete(&self, name: String) -> Result>; async fn list(&self) -> Result>; } -struct MapUuidStore(Arc>>); +struct HeedUuidStore { + env: Env, + db: Database, +} + +fn open_or_create_database(env: &Env, name: Option<&str>) -> heed::Result> { + match env.open_database(name)? { + Some(db) => Ok(db), + None => env.create_database(name), + } +} + +impl HeedUuidStore { + fn new(path: impl AsRef) -> anyhow::Result { + let path = path.as_ref().join("index_uuids"); + create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(1_073_741_824); // 1GB + let env = options.open(path)?; + let db = open_or_create_database(&env, None)?; + Ok(Self { env, db }) + } +} #[async_trait::async_trait] -impl UuidStore for MapUuidStore { +impl UuidStore for HeedUuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result { - match self.0.write().await.entry(name) { - Entry::Occupied(entry) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - Ok(entry.get().clone()) + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + let uuid = Uuid::from_slice(uuid)?; + Ok(uuid) + } + } + None => { + let uuid = Uuid::new_v4(); + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(uuid) } } - Entry::Vacant(entry) => { - let uuid = Uuid::new_v4(); - let uuid = entry.insert(uuid); - Ok(uuid.clone()) + }).await? + } + + async fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + Ok(Some(uuid)) + } + None => Ok(None), } - } + }).await? } - async fn get_uuid(&self, name: &str) -> Result> { - Ok(self.0.read().await.get(name).cloned()) - } - - async fn delete(&self, name: &str) -> Result> { - Ok(self.0.write().await.remove(name)) + async fn delete(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let mut txn = env.write_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + db.delete(&mut txn, &name)?; + txn.commit()?; + Ok(None) + } + None => Ok(None) + } + }).await? } async fn list(&self) -> Result> { - let list = self - .0 - .read() - .await - .iter() - .map(|(name, uuid)| (name.to_owned(), uuid.clone())) - .collect(); - Ok(list) + let env = self.env.clone(); + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = env.read_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (name, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push((name.to_owned(), uuid)) + } + Ok(entries) + }).await? + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + use std::collections::hash_map::Entry; + use std::sync::Arc; + + use tokio::sync::RwLock; + + use super::*; + + struct MapUuidStore(Arc>>); + + #[async_trait::async_trait] + impl UuidStore for MapUuidStore { + async fn create_uuid(&self, name: String, err: bool) -> Result { + match self.0.write().await.entry(name) { + Entry::Occupied(entry) => { + if err { + Err(UuidError::NameAlreadyExist) + } else { + Ok(entry.get().clone()) + } + } + Entry::Vacant(entry) => { + let uuid = Uuid::new_v4(); + let uuid = entry.insert(uuid); + Ok(uuid.clone()) + } + } + } + + async fn get_uuid(&self, name: String) -> Result> { + Ok(self.0.read().await.get(&name).cloned()) + } + + async fn delete(&self, name: String) -> Result> { + Ok(self.0.write().await.remove(&name)) + } + + async fn list(&self) -> Result> { + let list = self + .0 + .read() + .await + .iter() + .map(|(name, uuid)| (name.to_owned(), uuid.clone())) + .collect(); + Ok(list) + } } } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 818c87303..5c6e3f5a9 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -140,7 +140,7 @@ async fn get_update_status( Ok(HttpResponse::Ok().body(json)) } Ok(None) => { - let e = format!("udpate {} for index {:?} doesn't exists.", path.update_id, path.index_uid); + let e = format!("update {} for index {:?} doesn't exists.", path.update_id, path.index_uid); Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } Err(e) => { diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 58a8de200..a14769ee1 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -73,7 +73,7 @@ impl Index<'_> { let url = format!("/indexes/{}/updates/{}", self.uid, update_id); for _ in 0..10 { let (response, status_code) = self.service.get(&url).await; - assert_eq!(status_code, 200); + assert_eq!(status_code, 200, "response: {}", response); if response["status"] == "processed" || response["status"] == "failed" { return response; @@ -84,8 +84,8 @@ impl Index<'_> { panic!("Timeout waiting for update id"); } - pub async fn get_update(&self, udpate_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates/{}", self.uid, udpate_id); + pub async fn get_update(&self, update_id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); self.service.get(url).await } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 03b307daf..64b5b560e 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -8,7 +8,7 @@ async fn get_update_unexisting_index() { } #[actix_rt::test] -async fn get_unexisting_udpate_status() { +async fn get_unexisting_update_status() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; From 2ae90f9c5d64cdac3847ef0044e18c1261db8ab8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 14:23:11 +0100 Subject: [PATCH 43/69] lazy load update store --- .../src/index_controller/update_actor.rs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 370912dcf..32eda7bcb 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -301,13 +301,33 @@ impl UpdateStoreStore for MapUpdateStoreStore { } async fn get(&self, uuid: &Uuid) -> Result>> { - Ok(self.db.read().await.get(uuid).cloned()) + // attemps to get pre-loaded ref to the index + match self.db.read().await.get(uuid) { + Some(uuid) => Ok(Some(uuid.clone())), + None => { + // otherwise we try to check if it exists, and load it. + let path = self.path.clone().join(format!("updates-{}", uuid)); + if path.exists() { + let index_handle = self.index_handle.clone(); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .unwrap(); + self.db.write().await.insert(uuid.clone(), store.clone()); + Ok(Some(store)) + } else { + Ok(None) + } + } + } } async fn delete(&self, uuid: &Uuid) -> Result>> { let store = self.db.write().await.remove(&uuid); - if store.is_some() { - let path = self.path.clone().join(format!("updates-{}", uuid)); + let path = self.path.clone().join(format!("updates-{}", uuid)); + if store.is_some() || path.exists() { remove_dir_all(path).unwrap(); } Ok(store) From 1fad72e019ef09279571335edd677cb8453f3e28 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 17:59:47 +0100 Subject: [PATCH 44/69] fix test bug with tempdir --- .../src/index_controller/index_actor.rs | 316 ++++++++++-------- .../local_index_controller/index_store.rs | 3 +- meilisearch-http/src/index_controller/mod.rs | 2 +- .../src/index_controller/update_actor.rs | 13 +- .../src/index_controller/uuid_resolver.rs | 3 +- meilisearch-http/tests/common/server.rs | 9 +- .../tests/settings/get_settings.rs | 3 +- 7 files changed, 188 insertions(+), 161 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 1ddc041a1..d8dc5f014 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -1,4 +1,4 @@ -use std::collections::{hash_map::Entry, HashMap}; +use std::collections::HashMap; use std::fs::{create_dir_all, remove_dir_all, File}; use std::future::Future; use std::path::{Path, PathBuf}; @@ -8,11 +8,15 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::EnvOpenOptions; -use log::info; +use heed::{ + types::{ByteSlice, SerdeBincode}, + Database, Env, EnvOpenOptions, +}; +use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::{sync::{mpsc, oneshot, RwLock}}; +use tokio::task::spawn_blocking; use uuid::Uuid; use super::get_arc_ownership_blocking; @@ -96,6 +100,8 @@ pub enum IndexError { IndexAlreadyExists, #[error("Index doesn't exists")] UnexistingIndex, + #[error("Heed error: {0}")] + HeedError(#[from] heed::Error), } #[async_trait::async_trait] @@ -105,10 +111,9 @@ trait IndexStore { where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static; - async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: &Uuid) -> Result>; - async fn get_meta(&self, uuid: &Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>; + async fn get_meta(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -118,8 +123,8 @@ impl IndexActor { store: S, ) -> Result { let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options) - .map_err(|e| IndexError::Error(e.into()))?; + let update_handler = + UpdateHandler::new(&options).map_err(|e| IndexError::Error(e.into()))?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -227,11 +232,12 @@ impl IndexActor { } async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> anyhow::Result { - let index = self.store + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || index.perform_search(query)).await? + spawn_blocking(move || index.perform_search(query)).await? } async fn handle_create_index( @@ -247,15 +253,14 @@ impl IndexActor { meta: Processing, data: File, ) -> Result { - info!("Processing update {}", meta.id()); + debug!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); let handle = self .store .update_index(uuid, |index| { - let handle = tokio::task::spawn_blocking(move || { - update_handler.handle_update(meta, data, index) - }); + let handle = + spawn_blocking(move || update_handler.handle_update(meta, data, index)); Ok(handle) }) .await?; @@ -264,11 +269,12 @@ impl IndexActor { } async fn handle_settings(&self, uuid: Uuid) -> Result { - let index = self.store + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) .await .map_err(|e| IndexError::Error(e.into()))? } @@ -280,10 +286,12 @@ impl IndexActor { limit: usize, attributes_to_retrieve: Option>, ) -> Result> { - let index = self.store.get(uuid) + let index = self + .store + .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { index .retrieve_documents(offset, limit, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)) @@ -303,7 +311,7 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { index .retrieve_document(doc_id, attributes_to_retrieve) .map_err(|e| IndexError::Error(e)) @@ -313,15 +321,15 @@ impl IndexActor { } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let index = self.store.delete(&uuid).await?; + let index = self.store.delete(uuid).await?; if let Some(index) = index { tokio::task::spawn(async move { let index = index.0; let store = get_arc_ownership_blocking(index).await; - tokio::task::spawn_blocking(move || { + spawn_blocking(move || { store.prepare_for_closing().wait(); - info!("Index {} was closed.", uuid); + debug!("Index closed"); }); }); } @@ -330,7 +338,7 @@ impl IndexActor { } async fn handle_get_meta(&self, uuid: Uuid) -> Result> { - let result = self.store.get_meta(&uuid).await?; + let result = self.store.get_meta(uuid).await?; Ok(result) } } @@ -346,7 +354,7 @@ impl IndexActorHandle { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = MapIndexStore::new(path); + let store = HeedIndexStore::new(path)?; let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -442,149 +450,165 @@ impl IndexActorHandle { } } -struct MapIndexStore { - root: PathBuf, - meta_store: AsyncMap, +struct HeedIndexStore { + env: Env, + db: Database>, index_store: AsyncMap, + path: PathBuf, +} + +impl HeedIndexStore { + fn new(path: impl AsRef) -> anyhow::Result { + let mut options = EnvOpenOptions::new(); + options.map_size(1_073_741_824); //1GB + let path = path.as_ref().join("indexes/"); + create_dir_all(&path)?; + let env = options.open(&path)?; + let db = env.create_database(None)?; + let index_store = Arc::new(RwLock::new(HashMap::new())); + Ok(Self { + env, + db, + index_store, + path, + }) + } } #[async_trait::async_trait] -impl IndexStore for MapIndexStore { +impl IndexStore for HeedIndexStore { async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { - let meta = match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(entry) => { - let now = Utc::now(); - let meta = IndexMeta { - uuid, - created_at: now.clone(), - updated_at: now, - primary_key, - }; - entry.insert(meta).clone() - } - Entry::Occupied(_) => return Err(IndexError::IndexAlreadyExists), - }; + let path = self.path.join(format!("index-{}", uuid)); - let db_path = self.root.join(format!("index-{}", meta.uuid)); + if path.exists() { + return Err(IndexError::IndexAlreadyExists); + } - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path).map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) + let env = self.env.clone(); + let db = self.db.clone(); + let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> { + let now = Utc::now(); + let meta = IndexMeta { + uuid: uuid.clone(), + created_at: now.clone(), + updated_at: now, + primary_key, + }; + let mut txn = env.write_txn()?; + db.put(&mut txn, uuid.as_bytes(), &meta)?; + txn.commit()?; + + let index = open_index(&path, 4096 * 100_000)?; + + Ok((index, meta)) }) .await - .expect("thread died"); + .expect("thread died")?; - self.index_store - .write() - .await - .insert(meta.uuid.clone(), index?); + self.index_store.write().await.insert(uuid.clone(), index); Ok(meta) } - async fn get_or_create(&self, uuid: Uuid, primary_key: Option) -> Result { - match self.index_store.write().await.entry(uuid.clone()) { - Entry::Vacant(index_entry) => match self.meta_store.write().await.entry(uuid.clone()) { - Entry::Vacant(meta_entry) => { - let now = Utc::now(); - let meta = IndexMeta { - uuid, - created_at: now.clone(), - updated_at: now, - primary_key, - }; - let meta = meta_entry.insert(meta); - let db_path = self.root.join(format!("index-{}", meta.uuid)); - - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) - }) - .await - .expect("thread died"); - - Ok(index_entry.insert(index?).clone()) - } - Entry::Occupied(entry) => { - let meta = entry.get(); - let db_path = self.root.join(format!("index-{}", meta.uuid)); - - let index: Result = tokio::task::spawn_blocking(move || { - create_dir_all(&db_path).expect("can't create db"); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let index = milli::Index::new(options, &db_path) - .map_err(|e| IndexError::Error(e))?; - let index = Index(Arc::new(index)); - Ok(index) - }) - .await - .expect("thread died"); - - Ok(index_entry.insert(index?).clone()) - } - }, - Entry::Occupied(entry) => Ok(entry.get().clone()), - } - } - - async fn get(&self, uuid: Uuid) -> Result> { - Ok(self.index_store.read().await.get(&uuid).cloned()) - } - - async fn delete(&self, uuid: &Uuid) -> Result> { - let index = self.index_store.write().await.remove(&uuid); - if index.is_some() { - let db_path = self.root.join(format!("index-{}", uuid)); - remove_dir_all(db_path).unwrap(); - } - Ok(index) - } - - async fn get_meta(&self, uuid: &Uuid) -> Result> { - Ok(self.meta_store.read().await.get(uuid).cloned()) - } - async fn update_index(&self, uuid: Uuid, f: F) -> Result where F: FnOnce(Index) -> Result + Send + Sync + 'static, R: Sync + Send + 'static, { - let index = self.get_or_create(uuid.clone(), None).await?; - let mut meta = self - .get_meta(&uuid) - .await? - .ok_or(IndexError::UnexistingIndex)?; - match f(index) { - Ok(r) => { - meta.updated_at = Utc::now(); - self.meta_store.write().await.insert(uuid, meta); - Ok(r) + let guard = self.index_store.read().await; + let index = match guard.get(&uuid) { + Some(index) => index.clone(), + None => { + drop(guard); + self.create_index(uuid.clone(), None).await?; + self.index_store + .read() + .await + .get(&uuid) + .expect("Index should exist") + .clone() + } + }; + + let env = self.env.clone(); + let db = self.db.clone(); + spawn_blocking(move || { + let mut txn = env.write_txn()?; + let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index"); + match f(index) { + Ok(r) => { + meta.updated_at = Utc::now(); + db.put(&mut txn, uuid.as_bytes(), &meta)?; + txn.commit()?; + Ok(r) + } + Err(e) => Err(e), + } + }) + .await + .expect("thread died") + } + + async fn get(&self, uuid: Uuid) -> Result> { + let guard = self.index_store.read().await; + match guard.get(&uuid) { + Some(index) => Ok(Some(index.clone())), + None => { + // drop the guard here so we can perform the write after without deadlocking; + drop(guard); + let path = self.path.join(format!("index-{}", uuid)); + if !path.exists() { + return Ok(None); + } + + // TODO: set this info from the database + let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) + .await + .expect("thread died")?; + self.index_store + .write() + .await + .insert(uuid.clone(), index.clone()); + println!("here"); + Ok(Some(index)) } - Err(e) => Err(e), } } + + async fn delete(&self, uuid: Uuid) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + let db_path = self.path.join(format!("index-{}", uuid)); + spawn_blocking(move || -> Result<()> { + let mut txn = env.write_txn()?; + db.delete(&mut txn, uuid.as_bytes())?; + txn.commit()?; + remove_dir_all(db_path).unwrap(); + Ok(()) + }) + .await + .expect("thread died")?; + let index = self.index_store.write().await.remove(&uuid); + Ok(index) + } + + async fn get_meta(&self, uuid: Uuid) -> Result> { + let env = self.env.clone(); + let db = self.db.clone(); + spawn_blocking(move || { + let txn = env.read_txn()?; + let meta = db.get(&txn, uuid.as_bytes())?; + Ok(meta) + }) + .await + .expect("thread died") + } } -impl MapIndexStore { - fn new(root: impl AsRef) -> Self { - let mut root = root.as_ref().to_owned(); - root.push("indexes/"); - let meta_store = Arc::new(RwLock::new(HashMap::new())); - let index_store = Arc::new(RwLock::new(HashMap::new())); - Self { - meta_store, - index_store, - root, - } - } +fn open_index(path: impl AsRef, size: usize) -> Result { + create_dir_all(&path).expect("can't create db"); + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; + Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs index 3fe8a3f59..a690abaf4 100644 --- a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs +++ b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs @@ -12,8 +12,7 @@ use heed::{ }; use log::{error, info}; use milli::Index; -use rayon::ThreadPool; -use serde::{Deserialize, Serialize}; +use rayon::ThreadPool; use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::update_handler::UpdateHandler; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 85469728b..2c181817b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -71,7 +71,7 @@ impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path)?; Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 32eda7bcb..708e8fd19 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -68,10 +68,11 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> Self { + fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_owned().join("update_files"); - create_dir_all(&path).unwrap(); - Self { store, inbox, path } + create_dir_all(&path)?; + assert!(path.exists()); + Ok(Self { store, inbox, path }) } async fn run(mut self) { @@ -211,15 +212,15 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); let store = MapUpdateStoreStore::new(index_handle, &path); - let actor = UpdateActor::new(store, receiver, path); + let actor = UpdateActor::new(store, receiver, path)?; tokio::task::spawn(actor.run()); - Self { sender } + Ok(Self { sender }) } pub async fn update( diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index a369790f3..d8d39d922 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -277,8 +277,7 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? - } + }).await? } } #[cfg(test)] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 943284736..090524601 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -12,15 +12,17 @@ use super::service::Service; pub struct Server { pub service: Service, + // hod ownership to the tempdir while we use the server instance. + _dir: tempdir::TempDir, } impl Server { pub async fn new() -> Self { - let tmp_dir = TempDir::new("meilisearch").unwrap(); + let dir = TempDir::new("meilisearch").unwrap(); let opt = Opt { - db_path: tmp_dir.path().join("db"), - dumps_dir: tmp_dir.path().join("dump"), + db_path: dir.path().join("db"), + dumps_dir: dir.path().join("dump"), dump_batch_size: 16, http_addr: "127.0.0.1:7700".to_owned(), master_key: None, @@ -55,6 +57,7 @@ impl Server { Server { service, + _dir: dir, } } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e5fbbde35..ba5d9651c 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -35,7 +35,8 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + println!("response: {}", response); index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); From 79a4bc8129349d63ed505a1e4a7a79aab8b2828a Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 19:40:18 +0100 Subject: [PATCH 45/69] use meta from milli --- Cargo.lock | 228 ++++++++++++------ meilisearch-http/Cargo.toml | 3 +- meilisearch-http/src/index/search.rs | 12 +- meilisearch-http/src/index/updates.rs | 2 +- .../src/index_controller/index_actor.rs | 172 ++++--------- 5 files changed, 207 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84c9b6854..575f03d98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,8 +159,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -169,8 +169,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -363,9 +363,9 @@ version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -462,9 +462,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -473,9 +473,9 @@ version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -925,9 +925,9 @@ version = "0.99.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -982,9 +982,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1029,9 +1029,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "synstructure", ] @@ -1170,9 +1170,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1721,6 +1721,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "logging_timer" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" +dependencies = [ + "log", + "logging_timer_proc_macros", +] + +[[package]] +name = "logging_timer_proc_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" +dependencies = [ + "log", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1866,11 +1888,11 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1884,6 +1906,7 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "logging_timer", "meilisearch-tokenizer", "memmap", "num-traits", @@ -1897,9 +1920,11 @@ dependencies = [ "roaring", "serde", "serde_json", + "slice-group-by", "smallstr", "smallvec", "tempfile", + "tinytemplate", "uuid", ] @@ -2170,9 +2195,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2248,9 +2273,9 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2259,9 +2284,9 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2301,9 +2326,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "version_check", ] @@ -2313,8 +2338,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "version_check", ] @@ -2330,13 +2355,22 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.1", ] [[package]] @@ -2345,13 +2379,22 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.24", ] [[package]] @@ -2799,9 +2842,9 @@ version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2987,11 +3030,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", - "syn", + "syn 1.0.63", ] [[package]] @@ -3001,13 +3044,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.63", ] [[package]] @@ -3041,9 +3084,20 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", ] [[package]] @@ -3052,9 +3106,9 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "unicode-xid 0.2.1", ] [[package]] @@ -3072,10 +3126,10 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", + "unicode-xid 0.2.1", ] [[package]] @@ -3146,9 +3200,9 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3212,10 +3266,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "standback", - "syn", + "syn 1.0.63", +] + +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", ] [[package]] @@ -3281,9 +3345,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3479,6 +3543,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -3592,9 +3662,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-shared", ] @@ -3616,7 +3686,7 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ - "quote", + "quote 1.0.9", "wasm-bindgen-macro-support", ] @@ -3626,9 +3696,9 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3788,8 +3858,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.24", + "syn 1.0.63", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b09c52b87..15319dea6 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -38,7 +38,8 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { path = "../../milli/milli" } +#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1264d49d6..fb830ec60 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -6,7 +6,7 @@ use either::Either; use anyhow::bail; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{FacetCondition, facet::FacetValue}; +use milli::{FacetCondition, MatchingWords, facet::FacetValue}; use serde::{Serialize, Deserialize}; use serde_json::{Value, Map}; @@ -71,7 +71,7 @@ impl Index { let milli::SearchResult { documents_ids, - found_words, + matching_words, candidates, .. } = search.execute()?; @@ -102,7 +102,7 @@ impl Index { for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } documents.push(object); } @@ -173,7 +173,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer } } - pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -183,7 +183,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { let analyzed = self.analyzer.analyze(&old_string); for (word, token) in analyzed.reconstruct() { if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); + let to_highlight = words_to_highlight.matches(token.text()); if to_highlight { string.push_str("") } @@ -215,7 +215,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { pub fn highlight_record( &self, object: &mut Map, - words_to_highlight: &HashSet, + words_to_highlight: &MatchingWords, attributes_to_highlight: &HashSet, ) { // TODO do we need to create a string for element that are not and needs to be highlight? diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index d339406f7..129a340a7 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -12,7 +12,7 @@ use super::Index; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: usize }, + DocumentDeletion { deleted: u64 }, Other, } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index d8dc5f014..b5d5792b2 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::fs::{create_dir_all, remove_dir_all, File}; +use std::fs::{create_dir_all, File}; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -8,15 +8,13 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::{ - types::{ByteSlice, SerdeBincode}, - Database, Env, EnvOpenOptions, -}; +use heed::EnvOpenOptions; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::{sync::{mpsc, oneshot, RwLock}}; +use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::spawn_blocking; +use tokio::fs::remove_dir_all; use uuid::Uuid; use super::get_arc_ownership_blocking; @@ -36,12 +34,21 @@ type UpdateResult = std::result::Result, Failed, updated_at: DateTime, primary_key: Option, } +impl IndexMeta { + fn new(index: &Index) -> Result { + let txn = index.read_txn()?; + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; + let primary_key = index.primary_key(&txn)?.map(String::from); + Ok(Self { primary_key, updated_at, created_at }) + } +} + enum IndexMsg { CreateIndex { uuid: Uuid, @@ -106,14 +113,9 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static; + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: Uuid) -> Result>; - async fn get_meta(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -245,7 +247,11 @@ impl IndexActor { uuid: Uuid, primary_key: Option, ) -> Result { - self.store.create_index(uuid, primary_key).await + let index = self.store.create(uuid, primary_key).await?; + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(meta) } async fn handle_update( @@ -256,16 +262,13 @@ impl IndexActor { debug!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); - let handle = self - .store - .update_index(uuid, |index| { - let handle = - spawn_blocking(move || update_handler.handle_update(meta, data, index)); - Ok(handle) - }) - .await?; - - handle.await.map_err(|e| IndexError::Error(e.into())) + let index = match self.store.get(uuid.clone()).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, + }; + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) } async fn handle_settings(&self, uuid: Uuid) -> Result { @@ -338,8 +341,15 @@ impl IndexActor { } async fn handle_get_meta(&self, uuid: Uuid) -> Result> { - let result = self.store.get_meta(uuid).await?; - Ok(result) + match self.store.get(uuid).await? { + Some(index) => { + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(Some(meta)) + } + None => Ok(None), + } } } @@ -451,24 +461,15 @@ impl IndexActorHandle { } struct HeedIndexStore { - env: Env, - db: Database>, index_store: AsyncMap, path: PathBuf, } impl HeedIndexStore { fn new(path: impl AsRef) -> anyhow::Result { - let mut options = EnvOpenOptions::new(); - options.map_size(1_073_741_824); //1GB let path = path.as_ref().join("indexes/"); - create_dir_all(&path)?; - let env = options.open(&path)?; - let db = env.create_database(None)?; let index_store = Arc::new(RwLock::new(HashMap::new())); Ok(Self { - env, - db, index_store, path, }) @@ -477,76 +478,22 @@ impl HeedIndexStore { #[async_trait::async_trait] impl IndexStore for HeedIndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { let path = self.path.join(format!("index-{}", uuid)); - if path.exists() { return Err(IndexError::IndexAlreadyExists); } - let env = self.env.clone(); - let db = self.db.clone(); - let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> { - let now = Utc::now(); - let meta = IndexMeta { - uuid: uuid.clone(), - created_at: now.clone(), - updated_at: now, - primary_key, - }; - let mut txn = env.write_txn()?; - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - + let index = spawn_blocking(move || -> Result { let index = open_index(&path, 4096 * 100_000)?; - - Ok((index, meta)) + Ok(index) }) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid.clone(), index); + self.index_store.write().await.insert(uuid.clone(), index.clone()); - Ok(meta) - } - - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static, - { - let guard = self.index_store.read().await; - let index = match guard.get(&uuid) { - Some(index) => index.clone(), - None => { - drop(guard); - self.create_index(uuid.clone(), None).await?; - self.index_store - .read() - .await - .get(&uuid) - .expect("Index should exist") - .clone() - } - }; - - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let mut txn = env.write_txn()?; - let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index"); - match f(index) { - Ok(r) => { - meta.updated_at = Utc::now(); - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - Ok(r) - } - Err(e) => Err(e), - } - }) - .await - .expect("thread died") + Ok(index) } async fn get(&self, uuid: Uuid) -> Result> { @@ -561,54 +508,33 @@ impl IndexStore for HeedIndexStore { return Ok(None); } - // TODO: set this info from the database let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; self.index_store .write() .await .insert(uuid.clone(), index.clone()); - println!("here"); Ok(Some(index)) } } } async fn delete(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); let db_path = self.path.join(format!("index-{}", uuid)); - spawn_blocking(move || -> Result<()> { - let mut txn = env.write_txn()?; - db.delete(&mut txn, uuid.as_bytes())?; - txn.commit()?; - remove_dir_all(db_path).unwrap(); - Ok(()) - }) - .await - .expect("thread died")?; + remove_dir_all(db_path).await + .map_err(|e| IndexError::Error(e.into()))?; let index = self.index_store.write().await.remove(&uuid); Ok(index) } - - async fn get_meta(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let txn = env.read_txn()?; - let meta = db.get(&txn, uuid.as_bytes())?; - Ok(meta) - }) - .await - .expect("thread died") - } } fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path).expect("can't create db"); + create_dir_all(&path) + .map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &path) + .map_err(|e| IndexError::Error(e))?; Ok(Index(Arc::new(index))) } From 3f68460d6c3dc10e5294e013c531e2fb5ed3ad76 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 20:58:51 +0100 Subject: [PATCH 46/69] fix update dedup --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + .../src/index_controller/update_actor.rs | 69 +++++++--------- .../src/index_controller/update_store.rs | 79 +++++++++++-------- 4 files changed, 79 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 575f03d98..3620a6885 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1822,6 +1822,7 @@ dependencies = [ "milli", "mime", "once_cell", + "parking_lot", "rand 0.7.3", "rayon", "regex", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 15319dea6..7b09b3042 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -62,6 +62,7 @@ either = "1.6.1" async-trait = "0.1.42" thiserror = "1.0.24" async-stream = "0.3.0" +parking_lot = "0.11.1" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 708e8fd19..8614e297b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -3,18 +3,17 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use itertools::Itertools; -use log::info; use super::index_actor::IndexActorHandle; +use log::info; use thiserror::Error; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, oneshot, RwLock}; use uuid::Uuid; +use super::get_arc_ownership_blocking; use crate::index::UpdateResult; use crate::index_controller::{UpdateMeta, UpdateStatus}; -use super::get_arc_ownership_blocking; pub type Result = std::result::Result; type UpdateStore = super::update_store::UpdateStore; @@ -68,7 +67,11 @@ where D: AsRef<[u8]> + Sized + 'static, S: UpdateStoreStore, { - fn new(store: S, inbox: mpsc::Receiver>, path: impl AsRef) -> anyhow::Result { + fn new( + store: S, + inbox: mpsc::Receiver>, + path: impl AsRef, + ) -> anyhow::Result { let path = path.as_ref().to_owned().join("update_files"); create_dir_all(&path)?; assert!(path.exists()); @@ -87,12 +90,12 @@ where meta, data, ret, - }) => { + }) => { let _ = ret.send(self.handle_update(uuid, meta, data).await); } Some(ListUpdates { uuid, ret }) => { let _ = ret.send(self.handle_list_updates(uuid).await); - } , + } Some(GetUpdate { uuid, ret, id }) => { let _ = ret.send(self.handle_get_update(uuid, id).await); } @@ -113,13 +116,15 @@ where let update_store = self.store.get_or_create(uuid).await?; let update_file_id = uuid::Uuid::new_v4(); let path = self.path.join(format!("update_{}", update_file_id)); - let mut file = File::create(&path).await + let mut file = File::create(&path) + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; while let Some(bytes) = payload.recv().await { match bytes { Ok(bytes) => { - file.write_all(bytes.as_ref()).await + file.write_all(bytes.as_ref()) + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; } Err(e) => { @@ -128,7 +133,8 @@ where } } - file.flush().await + file.flush() + .await .map_err(|e| UpdateError::Error(Box::new(e)))?; let file = file.into_std().await; @@ -144,50 +150,33 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_list_updates( - &self, - uuid: Uuid, - ) -> Result> { - let store = self.store.get(&uuid).await?; + async fn handle_list_updates(&self, uuid: Uuid) -> Result> { + let update_store = self.store.get(&uuid).await?; tokio::task::spawn_blocking(move || { - let result = match store { - Some(update_store) => { - let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { - Ok(processing - .map(UpdateStatus::from) - .into_iter() - .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) - .chain(aborted.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(processed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(failed.filter_map(std::result::Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .sorted_by(|a, b| a.id().cmp(&b.id())) - .collect()) - }) - .map_err(|e| UpdateError::Error(Box::new(e)))?; - Ok(updates) - } - None => Err(UpdateError::UnexistingIndex(uuid)), - }; - result - }).await + let result = update_store + .ok_or(UpdateError::UnexistingIndex(uuid))? + .list() + .map_err(|e| UpdateError::Error(e.into()))?; + Ok(result) + }) + .await .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { - let store = self.store + let store = self + .store .get(&uuid) .await? .ok_or(UpdateError::UnexistingIndex(uuid))?; - let result = store.meta(id) + let result = store + .meta(id) .map_err(|e| UpdateError::Error(Box::new(e)))?; Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store - .delete(&uuid) - .await?; + let store = self.store.delete(&uuid).await?; if let Some(store) = store { tokio::task::spawn(async move { diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index fe04b205d..a7a65e494 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -1,6 +1,6 @@ use std::fs::remove_file; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{Database, Env, EnvOpenOptions}; @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; +use parking_lot::RwLock; use crate::index_controller::updates::*; @@ -206,7 +207,7 @@ where // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); - self.processing.write().unwrap().replace(processing.clone()); + self.processing.write().replace(processing.clone()); let file = File::open(&content_path)?; // Process the pending update using the provided user function. let result = handler.handle_update(processing, file)?; @@ -216,7 +217,7 @@ where // we must remove the content from the pending and processing stores and // write the *new* meta to the processed-meta store and commit. let mut wtxn = self.env.write_txn()?; - self.processing.write().unwrap().take(); + self.processing.write().take(); self.pending_meta.delete(&mut wtxn, &first_id)?; remove_file(&content_path)?; self.pending.delete(&mut wtxn, &first_id)?; @@ -232,36 +233,52 @@ where } } - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { + pub fn list(&self) -> anyhow::Result>> { let rtxn = self.env.read_txn()?; + let mut updates = Vec::new(); - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; + let processing = self.processing.read(); + if let Some(ref processing) = *processing { + let update = UpdateStatus::from(processing.clone()); + updates.push(update); + } - // We execute the user defined function with both iterators. - (f)( - processing, - processed_iter, - aborted_iter, - pending_iter, - failed_iter, - ) + let pending = self + .pending_meta + .iter(&rtxn)? + .filter_map(Result::ok) + .filter_map(|(_, p)| (Some(p.id()) != processing.as_ref().map(|p| p.id())).then(|| p)) + .map(UpdateStatus::from); + + updates.extend(pending); + + let aborted = + self.aborted_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(aborted); + + let processed = + self.processed_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(processed); + + let failed = + self.failed_meta.iter(&rtxn)? + .filter_map(Result::ok) + .map(|(_, p)| p) + .map(UpdateStatus::from); + + updates.extend(failed); + + updates.sort_unstable_by(|a, b| a.id().cmp(&b.id())); + + Ok(updates) } /// Returns the update associated meta or `None` if the update doesn't exist. @@ -269,7 +286,7 @@ where let rtxn = self.env.read_txn()?; let key = BEU64::new(update_id); - if let Some(ref meta) = *self.processing.read().unwrap() { + if let Some(ref meta) = *self.processing.read() { if meta.id() == update_id { return Ok(Some(UpdateStatus::Processing(meta.clone()))); } From 40b3451a4e46cae9300c0f3e279bc459ed9a54a0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:11:58 +0100 Subject: [PATCH 47/69] fix unexisting update store + race conditions --- meilisearch-http/src/index_controller/mod.rs | 1 + .../src/index_controller/update_actor.rs | 60 ++++++++++++++----- meilisearch-http/tests/updates/mod.rs | 1 + 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 2c181817b..318b7270c 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -144,6 +144,7 @@ impl IndexController { let name = name.unwrap(); let uuid = self.uuid_resolver.create(name.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; + let _ = self.update_handle.create(uuid).await?; let meta = IndexMetadata { name, meta }; Ok(meta) diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 8614e297b..2c16eb60b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -47,6 +47,10 @@ enum UpdateMsg { uuid: Uuid, ret: oneshot::Sender>, }, + Create { + uuid: Uuid, + ret: oneshot::Sender>, + } } struct UpdateActor { @@ -102,7 +106,10 @@ where Some(Delete { uuid, ret }) => { let _ = ret.send(self.handle_delete(uuid).await); } - None => {} + Some(Create { uuid, ret }) => { + let _ = ret.send(self.handle_create(uuid).await); + } + None => break, } } } @@ -190,6 +197,11 @@ where Ok(()) } + + async fn handle_create(&self, uuid: Uuid) -> Result<()> { + let _ = self.store.get_or_create(uuid).await?; + Ok(()) + } } #[derive(Clone)] @@ -249,6 +261,13 @@ where let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } + + pub async fn create(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Create { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } } struct MapUpdateStoreStore { @@ -282,7 +301,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) }) - .unwrap(); + .map_err(|e| UpdateError::Error(e.into()))?; let store = e.insert(store); Ok(store.clone()) } @@ -291,22 +310,35 @@ impl UpdateStoreStore for MapUpdateStoreStore { } async fn get(&self, uuid: &Uuid) -> Result>> { - // attemps to get pre-loaded ref to the index - match self.db.read().await.get(uuid) { + let guard = self.db.read().await; + match guard.get(uuid) { Some(uuid) => Ok(Some(uuid.clone())), None => { - // otherwise we try to check if it exists, and load it. + // The index is not found in the found in the loaded indexes, so we attempt to load + // it from disk. We need to acquire a write lock **before** attempting to open the + // index, because someone could be trying to open it at the same time as us. + drop(guard); let path = self.path.clone().join(format!("updates-{}", uuid)); if path.exists() { - let index_handle = self.index_handle.clone(); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); - let store = UpdateStore::open(options, &path, move |meta, file| { - futures::executor::block_on(index_handle.update(meta, file)) - }) - .unwrap(); - self.db.write().await.insert(uuid.clone(), store.clone()); - Ok(Some(store)) + let mut guard = self.db.write().await; + match guard.entry(uuid.clone()) { + Entry::Vacant(entry) => { + // We can safely load the index + let index_handle = self.index_handle.clone(); + let mut options = heed::EnvOpenOptions::new(); + options.map_size(4096 * 100_000); + let store = UpdateStore::open(options, &path, move |meta, file| { + futures::executor::block_on(index_handle.update(meta, file)) + }) + .map_err(|e| UpdateError::Error(e.into()))?; + let store = entry.insert(store.clone()); + Ok(Some(store.clone())) + } + Entry::Occupied(entry) => { + // The index was loaded while we attempted to to iter + Ok(Some(entry.get().clone())) + } + } } else { Ok(None) } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 64b5b560e..713936b8c 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -46,6 +46,7 @@ async fn list_no_updates() { let index = server.index("test"); index.create(None).await; let (response, code) = index.list_updates().await; + println!("response: {}", response); assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } From 30dd790884b01d3bf43b0b38fc7e5c3439536af0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:23:48 +0100 Subject: [PATCH 48/69] handle badly formatted index uid --- .../src/index_controller/uuid_resolver.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index d8d39d922..4fe079518 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -72,10 +72,16 @@ impl UuidResolverActor { } async fn handle_create(&self, name: String) -> Result { + if !is_index_uid_valid(&name) { + return Err(UuidError::BadlyFormatted(name)) + } self.store.create_uuid(name, true).await } async fn handle_get_or_create(&self, name: String) -> Result { + if !is_index_uid_valid(&name) { + return Err(UuidError::BadlyFormatted(name)) + } self.store.create_uuid(name, false).await } @@ -99,6 +105,10 @@ impl UuidResolverActor { } } +fn is_index_uid_valid(uid: &str) -> bool { + uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') +} + #[derive(Clone)] pub struct UuidResolverHandle { sender: mpsc::Sender, @@ -171,6 +181,8 @@ pub enum UuidError { Heed(#[from] heed::Error), #[error("Uuid error: {0}")] Uuid(#[from] uuid::Error), + #[error("Badly formatted index uid: {0}")] + BadlyFormatted(String), } #[async_trait::async_trait] From 66b64c1f8078f2000a0e9605a564e977e0e13864 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:33:31 +0100 Subject: [PATCH 49/69] correct error on settings delete unexisting index --- meilisearch-http/src/data/updates.rs | 5 +++-- meilisearch-http/src/index_controller/mod.rs | 8 ++++++-- meilisearch-http/src/routes/settings/mod.rs | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index c6e30ea02..31c886673 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -23,9 +23,10 @@ impl Data { pub async fn update_settings( &self, index: String, - settings: Settings + settings: Settings, + create: bool, ) -> anyhow::Result { - let update = self.index_controller.update_settings(index, settings).await?; + let update = self.index_controller.update_settings(index, settings, create).await?; Ok(update.into()) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 318b7270c..56693e204 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -129,8 +129,12 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index_uid).await?; + pub async fn update_settings(&self, index_uid: String, settings: Settings, create: bool) -> anyhow::Result { + let uuid = if create { + self.uuid_resolver.get_or_create(index_uid).await? + } else { + self.uuid_resolver.resolve(index_uid).await? + }; let meta = UpdateMeta::Settings(settings); // Nothing so send, drop the sender right away, as not to block the update actor. let (_, receiver) = mpsc::channel(1); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 0f904f572..4e7219b1c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -26,7 +26,7 @@ macro_rules! make_setting_route { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -48,7 +48,7 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, true).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -137,7 +137,7 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.into_inner(), body.into_inner()).await { + match data.update_settings(index_uid.into_inner(), body.into_inner(), true).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -170,7 +170,7 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.into_inner(), settings).await { + match data.update_settings(index_uid.into_inner(), settings, false).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) From 8617bcf8bdd723926394a7d9e9ea1c793e98dc54 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:39:16 +0100 Subject: [PATCH 50/69] add ranking rules --- meilisearch-http/src/index/mod.rs | 8 +++++++- meilisearch-http/src/index/updates.rs | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index c50c2873c..1afeb3478 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -44,11 +44,17 @@ impl Index { .map(|(k, v)| (k, v.to_string())) .collect(); + let criteria = self + .criteria(&txn)? + .into_iter() + .map(|c| c.to_string()) + .collect(); + Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), faceted_attributes: Some(Some(faceted_attributes)), - criteria: None, + ranking_rules: Some(Some(criteria)), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 129a340a7..70c2dfc2b 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -42,7 +42,7 @@ pub struct Settings { deserialize_with = "deserialize_some", skip_serializing_if = "Option::is_none", )] - pub criteria: Option>>, + pub ranking_rules: Option>>, } impl Settings { @@ -51,7 +51,7 @@ impl Settings { displayed_attributes: Some(None), searchable_attributes: Some(None), faceted_attributes: Some(None), - criteria: Some(None), + ranking_rules: Some(None), } } } @@ -164,7 +164,7 @@ impl Index { } // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { + if let Some(ref criteria) = settings.ranking_rules { match criteria { Some(criteria) => builder.set_criteria(criteria.clone()), None => builder.reset_criteria(), From 271c8ba991834560b35c25ac7bd3af0724e88436 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:47:29 +0100 Subject: [PATCH 51/69] change index name to uid --- meilisearch-http/src/data/mod.rs | 2 +- meilisearch-http/src/index_controller/mod.rs | 70 ++++++++++---------- meilisearch-http/tests/index/create_index.rs | 8 +-- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index b011c18b3..f2b963eec 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -90,7 +90,7 @@ impl Data { pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { - name: Some(name.as_ref().to_string()), + uid: Some(name.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 56693e204..c76cd8a70 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -26,7 +26,7 @@ pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { - name: String, + uid: String, #[serde(flatten)] meta: index_actor::IndexMeta, } @@ -47,7 +47,7 @@ pub enum UpdateMeta { #[derive(Clone, Debug)] pub struct IndexSettings { - pub name: Option, + pub uid: Option, pub primary_key: Option, } @@ -77,13 +77,13 @@ impl IndexController { pub async fn add_documents( &self, - index: String, + uid: String, method: milli::update::IndexDocumentsMethod, format: milli::update::UpdateFormat, mut payload: Payload, primary_key: Option, ) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index).await?; + let uuid = self.uuid_resolver.get_or_create(uid).await?; let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; let (sender, receiver) = mpsc::channel(10); @@ -106,16 +106,16 @@ impl IndexController { Ok(status) } - pub async fn clear_documents(&self, index: String) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?; + pub async fn clear_documents(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::ClearDocuments; let (_, receiver) = mpsc::channel(1); let status = self.update_handle.update(meta, receiver, uuid).await?; Ok(status) } - pub async fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(index).await?; + pub async fn delete_documents(&self, uid: String, document_ids: Vec) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -129,11 +129,11 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, index_uid: String, settings: Settings, create: bool) -> anyhow::Result { + pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { let uuid = if create { - self.uuid_resolver.get_or_create(index_uid).await? + self.uuid_resolver.get_or_create(uid).await? } else { - self.uuid_resolver.resolve(index_uid).await? + self.uuid_resolver.resolve(uid).await? }; let meta = UpdateMeta::Settings(settings); // Nothing so send, drop the sender right away, as not to block the update actor. @@ -144,36 +144,36 @@ impl IndexController { } pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let IndexSettings { name, primary_key } = index_settings; - let name = name.unwrap(); - let uuid = self.uuid_resolver.create(name.clone()).await?; + let IndexSettings { uid: name, primary_key } = index_settings; + let uid = name.unwrap(); + let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; - let meta = IndexMetadata { name, meta }; + let meta = IndexMetadata { uid, meta }; Ok(meta) } - pub async fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { + pub async fn delete_index(&self, uid: String) -> anyhow::Result<()> { let uuid = self.uuid_resolver - .delete(index_uid) + .delete(uid) .await?; self.update_handle.delete(uuid.clone()).await?; self.index_handle.delete(uuid).await?; Ok(()) } - pub async fn update_status(&self, index: String, id: u64) -> anyhow::Result> { + pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index) + .resolve(uid) .await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } - pub async fn all_update_status(&self, index: String) -> anyhow::Result> { + pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index).await?; + .resolve(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -183,9 +183,9 @@ impl IndexController { let mut ret = Vec::new(); - for (name, uuid) in uuids { + for (uid, uuid) in uuids { if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { - let meta = IndexMetadata { name, meta }; + let meta = IndexMetadata { uid, meta }; ret.push(meta); } } @@ -193,9 +193,9 @@ impl IndexController { Ok(ret) } - pub async fn settings(&self, index: String) -> anyhow::Result { + pub async fn settings(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) @@ -203,13 +203,13 @@ impl IndexController { pub async fn documents( &self, - index: String, + uid: String, offset: usize, limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result> { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) @@ -217,33 +217,33 @@ impl IndexController { pub async fn document( &self, - index: String, + uid: String, doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result { let uuid = self.uuid_resolver - .resolve(index.clone()) + .resolve(uid.clone()) .await?; let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } - fn update_index(&self, name: String, index_settings: IndexSettings) -> anyhow::Result { + fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { todo!() } - pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await?; + pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.index_handle.search(uuid, query).await?; Ok(result) } - pub async fn get_index(&self, name: String) -> anyhow::Result> { - let uuid = self.uuid_resolver.resolve(name.clone()).await?; + pub async fn get_index(&self, uid: String) -> anyhow::Result> { + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let result = self.index_handle .get_index_meta(uuid) .await? - .map(|meta| IndexMetadata { name, meta }); + .map(|meta| IndexMetadata { uid, meta }); Ok(result) } } diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index c26941b91..b7c85d748 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,14 +7,15 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; + println!("response: {}", response); + assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] @@ -25,12 +26,11 @@ async fn create_index_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in From 7d9637861fc71c29ff287313657ca0bfc2b95d8a Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 22:54:38 +0100 Subject: [PATCH 52/69] fix add primary key on index creation --- meilisearch-http/src/index_controller/index_actor.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index b5d5792b2..10a8db9f4 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -486,6 +486,11 @@ impl IndexStore for HeedIndexStore { let index = spawn_blocking(move || -> Result { let index = open_index(&path, 4096 * 100_000)?; + if let Some(primary_key) = primary_key { + let mut txn = index.write_txn()?; + index.put_primary_key(&mut txn, &primary_key)?; + txn.commit()?; + } Ok(index) }) .await From e4d45b05002838564bd6ac0c0ea3fdaefe3e6266 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 00:37:43 +0100 Subject: [PATCH 53/69] fix various bugs --- .../src/index_controller/index_actor.rs | 1 + meilisearch-http/src/index_controller/mod.rs | 11 +++- .../src/index_controller/uuid_resolver.rs | 55 +------------------ meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 6 +- meilisearch-http/tests/index/get_index.rs | 6 +- .../tests/settings/get_settings.rs | 11 ++-- 7 files changed, 27 insertions(+), 65 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 10a8db9f4..8ae3f8946 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -343,6 +343,7 @@ impl IndexActor { async fn handle_get_meta(&self, uuid: Uuid) -> Result> { match self.store.get(uuid).await? { Some(index) => { + println!("geting meta yoyo"); let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index c76cd8a70..0524b6e1b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -131,7 +131,16 @@ impl IndexController { pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { let uuid = if create { - self.uuid_resolver.get_or_create(uid).await? + let uuid = self.uuid_resolver.get_or_create(uid).await?; + // We need to create the index upfront, since it would otherwise only be created when + // the update is processed. This would make calls to GET index to fail until the update + // is complete. Since this is get or create, we ignore the error when the index already + // exists. + match self.index_handle.create_index(uuid.clone(), None).await { + Ok(_) | Err(index_actor::IndexError::IndexAlreadyExists) => (), + Err(e) => return Err(e.into()), + } + uuid } else { self.uuid_resolver.resolve(uid).await? }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 4fe079518..aa83db2c7 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -270,7 +270,7 @@ impl UuidStore for HeedUuidStore { let uuid = Uuid::from_slice(uuid)?; db.delete(&mut txn, &name)?; txn.commit()?; - Ok(None) + Ok(Some(uuid)) } None => Ok(None) } @@ -289,57 +289,6 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? } -} - -#[cfg(test)] -mod test { - use std::collections::HashMap; - use std::collections::hash_map::Entry; - use std::sync::Arc; - - use tokio::sync::RwLock; - - use super::*; - - struct MapUuidStore(Arc>>); - - #[async_trait::async_trait] - impl UuidStore for MapUuidStore { - async fn create_uuid(&self, name: String, err: bool) -> Result { - match self.0.write().await.entry(name) { - Entry::Occupied(entry) => { - if err { - Err(UuidError::NameAlreadyExist) - } else { - Ok(entry.get().clone()) - } - } - Entry::Vacant(entry) => { - let uuid = Uuid::new_v4(); - let uuid = entry.insert(uuid); - Ok(uuid.clone()) - } - } - } - - async fn get_uuid(&self, name: String) -> Result> { - Ok(self.0.read().await.get(&name).cloned()) - } - - async fn delete(&self, name: String) -> Result> { - Ok(self.0.write().await.remove(&name)) - } - - async fn list(&self) -> Result> { - let list = self - .0 - .read() - .await - .iter() - .map(|(name, uuid)| (name.to_owned(), uuid.clone())) - .collect(); - Ok(list) - } + }).await? } } diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index b7c85d748..219ad692b 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -70,5 +70,5 @@ async fn test_create_multiple_indexes() { assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); assert_eq!(index3.get().await.1, 200); - assert_eq!(index4.get().await.1, 400); + assert_eq!(index4.get().await.1, 404); } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 39e79daaf..b15cc306e 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,13 +6,17 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; + println!("response: {}", _response); + assert_eq!(code, 200); let (_response, code) = index.delete().await; + println!("response: {}", _response); + assert_eq!(code, 200); - assert_eq!(index.get().await.1, 400); + assert_eq!(index.get().await.1, 404); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 3b8390551..a1a95eef8 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -13,12 +13,11 @@ async fn create_and_get_index() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } // TODO: partial test since we are testing error, amd error is not yet fully implemented in @@ -30,7 +29,7 @@ async fn get_unexisting_index() { let (_response, code) = index.get().await; - assert_eq!(code, 400); + assert_eq!(code, 404); } #[actix_rt::test] @@ -55,5 +54,4 @@ async fn list_multiple_indexes() { assert_eq!(arr.len(), 2); assert!(arr.iter().find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null).is_some()); assert!(arr.iter().find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key").is_some()); - } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index ba5d9651c..b8217c5f7 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -43,7 +43,8 @@ async fn test_partial_update() { assert_eq!(response["displayedAttributes"],json!(["foo"])); assert_eq!(response["searchableAttributes"],json!(["*"])); - index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + println!("resp: {}", response); index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -125,10 +126,10 @@ macro_rules! test_setting_routes { .chars() .map(|c| if c == '_' { '-' } else { c }) .collect::()); - let (_response, code) = server.service.post(url, serde_json::Value::Null).await; - assert_eq!(code, 200); - let (_response, code) = server.index("test").get().await; - assert_eq!(code, 200); + let (response, code) = server.service.post(url, serde_json::Value::Null).await; + assert_eq!(code, 200, "{}", response); + let (response, code) = server.index("test").get().await; + assert_eq!(code, 200, "{}", response); } #[actix_rt::test] From 77d5dd452fb0e0b95269fb62fa63e60470537fe3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:16:54 +0100 Subject: [PATCH 54/69] remove open_or_create --- meilisearch-http/src/index_controller/uuid_resolver.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index aa83db2c7..fcf417248 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -200,13 +200,6 @@ struct HeedUuidStore { db: Database, } -fn open_or_create_database(env: &Env, name: Option<&str>) -> heed::Result> { - match env.open_database(name)? { - Some(db) => Ok(db), - None => env.create_database(name), - } -} - impl HeedUuidStore { fn new(path: impl AsRef) -> anyhow::Result { let path = path.as_ref().join("index_uuids"); @@ -214,7 +207,7 @@ impl HeedUuidStore { let mut options = EnvOpenOptions::new(); options.map_size(1_073_741_824); // 1GB let env = options.open(path)?; - let db = open_or_create_database(&env, None)?; + let db = env.create_database(None)?; Ok(Self { env, db }) } } From c4846dafca3137f561357c55abbf973b87c51888 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:48:43 +0100 Subject: [PATCH 55/69] implement update index --- meilisearch-http/src/data/updates.rs | 15 +++--- .../src/index_controller/index_actor.rs | 54 ++++++++++++++++++- meilisearch-http/src/index_controller/mod.rs | 14 ++++- meilisearch-http/src/routes/index.rs | 2 +- meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/update_index.rs | 3 +- 6 files changed, 75 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 31c886673..0e2c56e42 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,7 +1,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use actix_web::web::Payload; -use crate::index_controller::{UpdateStatus, IndexMetadata}; +use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; use crate::index::Settings; use super::Data; @@ -63,18 +63,17 @@ impl Data { self.index_controller.all_update_status(index.as_ref().to_string()).await } - pub fn update_index( + pub async fn update_index( &self, name: impl AsRef, primary_key: Option>, new_name: Option> ) -> anyhow::Result { - todo!() - //let settings = IndexSettings { - //name: new_name.map(|s| s.as_ref().to_string()), - //primary_key: primary_key.map(|s| s.as_ref().to_string()), - //}; + let settings = IndexSettings { + uid: new_name.map(|s| s.as_ref().to_string()), + primary_key: primary_key.map(|s| s.as_ref().to_string()), + }; - //self.index_controller.update_index(name, settings) + self.index_controller.update_index(name.as_ref().to_string(), settings).await } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 8ae3f8946..63be1eec5 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -17,7 +17,7 @@ use tokio::task::spawn_blocking; use tokio::fs::remove_dir_all; use uuid::Uuid; -use super::get_arc_ownership_blocking; +use super::{IndexSettings, get_arc_ownership_blocking}; use super::update_handler::UpdateHandler; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; @@ -42,6 +42,10 @@ pub struct IndexMeta { impl IndexMeta { fn new(index: &Index) -> Result { let txn = index.read_txn()?; + Self::new_txn(index, &txn) + } + + fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); @@ -90,6 +94,11 @@ enum IndexMsg { uuid: Uuid, ret: oneshot::Sender>>, }, + UpdateIndex { + uuid: Uuid, + index_settings: IndexSettings, + ret: oneshot::Sender>, + } } struct IndexActor { @@ -109,6 +118,8 @@ pub enum IndexError { UnexistingIndex, #[error("Heed error: {0}")] HeedError(#[from] heed::Error), + #[error("Existing primary key")] + ExistingPrimaryKey, } #[async_trait::async_trait] @@ -230,6 +241,9 @@ impl IndexActor { GetMeta { uuid, ret } => { let _ = ret.send(self.handle_get_meta(uuid).await); } + UpdateIndex { uuid, index_settings, ret } => { + let _ = ret.send(self.handle_update_index(uuid, index_settings).await); + } } } @@ -352,6 +366,33 @@ impl IndexActor { None => Ok(None), } } + + async fn handle_update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { + let index = self.store + .get(uuid) + .await? + .ok_or(IndexError::UnexistingIndex)?; + + spawn_blocking(move || { + match index_settings.primary_key { + Some(ref primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexError::ExistingPrimaryKey) + } + index.put_primary_key(&mut txn, primary_key)?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) + }, + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + }, + } + }).await + .map_err(|e| IndexError::Error(e.into()))? + } } #[derive(Clone)] @@ -459,6 +500,17 @@ impl IndexActorHandle { let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } + + pub async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::UpdateIndex { uuid, index_settings, ret }; + let _ = self.read_sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } } struct HeedIndexStore { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 0524b6e1b..1433562ff 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -9,6 +9,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; +use anyhow::bail; use actix_web::web::{Bytes, Payload}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; @@ -237,8 +238,17 @@ impl IndexController { Ok(document) } - fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { - todo!() + pub async fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { + if index_settings.uid.is_some() { + bail!("Can't change the index uid.") + } + + let uuid = self.uuid_resolver + .resolve(uid.clone()) + .await?; + let meta = self.index_handle.update_index(uuid, index_settings).await?; + let meta = IndexMetadata { uid, meta }; + Ok(meta) } pub async fn search(&self, uid: String, query: SearchQuery) -> anyhow::Result { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 5c6e3f5a9..68c94798a 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -95,7 +95,7 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()) { + match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 219ad692b..718e35899 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -28,7 +28,7 @@ async fn create_index_with_primary_key() { assert_eq!(response["uid"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); - assert_eq!(response["createdAt"], response["updatedAt"]); + //assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], "primary"); assert_eq!(response.as_object().unwrap().len(), 4); } diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 0078ad892..36670682a 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -13,7 +13,6 @@ async fn update_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert!(response.get("uuid").is_some()); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); @@ -22,7 +21,7 @@ async fn update_primary_key() { assert!(created_at < updated_at); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] From 2ee2e6a9b285be56c3b611de8f1020f49615c3a1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 14:57:24 +0100 Subject: [PATCH 56/69] clean project --- .../actor_index_controller/mod.rs | 127 ---- .../src/index_controller/index_actor.rs | 1 - .../local_index_controller/index_store.rs | 606 ------------------ .../local_index_controller/mod.rs | 228 ------- .../tests/documents/add_documents.rs | 1 - meilisearch-http/tests/index/create_index.rs | 1 - meilisearch-http/tests/index/delete_index.rs | 2 - .../tests/settings/get_settings.rs | 3 - meilisearch-http/tests/updates/mod.rs | 1 - 9 files changed, 970 deletions(-) delete mode 100644 meilisearch-http/src/index_controller/actor_index_controller/mod.rs delete mode 100644 meilisearch-http/src/index_controller/local_index_controller/index_store.rs delete mode 100644 meilisearch-http/src/index_controller/local_index_controller/mod.rs diff --git a/meilisearch-http/src/index_controller/actor_index_controller/mod.rs b/meilisearch-http/src/index_controller/actor_index_controller/mod.rs deleted file mode 100644 index 188d85580..000000000 --- a/meilisearch-http/src/index_controller/actor_index_controller/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -mod index_actor; -mod update_actor; -mod uuid_resolver; -mod update_store; -mod update_handler; - -use std::path::Path; - -use tokio::sync::{mpsc, oneshot}; -use uuid::Uuid; -use super::IndexMetadata; -use futures::stream::StreamExt; -use actix_web::web::Payload; -use super::UpdateMeta; -use crate::index::{SearchResult, SearchQuery}; -use actix_web::web::Bytes; - -use crate::index::Settings; -use super::UpdateStatus; - -pub struct IndexController { - uuid_resolver: uuid_resolver::UuidResolverHandle, - index_handle: index_actor::IndexActorHandle, - update_handle: update_actor::UpdateActorHandle, -} - -enum IndexControllerMsg { - CreateIndex { - uuid: Uuid, - primary_key: Option, - ret: oneshot::Sender>, - }, - Shutdown, -} - -impl IndexController { - pub fn new(path: impl AsRef) -> Self { - let uuid_resolver = uuid_resolver::UuidResolverHandle::new(); - let index_actor = index_actor::IndexActorHandle::new(&path); - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path); - Self { uuid_resolver, index_handle: index_actor, update_handle } - } - - pub async fn add_documents( - &self, - index: String, - method: milli::update::IndexDocumentsMethod, - format: milli::update::UpdateFormat, - mut payload: Payload, - primary_key: Option, - ) -> anyhow::Result { - let uuid = self.uuid_resolver.get_or_create(index).await?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let (sender, receiver) = mpsc::channel(10); - - // It is necessary to spawn a local task to senf the payload to the update handle to - // prevent dead_locking between the update_handle::update that waits for the update to be - // registered and the update_actor that waits for the the payload to be sent to it. - tokio::task::spawn_local(async move { - while let Some(bytes) = payload.next().await { - match bytes { - Ok(bytes) => { sender.send(Ok(bytes)).await; }, - Err(e) => { - let error: Box = Box::new(e); - sender.send(Err(error)).await; }, - } - } - }); - - // This must be done *AFTER* spawning the task. - let status = self.update_handle.update(meta, receiver, uuid).await?; - Ok(status) - } - - fn clear_documents(&self, index: String) -> anyhow::Result { - todo!() - } - - fn delete_documents(&self, index: String, document_ids: Vec) -> anyhow::Result { - todo!() - } - - fn update_settings(&self, index_uid: String, settings: Settings) -> anyhow::Result { - todo!() - } - - pub async fn create_index(&self, index_settings: super::IndexSettings) -> anyhow::Result { - let super::IndexSettings { name, primary_key } = index_settings; - let uuid = self.uuid_resolver.create(name.unwrap()).await?; - let index_meta = self.index_handle.create_index(uuid, primary_key).await?; - Ok(index_meta) - } - - fn delete_index(&self, index_uid: String) -> anyhow::Result<()> { - todo!() - } - - fn swap_indices(&self, index1_uid: String, index2_uid: String) -> anyhow::Result<()> { - todo!() - } - - pub fn index(&self, name: String) -> anyhow::Result>> { - todo!() - } - - fn update_status(&self, index: String, id: u64) -> anyhow::Result> { - todo!() - } - - fn all_update_status(&self, index: String) -> anyhow::Result> { - todo!() - } - - pub fn list_indexes(&self) -> anyhow::Result> { - todo!() - } - - fn update_index(&self, name: String, index_settings: super::IndexSettings) -> anyhow::Result { - todo!() - } - - pub async fn search(&self, name: String, query: SearchQuery) -> anyhow::Result { - let uuid = self.uuid_resolver.resolve(name).await.unwrap().unwrap(); - let result = self.index_handle.search(uuid, query).await?; - Ok(result) - } -} diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 63be1eec5..52a69b964 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -357,7 +357,6 @@ impl IndexActor { async fn handle_get_meta(&self, uuid: Uuid) -> Result> { match self.store.get(uuid).await? { Some(index) => { - println!("geting meta yoyo"); let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; diff --git a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs b/meilisearch-http/src/index_controller/local_index_controller/index_store.rs deleted file mode 100644 index a690abaf4..000000000 --- a/meilisearch-http/src/index_controller/local_index_controller/index_store.rs +++ /dev/null @@ -1,606 +0,0 @@ -use std::fs::{create_dir_all, remove_dir_all}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::Duration; - -use anyhow::{bail, Context}; -use chrono::{DateTime, Utc}; -use dashmap::{mapref::entry::Entry, DashMap}; -use heed::{ - types::{ByteSlice, SerdeJson, Str}, - Database, Env, EnvOpenOptions, RoTxn, RwTxn, -}; -use log::{error, info}; -use milli::Index; -use rayon::ThreadPool; use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use super::update_handler::UpdateHandler; -use super::{UpdateMeta, UpdateResult}; -use crate::option::IndexerOpts; - -type UpdateStore = super::update_store::UpdateStore; - -#[derive(Serialize, Deserialize, Debug, PartialEq)] -pub struct IndexMeta { - update_store_size: u64, - index_store_size: u64, - pub uuid: Uuid, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -impl IndexMeta { - fn open( - &self, - path: impl AsRef, - thread_pool: Arc, - indexer_options: &IndexerOpts, - ) -> anyhow::Result<(Arc, Arc)> { - let update_path = make_update_db_path(&path, &self.uuid); - let index_path = make_index_db_path(&path, &self.uuid); - - create_dir_all(&update_path)?; - create_dir_all(&index_path)?; - - let mut options = EnvOpenOptions::new(); - options.map_size(self.index_store_size as usize); - let index = Arc::new(Index::new(options, index_path)?); - - let mut options = EnvOpenOptions::new(); - options.map_size(self.update_store_size as usize); - let handler = UpdateHandler::new(indexer_options, index.clone(), thread_pool)?; - let update_store = UpdateStore::open(options, update_path, handler)?; - - Ok((index, update_store)) - } -} - -pub struct IndexStore { - env: Env, - name_to_uuid: Database, - uuid_to_index: DashMap, Arc)>, - uuid_to_index_meta: Database>, - - thread_pool: Arc, - indexer_options: IndexerOpts, -} - -impl IndexStore { - pub fn new(path: impl AsRef, indexer_options: IndexerOpts) -> anyhow::Result { - let env = EnvOpenOptions::new() - .map_size(4096 * 100) - .max_dbs(2) - .open(path)?; - - let uuid_to_index = DashMap::new(); - let name_to_uuid = open_or_create_database(&env, Some("name_to_uid"))?; - let uuid_to_index_meta = open_or_create_database(&env, Some("uid_to_index_db"))?; - - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(indexer_options.indexing_jobs.unwrap_or(0)) - .build()?; - let thread_pool = Arc::new(thread_pool); - - Ok(Self { - env, - name_to_uuid, - uuid_to_index, - uuid_to_index_meta, - - thread_pool, - indexer_options, - }) - } - - pub fn delete(&self, index_uid: impl AsRef) -> anyhow::Result<()> { - // we remove the references to the index from the index map so it is not accessible anymore - let mut txn = self.env.write_txn()?; - let uuid = self - .index_uuid(&txn, &index_uid)? - .with_context(|| format!("Index {:?} doesn't exist", index_uid.as_ref()))?; - self.name_to_uuid.delete(&mut txn, index_uid.as_ref())?; - self.uuid_to_index_meta.delete(&mut txn, uuid.as_bytes())?; - txn.commit()?; - // If the index was loaded (i.e it is present in the uuid_to_index map), then we need to - // close it. The process goes as follow: - // - // 1) We want to remove any pending updates from the store. - // 2) We try to get ownership on the update store so we can close it. It may take a - // couple of tries, but since the update store event loop only has a weak reference to - // itself, and we are the only other function holding a reference to it otherwise, we will - // get it eventually. - // 3) We request a closing of the update store. - // 4) We can take ownership on the index, and close it. - // 5) We remove all the files from the file system. - let index_uid = index_uid.as_ref().to_string(); - let path = self.env.path().to_owned(); - if let Some((_, (index, updates))) = self.uuid_to_index.remove(&uuid) { - std::thread::spawn(move || { - info!("Preparing for {:?} deletion.", index_uid); - // this error is non fatal, but may delay the deletion. - if let Err(e) = updates.abort_pendings() { - error!( - "error aborting pending updates when deleting index {:?}: {}", - index_uid, e - ); - } - let updates = get_arc_ownership_blocking(updates); - let close_event = updates.prepare_for_closing(); - close_event.wait(); - info!("closed update store for {:?}", index_uid); - - let index = get_arc_ownership_blocking(index); - let close_event = index.prepare_for_closing(); - close_event.wait(); - - let update_path = make_update_db_path(&path, &uuid); - let index_path = make_index_db_path(&path, &uuid); - - if let Err(e) = remove_dir_all(index_path) { - error!("error removing index {:?}: {}", index_uid, e); - } - - if let Err(e) = remove_dir_all(update_path) { - error!("error removing index {:?}: {}", index_uid, e); - } - - info!("index {:?} deleted.", index_uid); - }); - } - - Ok(()) - } - - fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uuid.get(txn, name.as_ref())? { - Some(bytes) => { - let uuid = Uuid::from_slice(bytes)?; - Ok(Some(uuid)) - } - None => Ok(None), - } - } - - fn retrieve_index( - &self, - txn: &RoTxn, - uid: Uuid, - ) -> anyhow::Result, Arc)>> { - match self.uuid_to_index.entry(uid.clone()) { - Entry::Vacant(entry) => match self.uuid_to_index_meta.get(txn, uid.as_bytes())? { - Some(meta) => { - let path = self.env.path(); - let (index, updates) = - meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; - entry.insert((index.clone(), updates.clone())); - Ok(Some((index, updates))) - } - None => Ok(None), - }, - Entry::Occupied(entry) => { - let (index, updates) = entry.get(); - Ok(Some((index.clone(), updates.clone()))) - } - } - } - - fn get_index_txn( - &self, - txn: &RoTxn, - name: impl AsRef, - ) -> anyhow::Result, Arc)>> { - match self.index_uuid(&txn, name)? { - Some(uid) => self.retrieve_index(&txn, uid), - None => Ok(None), - } - } - - pub fn index( - &self, - name: impl AsRef, - ) -> anyhow::Result, Arc)>> { - let txn = self.env.read_txn()?; - self.get_index_txn(&txn, name) - } - - /// Use this function to perform an update on an index. - /// This function also puts a lock on what index is allowed to perform an update. - pub fn update_index(&self, name: impl AsRef, f: F) -> anyhow::Result<(T, IndexMeta)> - where - F: FnOnce(&Index) -> anyhow::Result, - { - let mut txn = self.env.write_txn()?; - let (index, _) = self - .get_index_txn(&txn, &name)? - .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; - let result = f(index.as_ref()); - match result { - Ok(ret) => { - let meta = self.update_meta(&mut txn, name, |meta| meta.updated_at = Utc::now())?; - txn.commit()?; - Ok((ret, meta)) - } - Err(e) => Err(e), - } - } - - pub fn index_with_meta( - &self, - name: impl AsRef, - ) -> anyhow::Result, IndexMeta)>> { - let txn = self.env.read_txn()?; - let uuid = self.index_uuid(&txn, &name)?; - match uuid { - Some(uuid) => { - let meta = self - .uuid_to_index_meta - .get(&txn, uuid.as_bytes())? - .with_context(|| { - format!("unable to retrieve metadata for index {:?}", name.as_ref()) - })?; - let (index, _) = self - .retrieve_index(&txn, uuid)? - .with_context(|| format!("unable to retrieve index {:?}", name.as_ref()))?; - Ok(Some((index, meta))) - } - None => Ok(None), - } - } - - fn update_meta( - &self, - txn: &mut RwTxn, - name: impl AsRef, - f: F, - ) -> anyhow::Result - where - F: FnOnce(&mut IndexMeta), - { - let uuid = self - .index_uuid(txn, &name)? - .with_context(|| format!("Index {:?} doesn't exist", name.as_ref()))?; - let mut meta = self - .uuid_to_index_meta - .get(txn, uuid.as_bytes())? - .with_context(|| format!("couldn't retrieve metadata for index {:?}", name.as_ref()))?; - f(&mut meta); - self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; - Ok(meta) - } - - pub fn get_or_create_index( - &self, - name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { - let mut txn = self.env.write_txn()?; - match self.get_index_txn(&txn, name.as_ref())? { - Some(res) => Ok(res), - None => { - let uuid = Uuid::new_v4(); - let (index, updates, _) = - self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; - // If we fail to commit the transaction, we must delete the database from the - // file-system. - if let Err(e) = txn.commit() { - self.clean_db(uuid); - return Err(e)?; - } - Ok((index, updates)) - } - } - } - - // Remove all the files and data associated with a db uuid. - fn clean_db(&self, uuid: Uuid) { - let update_db_path = make_update_db_path(self.env.path(), &uuid); - let index_db_path = make_index_db_path(self.env.path(), &uuid); - - remove_dir_all(update_db_path).expect("Failed to clean database"); - remove_dir_all(index_db_path).expect("Failed to clean database"); - - self.uuid_to_index.remove(&uuid); - } - - fn create_index_txn( - &self, - txn: &mut RwTxn, - uuid: Uuid, - name: impl AsRef, - update_store_size: u64, - index_store_size: u64, - ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { - let created_at = Utc::now(); - let updated_at = created_at; - let meta = IndexMeta { - update_store_size, - index_store_size, - uuid: uuid.clone(), - created_at, - updated_at, - }; - - self.name_to_uuid.put(txn, name.as_ref(), uuid.as_bytes())?; - self.uuid_to_index_meta.put(txn, uuid.as_bytes(), &meta)?; - - let path = self.env.path(); - let (index, update_store) = - match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { - Ok(res) => res, - Err(e) => { - self.clean_db(uuid); - return Err(e); - } - }; - - self.uuid_to_index - .insert(uuid, (index.clone(), update_store.clone())); - - Ok((index, update_store, meta)) - } - - /// Same as `get_or_create`, but returns an error if the index already exists. - pub fn create_index( - &self, - name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc, IndexMeta)> { - let uuid = Uuid::new_v4(); - let mut txn = self.env.write_txn()?; - - if self.name_to_uuid.get(&txn, name.as_ref())?.is_some() { - bail!("index {:?} already exists", name.as_ref()) - } - - let result = self.create_index_txn(&mut txn, uuid, name, update_size, index_size)?; - // If we fail to commit the transaction, we must delete the database from the - // file-system. - if let Err(e) = txn.commit() { - self.clean_db(uuid); - return Err(e)?; - } - Ok(result) - } - - /// Returns each index associated with its metadata: - /// (index_name, IndexMeta, primary_key) - /// This method will force all the indexes to be loaded. - pub fn list_indexes(&self) -> anyhow::Result)>> { - let txn = self.env.read_txn()?; - let metas = self.name_to_uuid.iter(&txn)?.filter_map(|entry| { - entry - .map_err(|e| { - error!("error decoding entry while listing indexes: {}", e); - e - }) - .ok() - }); - let mut indexes = Vec::new(); - for (name, uuid) in metas { - // get index to retrieve primary key - let (index, _) = self - .get_index_txn(&txn, name)? - .with_context(|| format!("could not load index {:?}", name))?; - let primary_key = index.primary_key(&index.read_txn()?)?.map(String::from); - // retieve meta - let meta = self - .uuid_to_index_meta - .get(&txn, &uuid)? - .with_context(|| format!("could not retieve meta for index {:?}", name))?; - indexes.push((name.to_owned(), meta, primary_key)); - } - Ok(indexes) - } -} - -// Loops on an arc to get ownership on the wrapped value. This method sleeps 100ms before retrying. -fn get_arc_ownership_blocking(mut item: Arc) -> T { - loop { - match Arc::try_unwrap(item) { - Ok(item) => return item, - Err(item_arc) => { - item = item_arc; - std::thread::sleep(Duration::from_millis(100)); - continue; - } - } - } -} - -fn open_or_create_database( - env: &Env, - name: Option<&str>, -) -> anyhow::Result> { - match env.open_database::(name)? { - Some(db) => Ok(db), - None => Ok(env.create_database::(name)?), - } -} - -fn make_update_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { - let mut path = path.as_ref().to_path_buf(); - path.push(format!("update{}", uuid)); - path -} - -fn make_index_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { - let mut path = path.as_ref().to_path_buf(); - path.push(format!("index{}", uuid)); - path -} - -#[cfg(test)] -mod test { - use super::*; - use std::path::PathBuf; - - #[test] - fn test_make_update_db_path() { - let uuid = Uuid::new_v4(); - assert_eq!( - make_update_db_path("/home", &uuid), - PathBuf::from(format!("/home/update{}", uuid)) - ); - } - - #[test] - fn test_make_index_db_path() { - let uuid = Uuid::new_v4(); - assert_eq!( - make_index_db_path("/home", &uuid), - PathBuf::from(format!("/home/index{}", uuid)) - ); - } - - mod index_store { - use super::*; - - #[test] - fn test_index_uuid() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - - let name = "foobar"; - let txn = store.env.read_txn().unwrap(); - // name is not found if the uuid in not present in the db - assert!(store.index_uuid(&txn, &name).unwrap().is_none()); - drop(txn); - - // insert an uuid in the the name_to_uuid_db: - let uuid = Uuid::new_v4(); - let mut txn = store.env.write_txn().unwrap(); - store - .name_to_uuid - .put(&mut txn, &name, uuid.as_bytes()) - .unwrap(); - txn.commit().unwrap(); - - // check that the uuid is there - let txn = store.env.read_txn().unwrap(); - assert_eq!(store.index_uuid(&txn, &name).unwrap(), Some(uuid)); - } - - #[test] - fn test_retrieve_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let uuid = Uuid::new_v4(); - - let txn = store.env.read_txn().unwrap(); - assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - - let created_at = Utc::now(); - let updated_at = created_at; - - let meta = IndexMeta { - update_store_size: 4096 * 100, - index_store_size: 4096 * 100, - uuid: uuid.clone(), - created_at, - updated_at, - }; - let mut txn = store.env.write_txn().unwrap(); - store - .uuid_to_index_meta - .put(&mut txn, uuid.as_bytes(), &meta) - .unwrap(); - txn.commit().unwrap(); - - // the index cache should be empty - assert!(store.uuid_to_index.is_empty()); - - let txn = store.env.read_txn().unwrap(); - assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); - assert_eq!(store.uuid_to_index.len(), 1); - } - - #[test] - fn test_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - assert!(store.index(&name).unwrap().is_none()); - - let created_at = Utc::now(); - let updated_at = created_at; - - let uuid = Uuid::new_v4(); - let meta = IndexMeta { - update_store_size: 4096 * 100, - index_store_size: 4096 * 100, - uuid: uuid.clone(), - created_at, - updated_at, - }; - let mut txn = store.env.write_txn().unwrap(); - store - .name_to_uuid - .put(&mut txn, &name, uuid.as_bytes()) - .unwrap(); - store - .uuid_to_index_meta - .put(&mut txn, uuid.as_bytes(), &meta) - .unwrap(); - txn.commit().unwrap(); - - assert!(store.index(&name).unwrap().is_some()); - } - - #[test] - fn test_get_or_create_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - let update_store_size = 4096 * 100; - let index_store_size = 4096 * 100; - store - .get_or_create_index(&name, update_store_size, index_store_size) - .unwrap(); - let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); - assert_eq!(store.uuid_to_index.len(), 1); - assert!(uuid.is_some()); - let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store - .uuid_to_index_meta - .get(&txn, uuid.as_bytes()) - .unwrap() - .unwrap(); - assert_eq!(meta.update_store_size, update_store_size); - assert_eq!(meta.index_store_size, index_store_size); - assert_eq!(meta.uuid, uuid); - } - - #[test] - fn test_create_index() { - let temp = tempfile::tempdir().unwrap(); - let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); - let name = "foobar"; - - let update_store_size = 4096 * 100; - let index_store_size = 4096 * 100; - let uuid = Uuid::new_v4(); - let mut txn = store.env.write_txn().unwrap(); - store - .create_index_txn(&mut txn, uuid, name, update_store_size, index_store_size) - .unwrap(); - let uuid = store.name_to_uuid.get(&txn, &name).unwrap(); - assert_eq!(store.uuid_to_index.len(), 1); - assert!(uuid.is_some()); - let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = store - .uuid_to_index_meta - .get(&txn, uuid.as_bytes()) - .unwrap() - .unwrap(); - assert_eq!(meta.update_store_size, update_store_size); - assert_eq!(meta.index_store_size, index_store_size); - assert_eq!(meta.uuid, uuid); - } - } -} diff --git a/meilisearch-http/src/index_controller/local_index_controller/mod.rs b/meilisearch-http/src/index_controller/local_index_controller/mod.rs deleted file mode 100644 index 8ac600f5f..000000000 --- a/meilisearch-http/src/index_controller/local_index_controller/mod.rs +++ /dev/null @@ -1,228 +0,0 @@ -mod update_store; -mod index_store; -mod update_handler; - -use std::path::Path; -use std::sync::Arc; - -use anyhow::{bail, Context}; -use itertools::Itertools; -use milli::Index; - -use crate::option::IndexerOpts; -use index_store::IndexStore; -use super::IndexController; -use super::updates::UpdateStatus; -use super::{UpdateMeta, UpdateResult, IndexMetadata, IndexSettings}; - -pub struct LocalIndexController { - indexes: IndexStore, - update_db_size: u64, - index_db_size: u64, -} - -impl LocalIndexController { - pub fn new( - path: impl AsRef, - opt: IndexerOpts, - index_db_size: u64, - update_db_size: u64, - ) -> anyhow::Result { - let indexes = IndexStore::new(path, opt)?; - Ok(Self { indexes, index_db_size, update_db_size }) - } -} - -impl IndexController for LocalIndexController { - fn add_documents>( - &self, - index: S, - method: milli::update::IndexDocumentsMethod, - format: milli::update::UpdateFormat, - data: &[u8], - primary_key: Option, - ) -> anyhow::Result> { - let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; - let pending = update_store.register_update(meta, data)?; - Ok(pending.into()) - } - - fn update_settings>( - &self, - index: S, - settings: super::Settings - ) -> anyhow::Result> { - let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; - let meta = UpdateMeta::Settings(settings); - let pending = update_store.register_update(meta, &[])?; - Ok(pending.into()) - } - - fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let index_name = index_settings.name.context("Missing name for index")?; - let (index, _, meta) = self.indexes.create_index(&index_name, self.update_db_size, self.index_db_size)?; - if let Some(ref primary_key) = index_settings.primary_key { - if let Err(e) = update_primary_key(index, primary_key).context("error creating index") { - // TODO: creating index could not be completed, delete everything. - Err(e)? - } - } - - let meta = IndexMetadata { - uid: index_name, - uuid: meta.uuid.clone(), - created_at: meta.created_at, - updated_at: meta.created_at, - primary_key: index_settings.primary_key, - }; - - Ok(meta) - } - - fn delete_index>(&self, index_uid: S) -> anyhow::Result<()> { - self.indexes.delete(index_uid) - } - - fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { - todo!() - } - - fn index(&self, name: impl AsRef) -> anyhow::Result>> { - let index = self.indexes.index(name)?.map(|(i, _)| i); - Ok(index) - } - - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>> { - match self.indexes.index(&index)? { - Some((_, update_store)) => Ok(update_store.meta(id)?), - None => bail!("index {:?} doesn't exist", index.as_ref()), - } - } - - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { - match self.indexes.index(&index)? { - Some((_, update_store)) => { - let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { - Ok(processing - .map(UpdateStatus::from) - .into_iter() - .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) - .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .chain(failed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) - .sorted_by(|a, b| a.id().cmp(&b.id())) - .collect()) - })?; - Ok(updates) - } - None => bail!("index {} doesn't exist.", index.as_ref()), - } - - } - - fn list_indexes(&self) -> anyhow::Result> { - let metas = self.indexes.list_indexes()?; - let mut output_meta = Vec::new(); - for (uid, meta, primary_key) in metas { - let created_at = meta.created_at; - let uuid = meta.uuid; - let updated_at = self - .all_update_status(&uid)? - .iter() - .filter_map(|u| u.processed().map(|u| u.processed_at)) - .max() - .unwrap_or(created_at); - - let index_meta = IndexMetadata { - uid, - created_at, - updated_at, - uuid, - primary_key, - }; - output_meta.push(index_meta); - } - Ok(output_meta) - } - - fn update_index(&self, uid: impl AsRef, index_settings: IndexSettings) -> anyhow::Result { - if index_settings.name.is_some() { - bail!("can't update an index name.") - } - - let (primary_key, meta) = match index_settings.primary_key { - Some(ref primary_key) => { - self.indexes - .update_index(&uid, |index| { - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - bail!("primary key already exists.") - } - index.put_primary_key(&mut txn, primary_key)?; - txn.commit()?; - Ok(Some(primary_key.clone())) - })? - }, - None => { - let (index, meta) = self.indexes - .index_with_meta(&uid)? - .with_context(|| format!("index {:?} doesn't exist.", uid.as_ref()))?; - let primary_key = index - .primary_key(&index.read_txn()?)? - .map(String::from); - (primary_key, meta) - }, - }; - - Ok(IndexMetadata { - uid: uid.as_ref().to_string(), - uuid: meta.uuid.clone(), - created_at: meta.created_at, - updated_at: meta.updated_at, - primary_key, - }) - } - - fn clear_documents(&self, index: impl AsRef) -> anyhow::Result { - let (_, update_store) = self.indexes.index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let meta = UpdateMeta::ClearDocuments; - let pending = update_store.register_update(meta, &[])?; - Ok(pending.into()) - } - - fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result { - let (_, update_store) = self.indexes.index(&index)? - .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; - let meta = UpdateMeta::DeleteDocuments; - let content = serde_json::to_vec(&document_ids)?; - let pending = update_store.register_update(meta, &content)?; - Ok(pending.into()) - } -} - -fn update_primary_key(index: impl AsRef, primary_key: impl AsRef) -> anyhow::Result<()> { - let index = index.as_ref(); - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - bail!("primary key already set.") - } - index.put_primary_key(&mut txn, primary_key.as_ref())?; - txn.commit()?; - Ok(()) -} - -#[cfg(test)] -mod test { - use super::*; - use tempfile::tempdir; - use crate::make_index_controller_tests; - - make_index_controller_tests!({ - let options = IndexerOpts::default(); - let path = tempdir().unwrap(); - let size = 4096 * 100; - LocalIndexController::new(path, options, size, size).unwrap() - }); -} diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 63724af18..37b06f46f 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -28,7 +28,6 @@ async fn add_documents_no_index_creation() { let (response, code) = index.get_update(0).await; assert_eq!(code, 200); - println!("response: {}", response); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 718e35899..0bf2f15a5 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,6 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - println!("response: {}", response); assert_eq!(code, 200); assert_eq!(response["uid"], "test"); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index b15cc306e..5bc78950f 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,13 +6,11 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - println!("response: {}", _response); assert_eq!(code, 200); let (_response, code) = index.delete().await; - println!("response: {}", _response); assert_eq!(code, 200); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index b8217c5f7..feefe830f 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -36,7 +36,6 @@ async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; - println!("response: {}", response); index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); @@ -44,7 +43,6 @@ async fn test_partial_update() { assert_eq!(response["searchableAttributes"],json!(["*"])); let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; - println!("resp: {}", response); index.wait_update_id(1).await; let (response, code) = index.settings().await; @@ -96,7 +94,6 @@ async fn update_setting_unexisting_index_invalid_uid() { let server = Server::new().await; let index = server.index("test##! "); let (_response, code) = index.update_settings(json!({})).await; - println!("response: {}", _response); assert_eq!(code, 400); } diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 713936b8c..64b5b560e 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -46,7 +46,6 @@ async fn list_no_updates() { let index = server.index("test"); index.create(None).await; let (response, code) = index.list_updates().await; - println!("response: {}", response); assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } From 49b74b587a774de5dd21891864198e2d9d5a69af Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Mar 2021 17:44:39 +0100 Subject: [PATCH 57/69] enable jemalloc only on linux --- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/index_controller/mod.rs | 16 +++------------- .../src/index_controller/update_actor.rs | 2 -- meilisearch-http/src/lib.rs | 5 ----- 4 files changed, 4 insertions(+), 21 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7b09b3042..e88556f0a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -81,5 +81,5 @@ urlencoding = "1.1.1" [features] default = ["sentry"] -[target.'cfg(unix)'.dependencies] +[target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 1433562ff..5dc3374a0 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -14,9 +14,8 @@ use actix_web::web::{Bytes, Payload}; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::{Serialize, Deserialize}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use tokio::time::sleep; -use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; use crate::index::{SearchResult, SearchQuery, Document}; @@ -59,15 +58,6 @@ pub struct IndexController { update_handle: update_actor::UpdateActorHandle, } -enum IndexControllerMsg { - CreateIndex { - uuid: Uuid, - primary_key: Option, - ret: oneshot::Sender>, - }, - Shutdown, -} - impl IndexController { pub fn new(path: impl AsRef) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; @@ -94,10 +84,10 @@ impl IndexController { tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { match bytes { - Ok(bytes) => { sender.send(Ok(bytes)).await; }, + Ok(bytes) => { let _ = sender.send(Ok(bytes)).await; }, Err(e) => { let error: Box = Box::new(e); - sender.send(Err(error)).await; }, + let _ = sender.send(Err(error)).await; }, } } }); diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2c16eb60b..0eb473065 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -144,8 +144,6 @@ where .await .map_err(|e| UpdateError::Error(Box::new(e)))?; - let file = file.into_std().await; - tokio::task::spawn_blocking(move || { let result = update_store .register_update(meta, path, uuid) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index bd7379d7c..da1aee746 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,8 +1,3 @@ -#![allow(clippy::or_fun_call)] -#![allow(unused_must_use)] -#![allow(unused_variables)] -#![allow(dead_code)] - pub mod data; pub mod error; pub mod helpers; From 99c89cf2ba9bf96d53708442552af63a1c8ca520 Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 13 Mar 2021 10:09:10 +0100 Subject: [PATCH 58/69] use options max db sizes --- meilisearch-http/src/data/mod.rs | 4 +++- .../src/index_controller/index_actor.rs | 17 ++++++++++------- meilisearch-http/src/index_controller/mod.rs | 6 +++--- .../src/index_controller/update_actor.rs | 14 +++++++++----- meilisearch-http/tests/settings/get_settings.rs | 4 ++-- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index f2b963eec..176422efd 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -60,7 +60,9 @@ impl Data { let path = options.db_path.clone(); create_dir_all(&path)?; - let index_controller = IndexController::new(&path)?; + let index_size = options.max_mdb_size.get_bytes() as usize; + let update_store_size = options.max_udb_size.get_bytes() as usize; + let index_controller = IndexController::new(&path, index_size, update_store_size)?; let mut api_keys = ApiKeys { master: options.clone().master_key, diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 52a69b964..ee7a9e782 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -401,11 +401,11 @@ pub struct IndexActorHandle { } impl IndexActorHandle { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = HeedIndexStore::new(path)?; + let store = HeedIndexStore::new(path, index_size)?; let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -416,8 +416,7 @@ impl IndexActorHandle { pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { - ret, + let msg = IndexMsg::CreateIndex { ret, uuid, primary_key, }; @@ -515,15 +514,17 @@ impl IndexActorHandle { struct HeedIndexStore { index_store: AsyncMap, path: PathBuf, + index_size: usize, } impl HeedIndexStore { - fn new(path: impl AsRef) -> anyhow::Result { + fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { let path = path.as_ref().join("indexes/"); let index_store = Arc::new(RwLock::new(HashMap::new())); Ok(Self { index_store, path, + index_size, }) } } @@ -536,8 +537,9 @@ impl IndexStore for HeedIndexStore { return Err(IndexError::IndexAlreadyExists); } + let index_size = self.index_size; let index = spawn_blocking(move || -> Result { - let index = open_index(&path, 4096 * 100_000)?; + let index = open_index(&path, index_size)?; if let Some(primary_key) = primary_key { let mut txn = index.write_txn()?; index.put_primary_key(&mut txn, &primary_key)?; @@ -565,7 +567,8 @@ impl IndexStore for HeedIndexStore { return Ok(None); } - let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) + let index_size = self.index_size; + let index = spawn_blocking(move || open_index(path, index_size)) .await .map_err(|e| IndexError::Error(e.into()))??; self.index_store diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index 5dc3374a0..d82b304a2 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -59,10 +59,10 @@ pub struct IndexController { } impl IndexController { - pub fn new(path: impl AsRef) -> anyhow::Result { + pub fn new(path: impl AsRef, index_size: usize, update_store_size: usize) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; - let index_actor = index_actor::IndexActorHandle::new(&path)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path)?; + let index_actor = index_actor::IndexActorHandle::new(&path, index_size)?; + let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 0eb473065..2f66e8d20 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -211,10 +211,10 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef) -> anyhow::Result { + pub fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); - let store = MapUpdateStoreStore::new(index_handle, &path); + let store = MapUpdateStoreStore::new(index_handle, &path, update_store_size); let actor = UpdateActor::new(store, receiver, path)?; tokio::task::spawn(actor.run()); @@ -272,16 +272,18 @@ struct MapUpdateStoreStore { db: Arc>>>, index_handle: IndexActorHandle, path: PathBuf, + update_store_size: usize, } impl MapUpdateStoreStore { - fn new(index_handle: IndexActorHandle, path: impl AsRef) -> Self { + fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); Self { db, index_handle, path, + update_store_size, } } } @@ -292,7 +294,8 @@ impl UpdateStoreStore for MapUpdateStoreStore { match self.db.write().await.entry(uuid) { Entry::Vacant(e) => { let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); let path = self.path.clone().join(format!("updates-{}", e.key())); create_dir_all(&path).unwrap(); let index_handle = self.index_handle.clone(); @@ -324,7 +327,8 @@ impl UpdateStoreStore for MapUpdateStoreStore { // We can safely load the index let index_handle = self.index_handle.clone(); let mut options = heed::EnvOpenOptions::new(); - options.map_size(4096 * 100_000); + let update_store_size = self.update_store_size; + options.map_size(update_store_size); let store = UpdateStore::open(options, &path, move |meta, file| { futures::executor::block_on(index_handle.update(meta, file)) }) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index feefe830f..e00840f9b 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -35,14 +35,14 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - let (response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (_response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); assert_eq!(response["displayedAttributes"],json!(["foo"])); assert_eq!(response["searchableAttributes"],json!(["*"])); - let (response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (_response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; index.wait_update_id(1).await; let (response, code) = index.settings().await; From adc71a70ce3e93a399fab47aff8f273f49063c86 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 10:17:41 +0100 Subject: [PATCH 59/69] fix displayed attributes in document retrieval --- Cargo.lock | 78 ++++++++++++++----------------- meilisearch-http/src/index/mod.rs | 65 ++++++++++++++------------ 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3620a6885..ba4e8580e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,7 +160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" dependencies = [ "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -365,7 +365,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -464,7 +464,7 @@ checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -475,7 +475,7 @@ checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -655,9 +655,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "4.0.9" +version = "4.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8758c32833faaae35b24a73d332e62d0528e89076ae841c63940e37008b153" +checksum = "b9520900471c3a9bbcfe0fd4c7b6bcfeff41b20a76cf91c59b7474b09be1ee27" dependencies = [ "utf8-width", ] @@ -927,7 +927,7 @@ checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -984,7 +984,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -1031,7 +1031,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "synstructure", ] @@ -1172,7 +1172,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2198,7 +2198,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2276,7 +2276,7 @@ checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2287,7 +2287,7 @@ checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -2329,7 +2329,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "version_check", ] @@ -2571,14 +2571,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -2592,9 +2591,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" [[package]] name = "remove_dir_all" @@ -2845,7 +2844,7 @@ checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3035,7 +3034,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3051,7 +3050,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3087,7 +3086,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3103,9 +3102,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" +checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", @@ -3129,7 +3128,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "unicode-xid 0.2.1", ] @@ -3203,16 +3202,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", -] - -[[package]] -name = "thread_local" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" -dependencies = [ - "once_cell", + "syn 1.0.64", ] [[package]] @@ -3270,7 +3260,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", "standback", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3348,7 +3338,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", ] [[package]] @@ -3486,9 +3476,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "ucd-trie" @@ -3665,7 +3655,7 @@ dependencies = [ "log", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "wasm-bindgen-shared", ] @@ -3699,7 +3689,7 @@ checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.63", + "syn 1.0.64", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3860,7 +3850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ "proc-macro2 1.0.24", - "syn 1.0.63", + "syn 1.0.64", "synstructure", ] diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 1afeb3478..991746db3 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,15 +1,15 @@ mod search; mod updates; -use std::sync::Arc; use std::ops::Deref; +use std::sync::Arc; use anyhow::{bail, Context}; -use serde_json::{Value, Map}; use milli::obkv_to_json; +use serde_json::{Map, Value}; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; -pub use updates::{Settings, Facets, UpdateResult}; +pub use updates::{Facets, Settings, UpdateResult}; pub type Document = Map; @@ -76,7 +76,10 @@ impl Index { .iter() .filter_map(|f| fields_ids_map.id(f.as_ref())) .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), + None => match self.displayed_fields_ids(&txn)? { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }, }; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -97,36 +100,38 @@ impl Index { doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - let txn = self.read_txn()?; + let txn = self.read_txn()?; - let fields_ids_map = self.fields_ids_map(&txn)?; + let fields_ids_map = self.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => match self.displayed_fields_ids(&txn)? { + Some(fields) => fields, None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + }, + }; + let internal_id = self + .external_documents_ids(&txn)? + .get(doc_id.as_bytes()) + .with_context(|| format!("Document with id {} not found", doc_id))?; - let internal_id = self - .external_documents_ids(&txn)? - .get(doc_id.as_bytes()) - .with_context(|| format!("Document with id {} not found", doc_id))?; + let document = self + .documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d); - let document = self - .documents(&txn, std::iter::once(internal_id))? - .into_iter() - .next() - .map(|(_, d)| d); - - match document { - Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, - &fields_ids_map, - document, - )?), - None => bail!("Document with id {} not found", doc_id), - } + match document { + Some(document) => Ok(obkv_to_json( + &attributes_to_retrieve_ids, + &fields_ids_map, + document, + )?), + None => bail!("Document with id {} not found", doc_id), + } } } From 77c0a0fba5a4e3c917719c0f0b3fd24cf2aacf1d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 10:36:12 +0100 Subject: [PATCH 60/69] add test get document displayed attributes --- meilisearch-http/src/index/mod.rs | 1 + .../tests/documents/get_documents.rs | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 991746db3..fd5aa3004 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -114,6 +114,7 @@ impl Index { None => fields_ids_map.iter().map(|(id, _)| id).collect(), }, }; + let internal_id = self .external_documents_ids(&txn)? .get(doc_id.as_bytes()) diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 5affb8a7a..c9ed83316 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,6 +1,8 @@ use crate::common::Server; use crate::common::GetAllDocumentsOptions; +use serde_json::json; + // TODO: partial test since we are testing error, amd error is not yet fully implemented in // transplant #[actix_rt::test] @@ -147,3 +149,22 @@ async fn test_get_all_documents_attributes_to_retrieve() { assert_eq!(response.as_array().unwrap().len(), 20); assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 2); } + +#[actix_rt::test] +async fn get_documents_displayed_attributes() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings(json!({"displayedAttributes": ["gender"]})).await; + index.load_test_set().await; + + let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); + assert!(response.as_array().unwrap()[0].as_object().unwrap().get("gender").is_some()); + + let (response, code) = index.get_document(0, None).await; + assert_eq!(code, 200); + assert_eq!(response.as_object().unwrap().keys().count(), 1); + assert!(response.as_object().unwrap().get("gender").is_some()); +} From c079f60346adf23a11d02bde7c1f2faf606310cf Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 11:01:14 +0100 Subject: [PATCH 61/69] fixup! fix displayed attributes in document retrieval --- meilisearch-http/src/index/mod.rs | 51 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index fd5aa3004..06f1140a3 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -1,6 +1,7 @@ mod search; mod updates; +use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; @@ -70,17 +71,7 @@ impl Index { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => match self.displayed_fields_ids(&txn)? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }, - }; + let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -88,7 +79,7 @@ impl Index { for entry in iter { let (_id, obkv) = entry?; - let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; + let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; documents.push(object); } @@ -104,16 +95,7 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => match self.displayed_fields_ids(&txn)? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }, - }; + let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -128,11 +110,34 @@ impl Index { match document { Some(document) => Ok(obkv_to_json( - &attributes_to_retrieve_ids, + &fields_to_display, &fields_ids_map, document, )?), None => bail!("Document with id {} not found", doc_id), } } + + fn fields_to_display>( + &self, + txn: &heed::RoTxn, + attributes_to_retrieve: Option>, + fields_ids_map: &milli::FieldsIdsMap, + ) -> anyhow::Result> { + let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + Some(ids) => ids.into_iter().collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); + Ok(displayed_fields_ids) + } } From fcf1d4e922030b1fc64331933922aa7eaf075cb8 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 12:20:33 +0100 Subject: [PATCH 62/69] fix displayed attributes in search --- meilisearch-http/src/index/search.rs | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index fb830ec60..aa2fe0715 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -78,29 +78,13 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let displayed_fields_ids = self.displayed_fields_ids(&rtxn).unwrap(); - - let attributes_to_retrieve_ids = match query.attributes_to_retrieve { - Some(ref attrs) if attrs.iter().any(|f| f == "*") => None, - Some(ref attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f)) - .collect::>() - .into(), - None => None, - }; - - let displayed_fields_ids = match (displayed_fields_ids, attributes_to_retrieve_ids) { - (_, Some(ids)) => ids, - (Some(ids), None) => ids, - (None, None) => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; + let fields_to_display = self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); + let mut object = milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } From 55fadd7f87ab6953f3f573d59c730d4966a39449 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 13:53:50 +0100 Subject: [PATCH 63/69] change facetedAttributes to attributesForFaceting --- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/updates.rs | 6 +++--- meilisearch-http/src/routes/settings/mod.rs | 4 ++-- meilisearch-http/tests/settings/get_settings.rs | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 06f1140a3..72453091f 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -54,7 +54,7 @@ impl Index { Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(Some(faceted_attributes)), + attributes_for_faceting: Some(Some(faceted_attributes)), ranking_rules: Some(Some(criteria)), }) } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 70c2dfc2b..254f6e991 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -35,7 +35,7 @@ pub struct Settings { pub searchable_attributes: Option>>, #[serde(default)] - pub faceted_attributes: Option>>, + pub attributes_for_faceting: Option>>, #[serde( default, @@ -50,7 +50,7 @@ impl Settings { Self { displayed_attributes: Some(None), searchable_attributes: Some(None), - faceted_attributes: Some(None), + attributes_for_faceting: Some(None), ranking_rules: Some(None), } } @@ -158,7 +158,7 @@ impl Index { } // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { + if let Some(ref facet_types) = settings.attributes_for_faceting { let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); builder.set_faceted_fields(facet_types); } diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 4e7219b1c..345a86534 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -82,7 +82,7 @@ macro_rules! make_setting_route { make_setting_route!( "/indexes/{index_uid}/settings/attributes-for-faceting", std::collections::HashMap, - faceted_attributes + attributes_for_faceting ); make_setting_route!( @@ -126,7 +126,7 @@ macro_rules! create_services { } create_services!( - faceted_attributes, + attributes_for_faceting, displayed_attributes, searchable_attributes ); diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index e00840f9b..0a8972108 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -19,7 +19,8 @@ async fn get_settings() { assert_eq!(settings.keys().len(), 4); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); - assert_eq!(settings["facetedAttributes"], json!({})); + println!("{:?}", settings); + assert_eq!(settings["attributesForFaceting"], json!({})); assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); } From f727dcc8c6d8ebcd7d0f7a4becc26ec3b1fe1c3e Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:26:59 +0100 Subject: [PATCH 64/69] update milli --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba4e8580e..d40beeb70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1889,6 +1889,7 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" +source = "git+https://github.com/meilisearch/milli.git?rev=b7b23cd#b7b23cd4a8e62932c66c2ebedf9d89ddf089e299" dependencies = [ "anyhow", "bstr", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e88556f0a..5fb5d0ebf 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -17,7 +17,6 @@ actix-cors = { path = "../../actix-extras/actix-cors" } actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } -#actix-web = { version = "3", features = ["rustls"] } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } @@ -38,8 +37,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { path = "../../milli/milli" } -#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { git = "https://github.com/meilisearch/milli.git", rev = "b7b23cd" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" From 0c80d891c0b953a3154de77567dea7c33f453945 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:29:30 +0100 Subject: [PATCH 65/69] clean Cargo.toml --- meilisearch-http/Cargo.toml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5fb5d0ebf..9e148c46b 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -19,10 +19,14 @@ actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } anyhow = "1.0.36" async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] } +async-stream = "0.3.0" +async-trait = "0.1.42" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" +dashmap = "4.0.2" +either = "1.6.1" env_logger = "0.8.2" flate2 = "1.0.19" fst = "0.4.5" @@ -32,6 +36,7 @@ grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } +itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } @@ -40,6 +45,7 @@ memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", rev = "b7b23cd" } mime = "0.3.16" once_cell = "1.5.2" +parking_lot = "0.11.1" rand = "0.7.3" rayon = "1.5.0" regex = "1.4.2" @@ -52,19 +58,22 @@ slice-group-by = "0.2.6" structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" -tokio = { version = "1", features = ["full"] } -dashmap = "4.0.2" -uuid = "0.8.2" -itertools = "0.10.0" -either = "1.6.1" -async-trait = "0.1.42" thiserror = "1.0.24" -async-stream = "0.3.0" -parking_lot = "0.11.1" +tokio = { version = "1", features = ["full"] } +uuid = "0.8.2" [dependencies.sentry] default-features = false -features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"] +features = [ + "with_client_implementation", + "with_panic", + "with_failure", + "with_device_info", + "with_rust_info", + "with_reqwest_transport", + "with_rustls", + "with_env_logger" +] optional = true version = "0.18.1" From 01479dcf996d5cdcee30cbbf8d5bf912c2640cab Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 14:43:47 +0100 Subject: [PATCH 66/69] rename name to uid in code --- meilisearch-http/src/data/mod.rs | 12 ++-- meilisearch-http/src/data/updates.rs | 8 +-- .../src/index_controller/uuid_resolver.rs | 64 +++++++++---------- meilisearch-http/src/routes/index.rs | 4 +- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 176422efd..5d85650d0 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -78,21 +78,21 @@ impl Data { Ok(Data { inner }) } - pub async fn settings>(&self, index_uid: S) -> anyhow::Result { - self.index_controller.settings(index_uid.as_ref().to_string()).await + pub async fn settings>(&self, uid: S) -> anyhow::Result { + self.index_controller.settings(uid.as_ref().to_string()).await } pub async fn list_indexes(&self) -> anyhow::Result> { self.index_controller.list_indexes().await } - pub async fn index(&self, name: impl AsRef) -> anyhow::Result> { - self.index_controller.get_index(name.as_ref().to_string()).await + pub async fn index(&self, uid: impl AsRef) -> anyhow::Result> { + self.index_controller.get_index(uid.as_ref().to_string()).await } - pub async fn create_index(&self, name: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, uid: impl AsRef, primary_key: Option>) -> anyhow::Result { let settings = IndexSettings { - uid: Some(name.as_ref().to_string()), + uid: Some(uid.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index 0e2c56e42..ef9471fb5 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -65,15 +65,15 @@ impl Data { pub async fn update_index( &self, - name: impl AsRef, + uid: impl AsRef, primary_key: Option>, - new_name: Option> + new_uid: Option> ) -> anyhow::Result { let settings = IndexSettings { - uid: new_name.map(|s| s.as_ref().to_string()), + uid: new_uid.map(|s| s.as_ref().to_string()), primary_key: primary_key.map(|s| s.as_ref().to_string()), }; - self.index_controller.update_index(name.as_ref().to_string(), settings).await + self.index_controller.update_index(uid.as_ref().to_string(), settings).await } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index fcf417248..b113de37a 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -11,19 +11,19 @@ pub type Result = std::result::Result; #[derive(Debug)] enum UuidResolveMsg { Resolve { - name: String, + uid: String, ret: oneshot::Sender>, }, GetOrCreate { - name: String, + uid: String, ret: oneshot::Sender>, }, Create { - name: String, + uid: String, ret: oneshot::Sender>, }, Delete { - name: String, + uid: String, ret: oneshot::Sender>, }, List { @@ -48,16 +48,16 @@ impl UuidResolverActor { loop { match self.inbox.recv().await { - Some(Create { name, ret }) => { + Some(Create { uid: name, ret }) => { let _ = ret.send(self.handle_create(name).await); } - Some(GetOrCreate { name, ret }) => { + Some(GetOrCreate { uid: name, ret }) => { let _ = ret.send(self.handle_get_or_create(name).await); } - Some(Resolve { name, ret }) => { + Some(Resolve { uid: name, ret }) => { let _ = ret.send(self.handle_resolve(name).await); } - Some(Delete { name, ret }) => { + Some(Delete { uid: name, ret }) => { let _ = ret.send(self.handle_delete(name).await); } Some(List { ret }) => { @@ -71,32 +71,32 @@ impl UuidResolverActor { warn!("exiting uuid resolver loop"); } - async fn handle_create(&self, name: String) -> Result { - if !is_index_uid_valid(&name) { - return Err(UuidError::BadlyFormatted(name)) + async fn handle_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)) } - self.store.create_uuid(name, true).await + self.store.create_uuid(uid, true).await } - async fn handle_get_or_create(&self, name: String) -> Result { - if !is_index_uid_valid(&name) { - return Err(UuidError::BadlyFormatted(name)) + async fn handle_get_or_create(&self, uid: String) -> Result { + if !is_index_uid_valid(&uid) { + return Err(UuidError::BadlyFormatted(uid)) } - self.store.create_uuid(name, false).await + self.store.create_uuid(uid, false).await } - async fn handle_resolve(&self, name: String) -> Result { + async fn handle_resolve(&self, uid: String) -> Result { self.store - .get_uuid(name.clone()) + .get_uuid(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(name)) + .ok_or(UuidError::UnexistingIndex(uid)) } - async fn handle_delete(&self, name: String) -> Result { + async fn handle_delete(&self, uid: String) -> Result { self.store - .delete(name.clone()) + .delete(uid.clone()) .await? - .ok_or(UuidError::UnexistingIndex(name)) + .ok_or(UuidError::UnexistingIndex(uid)) } async fn handle_list(&self) -> Result> { @@ -125,7 +125,7 @@ impl UuidResolverHandle { pub async fn resolve(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Resolve { name, ret }; + let msg = UuidResolveMsg::Resolve { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -134,7 +134,7 @@ impl UuidResolverHandle { pub async fn get_or_create(&self, name: String) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::GetOrCreate { name, ret }; + let msg = UuidResolveMsg::GetOrCreate { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -143,7 +143,7 @@ impl UuidResolverHandle { pub async fn create(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Create { name, ret }; + let msg = UuidResolveMsg::Create { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -152,7 +152,7 @@ impl UuidResolverHandle { pub async fn delete(&self, name: String) -> anyhow::Result { let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::Delete { name, ret }; + let msg = UuidResolveMsg::Delete { uid: name, ret }; let _ = self.sender.send(msg).await; Ok(receiver .await @@ -189,9 +189,9 @@ pub enum UuidError { trait UuidStore { // Create a new entry for `name`. Return an error if `err` and the entry already exists, return // the uuid otherwise. - async fn create_uuid(&self, name: String, err: bool) -> Result; - async fn get_uuid(&self, name: String) -> Result>; - async fn delete(&self, name: String) -> Result>; + async fn create_uuid(&self, uid: String, err: bool) -> Result; + async fn get_uuid(&self, uid: String) -> Result>; + async fn delete(&self, uid: String) -> Result>; async fn list(&self) -> Result>; } @@ -253,15 +253,15 @@ impl UuidStore for HeedUuidStore { }).await? } - async fn delete(&self, name: String) -> Result> { + async fn delete(&self, uid: String) -> Result> { let env = self.env.clone(); let db = self.db.clone(); tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; - match db.get(&txn, &name)? { + match db.get(&txn, &uid)? { Some(uuid) => { let uuid = Uuid::from_slice(uuid)?; - db.delete(&mut txn, &name)?; + db.delete(&mut txn, &uid)?; txn.commit()?; Ok(Some(uuid)) } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 68c94798a..1cdfa9880 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -75,7 +75,7 @@ async fn create_index( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { - name: Option, + uid: Option, primary_key: Option, } @@ -95,7 +95,7 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.name.as_ref()).await { + match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.uid.as_ref()).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) From abbea5973207e38ae698e347033ebba6bedf4896 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 16:52:05 +0100 Subject: [PATCH 67/69] fix clippy warnings --- meilisearch-http/src/data/mod.rs | 14 +++---- meilisearch-http/src/data/search.rs | 12 +++--- meilisearch-http/src/data/updates.rs | 40 +++++++++---------- meilisearch-http/src/index/search.rs | 2 +- meilisearch-http/src/index/updates.rs | 28 +++++-------- .../src/index_controller/index_actor.rs | 38 +++++++++--------- meilisearch-http/src/index_controller/mod.rs | 23 +++++------ .../src/index_controller/update_actor.rs | 38 +++++++++--------- .../src/index_controller/update_store.rs | 4 +- .../src/index_controller/uuid_resolver.rs | 8 ++-- meilisearch-http/src/routes/document.rs | 2 +- meilisearch-http/src/routes/index.rs | 24 +++++------ meilisearch-http/src/routes/search.rs | 12 +++--- meilisearch-http/src/routes/settings/mod.rs | 4 +- meilisearch-http/tests/index/create_index.rs | 2 +- meilisearch-http/tests/index/delete_index.rs | 2 +- meilisearch-http/tests/index/get_index.rs | 2 +- 17 files changed, 124 insertions(+), 131 deletions(-) diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 5d85650d0..e0ae77ffb 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -78,22 +78,22 @@ impl Data { Ok(Data { inner }) } - pub async fn settings>(&self, uid: S) -> anyhow::Result { - self.index_controller.settings(uid.as_ref().to_string()).await + pub async fn settings(&self, uid: String) -> anyhow::Result { + self.index_controller.settings(uid).await } pub async fn list_indexes(&self) -> anyhow::Result> { self.index_controller.list_indexes().await } - pub async fn index(&self, uid: impl AsRef) -> anyhow::Result> { - self.index_controller.get_index(uid.as_ref().to_string()).await + pub async fn index(&self, uid: String) -> anyhow::Result { + self.index_controller.get_index(uid).await } - pub async fn create_index(&self, uid: impl AsRef, primary_key: Option>) -> anyhow::Result { + pub async fn create_index(&self, uid: String, primary_key: Option) -> anyhow::Result { let settings = IndexSettings { - uid: Some(uid.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), + uid: Some(uid), + primary_key, }; let meta = self.index_controller.create_index(settings).await?; diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 753083b7e..8838a11e0 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -4,12 +4,12 @@ use crate::index::{SearchQuery, SearchResult}; use super::Data; impl Data { - pub async fn search>( + pub async fn search( &self, - index: S, + index: String, search_query: SearchQuery, ) -> anyhow::Result { - self.index_controller.search(index.as_ref().to_string(), search_query).await + self.index_controller.search(index, search_query).await } pub async fn retrieve_documents( @@ -24,11 +24,11 @@ impl Data { pub async fn retrieve_document( &self, - index: impl AsRef + Sync + Send + 'static, - document_id: impl AsRef + Sync + Send + 'static, + index: String, + document_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - self.index_controller.document(index.as_ref().to_string(), document_id.as_ref().to_string(), attributes_to_retrieve).await + self.index_controller.document(index, document_id, attributes_to_retrieve).await } } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index ef9471fb5..d5a246223 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -9,14 +9,14 @@ use super::Data; impl Data { pub async fn add_documents( &self, - index: impl AsRef + Send + Sync + 'static, + index: String, method: IndexDocumentsMethod, format: UpdateFormat, stream: Payload, primary_key: Option, ) -> anyhow::Result { - let update_status = self.index_controller.add_documents(index.as_ref().to_string(), method, format, stream, primary_key).await?; + let update_status = self.index_controller.add_documents(index, method, format, stream, primary_key).await?; Ok(update_status) } @@ -27,53 +27,53 @@ impl Data { create: bool, ) -> anyhow::Result { let update = self.index_controller.update_settings(index, settings, create).await?; - Ok(update.into()) + Ok(update) } pub async fn clear_documents( &self, - index: impl AsRef + Sync + Send + 'static, + index: String, ) -> anyhow::Result { - let update = self.index_controller.clear_documents(index.as_ref().to_string()).await?; + let update = self.index_controller.clear_documents(index).await?; Ok(update) } pub async fn delete_documents( &self, - index: impl AsRef + Sync + Send + 'static, + index: String, document_ids: Vec, ) -> anyhow::Result { - let update = self.index_controller.delete_documents(index.as_ref().to_string(), document_ids).await?; - Ok(update.into()) + let update = self.index_controller.delete_documents(index, document_ids).await?; + Ok(update) } pub async fn delete_index( &self, - index: impl AsRef + Send + Sync + 'static, + index: String, ) -> anyhow::Result<()> { - self.index_controller.delete_index(index.as_ref().to_owned()).await?; + self.index_controller.delete_index(index).await?; Ok(()) } - pub async fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { - self.index_controller.update_status(index.as_ref().to_string(), uid).await + pub async fn get_update_status(&self, index: String, uid: u64) -> anyhow::Result { + self.index_controller.update_status(index, uid).await } - pub async fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { - self.index_controller.all_update_status(index.as_ref().to_string()).await + pub async fn get_updates_status(&self, index: String) -> anyhow::Result> { + self.index_controller.all_update_status(index).await } pub async fn update_index( &self, - uid: impl AsRef, - primary_key: Option>, - new_uid: Option> + uid: String, + primary_key: Option, + new_uid: Option ) -> anyhow::Result { let settings = IndexSettings { - uid: new_uid.map(|s| s.as_ref().to_string()), - primary_key: primary_key.map(|s| s.as_ref().to_string()), + uid: new_uid, + primary_key, }; - self.index_controller.update_index(uid.as_ref().to_string(), settings).await + self.index_controller.update_index(uid, settings).await } } diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index aa2fe0715..c7db9c313 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -120,7 +120,7 @@ impl Index { fn parse_facets_array( txn: &RoTxn, index: &Index, - arr: &Vec, + arr: &[Value], ) -> anyhow::Result> { let mut ands = Vec::new(); for value in arr { diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 254f6e991..b2d95c9ca 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -85,11 +85,8 @@ impl Index { let mut wtxn = self.write_txn()?; // Set the primary key if not set already, ignore if already set. - match (self.primary_key(&wtxn)?, primary_key) { - (None, Some(ref primary_key)) => { - self.put_primary_key(&mut wtxn, primary_key)?; - } - _ => (), + if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { + self.put_primary_key(&mut wtxn, primary_key)?; } let mut builder = update_builder.index_documents(&mut wtxn, self); @@ -109,13 +106,10 @@ impl Index { info!("document addition done: {:?}", result); - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()), - } + result.and_then(|addition_result| wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into)) } pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { @@ -128,7 +122,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -159,7 +153,7 @@ impl Index { // We transpose the settings JSON struct into a real setting update. if let Some(ref facet_types) = settings.attributes_for_faceting { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + let facet_types = facet_types.clone().unwrap_or_else(HashMap::new); builder.set_faceted_fields(facet_types); } @@ -179,7 +173,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -202,7 +196,7 @@ impl Index { .commit() .and(Ok(UpdateResult::Other)) .map_err(Into::into), - Err(e) => Err(e.into()), + Err(e) => Err(e), } } @@ -223,7 +217,7 @@ impl Index { .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e.into()) + Err(e) => Err(e) } } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index ee7a9e782..605dee22b 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -92,7 +92,7 @@ enum IndexMsg { }, GetMeta { uuid: Uuid, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, }, UpdateIndex { uuid: Uuid, @@ -137,7 +137,7 @@ impl IndexActor { ) -> Result { let options = IndexerOpts::default(); let update_handler = - UpdateHandler::new(&options).map_err(|e| IndexError::Error(e.into()))?; + UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -274,11 +274,11 @@ impl IndexActor { data: File, ) -> Result { debug!("Processing update {}", meta.id()); - let uuid = meta.index_uuid().clone(); + let uuid = meta.index_uuid(); let update_handler = self.update_handler.clone(); - let index = match self.store.get(uuid.clone()).await? { + let index = match self.store.get(*uuid).await? { Some(index) => index, - None => self.store.create(uuid, None).await?, + None => self.store.create(*uuid, None).await?, }; spawn_blocking(move || update_handler.handle_update(meta, data, index)) .await @@ -291,7 +291,7 @@ impl IndexActor { .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || index.settings().map_err(|e| IndexError::Error(e))) + spawn_blocking(move || index.settings().map_err(IndexError::Error)) .await .map_err(|e| IndexError::Error(e.into()))? } @@ -311,7 +311,7 @@ impl IndexActor { spawn_blocking(move || { index .retrieve_documents(offset, limit, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)) + .map_err(IndexError::Error) }) .await .map_err(|e| IndexError::Error(e.into()))? @@ -331,7 +331,7 @@ impl IndexActor { spawn_blocking(move || { index .retrieve_document(doc_id, attributes_to_retrieve) - .map_err(|e| IndexError::Error(e)) + .map_err(IndexError::Error) }) .await .map_err(|e| IndexError::Error(e.into()))? @@ -354,15 +354,15 @@ impl IndexActor { Ok(()) } - async fn handle_get_meta(&self, uuid: Uuid) -> Result> { + async fn handle_get_meta(&self, uuid: Uuid) -> Result { match self.store.get(uuid).await? { Some(index) => { let meta = spawn_blocking(move || IndexMeta::new(&index)) .await .map_err(|e| IndexError::Error(e.into()))??; - Ok(Some(meta)) + Ok(meta) } - None => Ok(None), + None => Err(IndexError::UnexistingIndex), } } @@ -405,7 +405,7 @@ impl IndexActorHandle { let (read_sender, read_receiver) = mpsc::channel(100); let (write_sender, write_receiver) = mpsc::channel(100); - let store = HeedIndexStore::new(path, index_size)?; + let store = HeedIndexStore::new(path, index_size); let actor = IndexActor::new(read_receiver, write_receiver, store)?; tokio::task::spawn(actor.run()); Ok(Self { @@ -492,7 +492,7 @@ impl IndexActorHandle { Ok(receiver.await.expect("IndexActor has been killed")?) } - pub async fn get_index_meta(&self, uuid: Uuid) -> Result> { + pub async fn get_index_meta(&self, uuid: Uuid) -> Result { let (ret, receiver) = oneshot::channel(); let msg = IndexMsg::GetMeta { uuid, ret }; let _ = self.read_sender.send(msg).await; @@ -518,14 +518,14 @@ struct HeedIndexStore { } impl HeedIndexStore { - fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + fn new(path: impl AsRef, index_size: usize) -> Self { let path = path.as_ref().join("indexes/"); let index_store = Arc::new(RwLock::new(HashMap::new())); - Ok(Self { + Self { index_store, path, index_size, - }) + } } } @@ -550,7 +550,7 @@ impl IndexStore for HeedIndexStore { .await .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid.clone(), index.clone()); + self.index_store.write().await.insert(uuid, index.clone()); Ok(index) } @@ -574,7 +574,7 @@ impl IndexStore for HeedIndexStore { self.index_store .write() .await - .insert(uuid.clone(), index.clone()); + .insert(uuid, index.clone()); Ok(Some(index)) } } @@ -595,6 +595,6 @@ fn open_index(path: impl AsRef, size: usize) -> Result { let mut options = EnvOpenOptions::new(); options.map_size(size); let index = milli::Index::new(options, &path) - .map_err(|e| IndexError::Error(e))?; + .map_err(IndexError::Error)?; Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index d82b304a2..ef8b16340 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -127,7 +127,7 @@ impl IndexController { // the update is processed. This would make calls to GET index to fail until the update // is complete. Since this is get or create, we ignore the error when the index already // exists. - match self.index_handle.create_index(uuid.clone(), None).await { + match self.index_handle.create_index(uuid, None).await { Ok(_) | Err(index_actor::IndexError::IndexAlreadyExists) => (), Err(e) => return Err(e.into()), } @@ -158,12 +158,12 @@ impl IndexController { let uuid = self.uuid_resolver .delete(uid) .await?; - self.update_handle.delete(uuid.clone()).await?; + self.update_handle.delete(uuid).await?; self.index_handle.delete(uuid).await?; Ok(()) } - pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result> { + pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { let uuid = self.uuid_resolver .resolve(uid) .await?; @@ -184,10 +184,9 @@ impl IndexController { let mut ret = Vec::new(); for (uid, uuid) in uuids { - if let Some(meta) = self.index_handle.get_index_meta(uuid).await? { - let meta = IndexMetadata { uid, meta }; - ret.push(meta); - } + let meta = self.index_handle.get_index_meta(uuid).await?; + let meta = IndexMetadata { uid, meta }; + ret.push(meta); } Ok(ret) @@ -247,13 +246,13 @@ impl IndexController { Ok(result) } - pub async fn get_index(&self, uid: String) -> anyhow::Result> { + pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; - let result = self.index_handle + let meta = self.index_handle .get_index_meta(uuid) - .await? - .map(|meta| IndexMetadata { uid, meta }); - Ok(result) + .await?; + let meta = IndexMetadata { uid, meta }; + Ok(meta) } } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2f66e8d20..2b9d05a4b 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -25,6 +25,8 @@ pub enum UpdateError { Error(Box), #[error("Index {0} doesn't exist.")] UnexistingIndex(Uuid), + #[error("Update {0} doesn't exist.")] + UnexistingUpdate(u64), } enum UpdateMsg { @@ -40,7 +42,7 @@ enum UpdateMsg { }, GetUpdate { uuid: Uuid, - ret: oneshot::Sender>>, + ret: oneshot::Sender>, id: u64, }, Delete { @@ -62,8 +64,8 @@ struct UpdateActor { #[async_trait::async_trait] trait UpdateStoreStore { async fn get_or_create(&self, uuid: Uuid) -> Result>; - async fn delete(&self, uuid: &Uuid) -> Result>>; - async fn get(&self, uuid: &Uuid) -> Result>>; + async fn delete(&self, uuid: Uuid) -> Result>>; + async fn get(&self, uuid: Uuid) -> Result>>; } impl UpdateActor @@ -145,18 +147,17 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))?; tokio::task::spawn_blocking(move || { - let result = update_store + update_store .register_update(meta, path, uuid) - .map(|pending| UpdateStatus::Pending(pending)) - .map_err(|e| UpdateError::Error(Box::new(e))); - result + .map(UpdateStatus::Pending) + .map_err(|e| UpdateError::Error(Box::new(e))) }) .await .map_err(|e| UpdateError::Error(Box::new(e)))? } async fn handle_list_updates(&self, uuid: Uuid) -> Result> { - let update_store = self.store.get(&uuid).await?; + let update_store = self.store.get(uuid).await?; tokio::task::spawn_blocking(move || { let result = update_store .ok_or(UpdateError::UnexistingIndex(uuid))? @@ -168,20 +169,21 @@ where .map_err(|e| UpdateError::Error(Box::new(e)))? } - async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result> { + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { let store = self .store - .get(&uuid) + .get(uuid) .await? .ok_or(UpdateError::UnexistingIndex(uuid))?; let result = store .meta(id) - .map_err(|e| UpdateError::Error(Box::new(e)))?; + .map_err(|e| UpdateError::Error(Box::new(e)))? + .ok_or(UpdateError::UnexistingUpdate(id))?; Ok(result) } async fn handle_delete(&self, uuid: Uuid) -> Result<()> { - let store = self.store.delete(&uuid).await?; + let store = self.store.delete(uuid).await?; if let Some(store) = store { tokio::task::spawn(async move { @@ -246,7 +248,7 @@ where receiver.await.expect("update actor killed.") } - pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result> { + pub async fn update_status(&self, uuid: Uuid, id: u64) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UpdateMsg::GetUpdate { uuid, id, ret }; let _ = self.sender.send(msg).await; @@ -310,9 +312,9 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } - async fn get(&self, uuid: &Uuid) -> Result>> { + async fn get(&self, uuid: Uuid) -> Result>> { let guard = self.db.read().await; - match guard.get(uuid) { + match guard.get(&uuid) { Some(uuid) => Ok(Some(uuid.clone())), None => { // The index is not found in the found in the loaded indexes, so we attempt to load @@ -322,7 +324,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { let path = self.path.clone().join(format!("updates-{}", uuid)); if path.exists() { let mut guard = self.db.write().await; - match guard.entry(uuid.clone()) { + match guard.entry(uuid) { Entry::Vacant(entry) => { // We can safely load the index let index_handle = self.index_handle.clone(); @@ -333,7 +335,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { futures::executor::block_on(index_handle.update(meta, file)) }) .map_err(|e| UpdateError::Error(e.into()))?; - let store = entry.insert(store.clone()); + let store = entry.insert(store); Ok(Some(store.clone())) } Entry::Occupied(entry) => { @@ -348,7 +350,7 @@ impl UpdateStoreStore for MapUpdateStoreStore { } } - async fn delete(&self, uuid: &Uuid) -> Result>> { + async fn delete(&self, uuid: Uuid) -> Result>> { let store = self.db.write().await.remove(&uuid); let path = self.path.clone().join(format!("updates-{}", uuid)); if store.is_some() || path.exists() { diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index a7a65e494..9ce2f242b 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -92,7 +92,7 @@ where let update_store_weak = Arc::downgrade(&update_store); tokio::task::spawn(async move { // Block and wait for something to process. - 'outer: while let Some(_) = notification_receiver.recv().await { + 'outer: while notification_receiver.recv().await.is_some() { loop { match update_store_weak.upgrade() { Some(update_store) => { @@ -276,7 +276,7 @@ where updates.extend(failed); - updates.sort_unstable_by(|a, b| a.id().cmp(&b.id())); + updates.sort_by_key(|u| u.id()); Ok(updates) } diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index b113de37a..69e9a626b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -216,7 +216,7 @@ impl HeedUuidStore { impl UuidStore for HeedUuidStore { async fn create_uuid(&self, name: String, err: bool) -> Result { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; match db.get(&txn, &name)? { @@ -240,7 +240,7 @@ impl UuidStore for HeedUuidStore { async fn get_uuid(&self, name: String) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let txn = env.read_txn()?; match db.get(&txn, &name)? { @@ -255,7 +255,7 @@ impl UuidStore for HeedUuidStore { async fn delete(&self, uid: String) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let mut txn = env.write_txn()?; match db.get(&txn, &uid)? { @@ -272,7 +272,7 @@ impl UuidStore for HeedUuidStore { async fn list(&self) -> Result> { let env = self.env.clone(); - let db = self.db.clone(); + let db = self.db; tokio::task::spawn_blocking(move || { let txn = env.read_txn()?; let mut entries = Vec::new(); diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 43d88e4b9..c53c3cb85 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -105,7 +105,7 @@ async fn get_all_documents( .attributes_to_retrieve .as_ref() .map(|attrs| attrs - .split(",") + .split(',') .map(String::from) .collect::>()); diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 1cdfa9880..cb1c08e24 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -37,13 +37,12 @@ async fn get_index( data: web::Data, path: web::Path, ) -> Result { - match data.index(&path.index_uid).await? { - Some(meta) => { + match data.index(path.index_uid.clone()).await { + Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) } - None => { - let e = format!("Index {:?} doesn't exist.", path.index_uid); + Err(e) => { Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } } @@ -61,7 +60,8 @@ async fn create_index( data: web::Data, body: web::Json, ) -> Result { - match data.create_index(&body.uid, body.primary_key.clone()).await { + let body = body.into_inner(); + match data.create_index(body.uid, body.primary_key).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -95,7 +95,8 @@ async fn update_index( path: web::Path, body: web::Json, ) -> Result { - match data.update_index(&path.index_uid, body.primary_key.as_ref(), body.uid.as_ref()).await { + let body = body.into_inner(); + match data.update_index(path.into_inner().index_uid, body.primary_key, body.uid).await { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -133,16 +134,13 @@ async fn get_update_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id).await; + let params = path.into_inner(); + let result = data.get_update_status(params.index_uid, params.update_id).await; match result { - Ok(Some(meta)) => { + Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) } - Ok(None) => { - let e = format!("update {} for index {:?} doesn't exists.", path.update_id, path.index_uid); - Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) - } Err(e) => { Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } @@ -154,7 +152,7 @@ async fn get_all_updates_status( data: web::Data, path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid).await; + let result = data.get_updates_status(path.into_inner().index_uid).await; match result { Ok(metas) => { let json = serde_json::to_string(&metas).unwrap(); diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index eb03cf94d..2e718ebe9 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -36,19 +36,19 @@ impl TryFrom for SearchQuery { fn try_from(other: SearchQueryGet) -> anyhow::Result { let attributes_to_retrieve = other .attributes_to_retrieve - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_crop = other .attributes_to_crop - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let attributes_to_highlight = other .attributes_to_highlight - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let facet_distributions = other .facet_distributions - .map(|attrs| attrs.split(",").map(String::from).collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); let facet_filters = match other.facet_filters { Some(ref f) => Some(serde_json::from_str(f)?), @@ -83,7 +83,7 @@ async fn search_with_url_query( return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) } }; - let search_result = data.search(&path.index_uid, query).await; + let search_result = data.search(path.into_inner().index_uid, query).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); @@ -101,7 +101,7 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(&path.index_uid, params.into_inner()).await; + let search_result = data.search(path.into_inner().index_uid, params.into_inner()).await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index 345a86534..df2fe9f15 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -64,7 +64,7 @@ macro_rules! make_setting_route { data: actix_web::web::Data, index_uid: actix_web::web::Path, ) -> std::result::Result { - match data.settings(index_uid.as_ref()).await { + match data.settings(index_uid.into_inner()).await { Ok(settings) => { let setting = settings.$attr; let json = serde_json::to_string(&setting).unwrap(); @@ -153,7 +153,7 @@ async fn get_all( data: web::Data, index_uid: web::Path, ) -> Result { - match data.settings(index_uid.as_ref()).await { + match data.settings(index_uid.into_inner()).await { Ok(settings) => { let json = serde_json::to_string(&settings).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 0bf2f15a5..8f7224602 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -69,5 +69,5 @@ async fn test_create_multiple_indexes() { assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); assert_eq!(index3.get().await.1, 200); - assert_eq!(index4.get().await.1, 404); + assert_eq!(index4.get().await.1, 400); } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 5bc78950f..9d145899e 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -14,7 +14,7 @@ async fn create_and_delete_index() { assert_eq!(code, 200); - assert_eq!(index.get().await.1, 404); + assert_eq!(index.get().await.1, 400); } #[actix_rt::test] diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index a1a95eef8..1a1815dfe 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -29,7 +29,7 @@ async fn get_unexisting_index() { let (_response, code) = index.get().await; - assert_eq!(code, 404); + assert_eq!(code, 400); } #[actix_rt::test] From c29b86849be93857056fbd0be9529d8011b15827 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 17:40:20 +0100 Subject: [PATCH 68/69] use actix cors git dependency --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d40beeb70..bcdcec71c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,6 +54,7 @@ dependencies = [ [[package]] name = "actix-cors" version = "0.5.4" +source = "git+https://github.com/MarinPostma/actix-extras.git?rev=8f7b1fd#8f7b1fdd7f58f3a15542a27e6b237666bac756d4" dependencies = [ "actix-web", "derive_more", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 9e148c46b..9ae7baae2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -13,7 +13,7 @@ path = "src/main.rs" vergen = "3.1.0" [dependencies] -actix-cors = { path = "../../actix-extras/actix-cors" } +actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "8f7b1fd" } actix-http = { version = "3.0.0-beta.4", features = ["cookies"] } actix-service = "2.0.0-beta.4" actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] } From dd324807f92db888561a5c80c57336c671c984c3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 15 Mar 2021 18:11:10 +0100 Subject: [PATCH 69/69] last review edits + fmt --- meilisearch-error/src/lib.rs | 33 +++- meilisearch-http/src/data/mod.rs | 16 +- meilisearch-http/src/data/search.rs | 13 +- meilisearch-http/src/data/updates.rs | 37 ++-- meilisearch-http/src/error.rs | 47 +++-- .../src/helpers/authentication.rs | 17 +- meilisearch-http/src/helpers/compression.rs | 4 +- meilisearch-http/src/helpers/mod.rs | 2 - .../src/helpers/normalize_path.rs | 86 -------- meilisearch-http/src/index/mod.rs | 19 +- meilisearch-http/src/index/search.rs | 35 ++-- meilisearch-http/src/index/updates.rs | 32 +-- .../src/index_controller/index_actor.rs | 88 +++++---- meilisearch-http/src/index_controller/mod.rs | 104 ++++++---- .../src/index_controller/update_actor.rs | 14 +- .../src/index_controller/update_handler.rs | 9 +- .../src/index_controller/update_store.rs | 185 +++++++++--------- .../src/index_controller/updates.rs | 4 +- .../src/index_controller/uuid_resolver.rs | 26 ++- meilisearch-http/src/lib.rs | 88 ++++----- meilisearch-http/src/main.rs | 29 +-- meilisearch-http/src/option.rs | 10 +- meilisearch-http/src/routes/document.rs | 51 +++-- meilisearch-http/src/routes/index.rs | 12 +- meilisearch-http/src/routes/key.rs | 2 +- meilisearch-http/src/routes/search.rs | 10 +- meilisearch-http/src/routes/settings/mod.rs | 29 +-- meilisearch-http/src/routes/stats.rs | 4 +- meilisearch-http/src/routes/stop_words.rs | 2 +- meilisearch-http/src/routes/synonym.rs | 2 +- meilisearch-http/tests/common/index.rs | 26 ++- meilisearch-http/tests/common/mod.rs | 2 +- meilisearch-http/tests/common/server.rs | 7 +- meilisearch-http/tests/common/service.rs | 23 ++- .../tests/documents/add_documents.rs | 40 ++-- .../tests/documents/delete_documents.rs | 31 ++- .../tests/documents/get_documents.rs | 125 +++++++++--- meilisearch-http/tests/documents/mod.rs | 2 +- meilisearch-http/tests/index/create_index.rs | 1 - meilisearch-http/tests/index/delete_index.rs | 2 - meilisearch-http/tests/index/get_index.rs | 10 +- meilisearch-http/tests/index/mod.rs | 2 +- meilisearch-http/tests/integration.rs | 2 +- meilisearch-http/tests/search/mod.rs | 1 - .../tests/settings/get_settings.rs | 43 ++-- meilisearch-http/tests/updates/mod.rs | 26 +-- 46 files changed, 764 insertions(+), 589 deletions(-) delete mode 100644 meilisearch-http/src/helpers/normalize_path.rs diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs index d0e00e9be..4cf801c72 100644 --- a/meilisearch-error/src/lib.rs +++ b/meilisearch-error/src/lib.rs @@ -81,7 +81,6 @@ pub enum Code { } impl Code { - /// ascociate a `Code` variant to the actual ErrCode fn err_code(&self) -> ErrCode { use Code::*; @@ -94,17 +93,23 @@ impl Code { // thrown when requesting an unexisting index IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), - OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + OpenIndex => { + ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR) + } // invalid state error InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), // thrown when no primary key has been set MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), // error thrown when trying to set an already existing primary key - PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + PrimaryKeyAlreadyPresent => { + ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST) + } // invalid document - MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MaxFieldsLimitExceeded => { + ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST) + } MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), // error related to facets @@ -117,16 +122,26 @@ impl Code { DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), - MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + MissingAuthorizationHeader => { + ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED) + } NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), - RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + RetrieveDocument => { + ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST) + } SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), - UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + UnsupportedMediaType => { + ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE) + } // error related to dump - DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), - DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + DumpAlreadyInProgress => { + ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT) + } + DumpProcessFailed => { + ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR) + } } } diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index e0ae77ffb..c312c2912 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -7,9 +7,9 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexMetadata, IndexSettings}; -use crate::index_controller::IndexController; use crate::index::Settings; +use crate::index_controller::IndexController; +use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; #[derive(Clone)] @@ -72,7 +72,11 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { index_controller, options, api_keys }; + let inner = DataInner { + index_controller, + options, + api_keys, + }; let inner = Arc::new(inner); Ok(Data { inner }) @@ -90,7 +94,11 @@ impl Data { self.index_controller.get_index(uid).await } - pub async fn create_index(&self, uid: String, primary_key: Option) -> anyhow::Result { + pub async fn create_index( + &self, + uid: String, + primary_key: Option, + ) -> anyhow::Result { let settings = IndexSettings { uid: Some(uid), primary_key, diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs index 8838a11e0..1a998b997 100644 --- a/meilisearch-http/src/data/search.rs +++ b/meilisearch-http/src/data/search.rs @@ -1,7 +1,7 @@ use serde_json::{Map, Value}; -use crate::index::{SearchQuery, SearchResult}; use super::Data; +use crate::index::{SearchQuery, SearchResult}; impl Data { pub async fn search( @@ -19,7 +19,9 @@ impl Data { limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result>> { - self.index_controller.documents(index, offset, limit, attributes_to_retrieve).await + self.index_controller + .documents(index, offset, limit, attributes_to_retrieve) + .await } pub async fn retrieve_document( @@ -27,8 +29,9 @@ impl Data { index: String, document_id: String, attributes_to_retrieve: Option>, - ) -> anyhow::Result> - { - self.index_controller.document(index, document_id, attributes_to_retrieve).await + ) -> anyhow::Result> { + self.index_controller + .document(index, document_id, attributes_to_retrieve) + .await } } diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs index d5a246223..518f9fcc3 100644 --- a/meilisearch-http/src/data/updates.rs +++ b/meilisearch-http/src/data/updates.rs @@ -1,10 +1,9 @@ -use milli::update::{IndexDocumentsMethod, UpdateFormat}; use actix_web::web::Payload; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; -use crate::index::Settings; use super::Data; - +use crate::index::Settings; +use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus}; impl Data { pub async fn add_documents( @@ -14,9 +13,11 @@ impl Data { format: UpdateFormat, stream: Payload, primary_key: Option, - ) -> anyhow::Result - { - let update_status = self.index_controller.add_documents(index, method, format, stream, primary_key).await?; + ) -> anyhow::Result { + let update_status = self + .index_controller + .add_documents(index, method, format, stream, primary_key) + .await?; Ok(update_status) } @@ -26,14 +27,14 @@ impl Data { settings: Settings, create: bool, ) -> anyhow::Result { - let update = self.index_controller.update_settings(index, settings, create).await?; + let update = self + .index_controller + .update_settings(index, settings, create) + .await?; Ok(update) } - pub async fn clear_documents( - &self, - index: String, - ) -> anyhow::Result { + pub async fn clear_documents(&self, index: String) -> anyhow::Result { let update = self.index_controller.clear_documents(index).await?; Ok(update) } @@ -43,14 +44,14 @@ impl Data { index: String, document_ids: Vec, ) -> anyhow::Result { - let update = self.index_controller.delete_documents(index, document_ids).await?; + let update = self + .index_controller + .delete_documents(index, document_ids) + .await?; Ok(update) } - pub async fn delete_index( - &self, - index: String, - ) -> anyhow::Result<()> { + pub async fn delete_index(&self, index: String) -> anyhow::Result<()> { self.index_controller.delete_index(index).await?; Ok(()) } @@ -67,7 +68,7 @@ impl Data { &self, uid: String, primary_key: Option, - new_uid: Option + new_uid: Option, ) -> anyhow::Result { let settings = IndexSettings { uid: new_uid, diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 70e9c4ed8..8bfdb3573 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,14 +1,13 @@ use std::error; use std::fmt; -use actix_web::dev::HttpResponseBuilder; -use actix_web::http::Error as HttpError; use actix_web as aweb; +use actix_web::dev::HttpResponseBuilder; use actix_web::error::{JsonPayloadError, QueryPayloadError}; +use actix_web::http::Error as HttpError; use actix_web::http::StatusCode; -use serde::ser::{Serialize, Serializer, SerializeStruct}; -use meilisearch_error::{ErrorCode, Code}; - +use meilisearch_error::{Code, ErrorCode}; +use serde::ser::{Serialize, SerializeStruct, Serializer}; #[derive(Debug)] pub struct ResponseError { @@ -32,19 +31,25 @@ impl fmt::Display for ResponseError { // TODO: remove this when implementing actual error handling impl From for ResponseError { fn from(other: anyhow::Error) -> ResponseError { - ResponseError { inner: Box::new(Error::NotFound(other.to_string())) } + ResponseError { + inner: Box::new(Error::NotFound(other.to_string())), + } } } impl From for ResponseError { fn from(error: Error) -> ResponseError { - ResponseError { inner: Box::new(error) } + ResponseError { + inner: Box::new(error), + } } } impl From for ResponseError { fn from(err: FacetCountError) -> ResponseError { - ResponseError { inner: Box::new(err) } + ResponseError { + inner: Box::new(err), + } } } @@ -130,7 +135,10 @@ impl ErrorCode for Error { pub enum FacetCountError { AttributeNotSet(String), SyntaxError(String), - UnexpectedToken { found: String, expected: &'static [&'static str] }, + UnexpectedToken { + found: String, + expected: &'static [&'static str], + }, NoFacetSet, } @@ -143,7 +151,10 @@ impl ErrorCode for FacetCountError { } impl FacetCountError { - pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError { + pub fn unexpected_token( + found: impl ToString, + expected: &'static [&'static str], + ) -> FacetCountError { let found = found.to_string(); FacetCountError::UnexpectedToken { expected, found } } @@ -162,7 +173,9 @@ impl fmt::Display for FacetCountError { match self { AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr), SyntaxError(msg) => write!(f, "Syntax error: {}", msg), - UnexpectedToken { expected, found } => write!(f, "Unexpected {} found, expected {:?}", found, expected), + UnexpectedToken { expected, found } => { + write!(f, "Unexpected {} found, expected {:?}", found, expected) + } NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"), } } @@ -276,10 +289,14 @@ impl From for Error { impl From for Error { fn from(err: JsonPayloadError) -> Error { match err { - JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)), + JsonPayloadError::Deserialize(err) => { + Error::BadRequest(format!("Invalid JSON: {}", err)) + } JsonPayloadError::Overflow => Error::PayloadTooLarge, JsonPayloadError::ContentType => Error::UnsupportedMediaType, - JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)), + JsonPayloadError::Payload(err) => { + Error::BadRequest(format!("Problem while decoding the request: {}", err)) + } } } } @@ -287,7 +304,9 @@ impl From for Error { impl From for Error { fn from(err: QueryPayloadError) -> Error { match err { - QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)), + QueryPayloadError::Deserialize(err) => { + Error::BadRequest(format!("Invalid query parameters: {}", err)) + } } } } diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs index 4fdbdeef5..9944c0bd4 100644 --- a/meilisearch-http/src/helpers/authentication.rs +++ b/meilisearch-http/src/helpers/authentication.rs @@ -3,7 +3,7 @@ use std::pin::Pin; use std::rc::Rc; use std::task::{Context, Poll}; -use actix_web::dev::{Transform, Service, ServiceResponse, ServiceRequest}; +use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use actix_web::web; use futures::future::{err, ok, Future, Ready}; @@ -70,10 +70,16 @@ where let auth_header = match req.headers().get("X-Meili-API-Key") { Some(auth) => match auth.to_str() { Ok(auth) => auth, - Err(_) => return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())), + Err(_) => { + return Box::pin(err( + ResponseError::from(Error::MissingAuthorizationHeader).into() + )) + } }, None => { - return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())); + return Box::pin(err( + ResponseError::from(Error::MissingAuthorizationHeader).into() + )); } }; @@ -93,9 +99,10 @@ where if authenticated { Box::pin(svc.call(req)) } else { - Box::pin(err( - ResponseError::from(Error::InvalidToken(auth_header.to_string())).into() + Box::pin(err(ResponseError::from(Error::InvalidToken( + auth_header.to_string(), )) + .into())) } } } diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index ff3e1258f..bbf14d578 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -1,9 +1,9 @@ -use flate2::Compression; use flate2::read::GzDecoder; use flate2::write::GzEncoder; +use flate2::Compression; use std::fs::{create_dir_all, File}; use std::path::Path; -use tar::{Builder, Archive}; +use tar::{Archive, Builder}; use crate::error::Error; diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index 5685345b2..d1204908f 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,6 +1,4 @@ pub mod authentication; -//pub mod normalize_path; pub mod compression; pub use authentication::Authentication; -//pub use normalize_path::NormalizePath; diff --git a/meilisearch-http/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs deleted file mode 100644 index 51c64c52b..000000000 --- a/meilisearch-http/src/helpers/normalize_path.rs +++ /dev/null @@ -1,86 +0,0 @@ -/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 -use actix_web::http::Error; -use actix_service::{Service, Transform}; -use actix_web::{ - dev::ServiceRequest, - dev::ServiceResponse, - http::uri::{PathAndQuery, Uri}, -}; -use futures::future::{ok, Ready}; -use regex::Regex; -use std::task::{Context, Poll}; -pub struct NormalizePath; - -impl Transform for NormalizePath -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = NormalizePathNormalization; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ok(NormalizePathNormalization { - service, - merge_slash: Regex::new("//+").unwrap(), - }) - } -} - -pub struct NormalizePathNormalization { - service: S, - merge_slash: Regex, -} - -impl Service for NormalizePathNormalization -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = S::Future; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, mut req: ServiceRequest) -> Self::Future { - let head = req.head_mut(); - - // always add trailing slash, might be an extra one - let path = head.uri.path().to_string() + "/"; - - if self.merge_slash.find(&path).is_some() { - // normalize multiple /'s to one / - let path = self.merge_slash.replace_all(&path, "/"); - - let path = if path.len() > 1 { - path.trim_end_matches('/') - } else { - &path - }; - - let mut parts = head.uri.clone().into_parts(); - let pq = parts.path_and_query.as_ref().unwrap(); - - let path = if let Some(q) = pq.query() { - bytes::Bytes::from(format!("{}?{}", path, q)) - } else { - bytes::Bytes::copy_from_slice(path.as_bytes()) - }; - parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap()); - - let uri = Uri::from_parts(parts).unwrap(); - req.match_info_mut().get_mut().update(&uri); - req.head_mut().uri = uri; - } - - self.service.call(req) - } -} diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 72453091f..188afd522 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -59,19 +59,17 @@ impl Index { }) } - pub fn retrieve_documents( + pub fn retrieve_documents>( &self, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> anyhow::Result>> - where - S: AsRef + Send + Sync + 'static, - { + ) -> anyhow::Result>> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -95,7 +93,8 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&txn, attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -109,11 +108,7 @@ impl Index { .map(|(_, d)| d); match document { - Some(document) => Ok(obkv_to_json( - &fields_to_display, - &fields_ids_map, - document, - )?), + Some(document) => Ok(obkv_to_json(&fields_to_display, &fields_ids_map, document)?), None => bail!("Document with id {} not found", doc_id), } } diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c7db9c313..a0de632be 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -1,14 +1,14 @@ -use std::time::Instant; -use std::collections::{HashSet, BTreeMap}; +use std::collections::{BTreeMap, HashSet}; use std::mem; +use std::time::Instant; -use either::Either; use anyhow::bail; +use either::Either; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{FacetCondition, MatchingWords, facet::FacetValue}; -use serde::{Serialize, Deserialize}; -use serde_json::{Value, Map}; +use milli::{facet::FacetValue, FacetCondition, MatchingWords}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; use super::Index; @@ -78,13 +78,15 @@ impl Index { let mut documents = Vec::new(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); - let fields_to_display = self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = + self.fields_to_display(&rtxn, query.attributes_to_retrieve, &fields_ids_map)?; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in self.documents(&rtxn, documents_ids)? { - let mut object = milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); + let mut object = + milli::obkv_to_json(&fields_to_display, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } @@ -183,15 +185,15 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } Value::Array(values) => Value::Array( values - .into_iter() - .map(|v| self.highlight_value(v, words_to_highlight)) - .collect(), + .into_iter() + .map(|v| self.highlight_value(v, words_to_highlight)) + .collect(), ), Value::Object(object) => Value::Object( object - .into_iter() - .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) - .collect(), + .into_iter() + .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) + .collect(), ), } } @@ -221,9 +223,6 @@ fn parse_facets( // Disabled for now //Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)), Value::Array(arr) => parse_facets_array(txn, index, arr), - v => bail!( - "Invalid facet expression, expected Array, found: {:?}", - v - ), + v => bail!("Invalid facet expression, expected Array, found: {:?}", v), } } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index b2d95c9ca..a519eaa82 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -4,8 +4,8 @@ use std::num::NonZeroUsize; use flate2::read::GzDecoder; use log::info; -use milli::update::{UpdateFormat, IndexDocumentsMethod, UpdateBuilder, DocumentAdditionResult}; -use serde::{Serialize, Deserialize, de::Deserializer}; +use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use serde::{de::Deserializer, Deserialize, Serialize}; use super::Index; @@ -23,14 +23,14 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub displayed_attributes: Option>>, #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub searchable_attributes: Option>>, @@ -40,7 +40,7 @@ pub struct Settings { #[serde( default, deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", + skip_serializing_if = "Option::is_none" )] pub ranking_rules: Option>>, } @@ -65,8 +65,9 @@ pub struct Facets { } fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> +where + T: Deserialize<'de>, + D: Deserializer<'de>, { Deserialize::deserialize(deserializer).map(Some) } @@ -85,7 +86,7 @@ impl Index { let mut wtxn = self.write_txn()?; // Set the primary key if not set already, ignore if already set. - if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { + if let (None, Some(ref primary_key)) = (self.primary_key(&wtxn)?, primary_key) { self.put_primary_key(&mut wtxn, primary_key)?; } @@ -106,10 +107,11 @@ impl Index { info!("document addition done: {:?}", result); - result.and_then(|addition_result| wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into)) + result.and_then(|addition_result| { + wtxn.commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into) + }) } pub fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { @@ -210,14 +212,16 @@ impl Index { let mut builder = update_builder.delete_documents(&mut txn, self)?; // We ignore unexisting document ids - ids.iter().for_each(|id| { builder.delete_external_id(id); }); + ids.iter().for_each(|id| { + builder.delete_external_id(id); + }); match builder.execute() { Ok(deleted) => txn .commit() .and(Ok(UpdateResult::DocumentDeletion { deleted })) .map_err(Into::into), - Err(e) => Err(e) + Err(e) => Err(e), } } } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index 605dee22b..d9ec8fcda 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -12,13 +12,13 @@ use heed::EnvOpenOptions; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tokio::fs::remove_dir_all; use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::spawn_blocking; -use tokio::fs::remove_dir_all; use uuid::Uuid; -use super::{IndexSettings, get_arc_ownership_blocking}; use super::update_handler::UpdateHandler; +use super::{get_arc_ownership_blocking, IndexSettings}; use crate::index::UpdateResult as UResult; use crate::index::{Document, Index, SearchQuery, SearchResult, Settings}; use crate::index_controller::{ @@ -49,7 +49,11 @@ impl IndexMeta { let created_at = index.created_at(&txn)?; let updated_at = index.updated_at(&txn)?; let primary_key = index.primary_key(&txn)?.map(String::from); - Ok(Self { primary_key, updated_at, created_at }) + Ok(Self { + primary_key, + updated_at, + created_at, + }) } } @@ -98,7 +102,7 @@ enum IndexMsg { uuid: Uuid, index_settings: IndexSettings, ret: oneshot::Sender>, - } + }, } struct IndexActor { @@ -136,8 +140,7 @@ impl IndexActor { store: S, ) -> Result { let options = IndexerOpts::default(); - let update_handler = - UpdateHandler::new(&options).map_err(IndexError::Error)?; + let update_handler = UpdateHandler::new(&options).map_err(IndexError::Error)?; let update_handler = Arc::new(update_handler); let read_receiver = Some(read_receiver); let write_receiver = Some(write_receiver); @@ -241,7 +244,11 @@ impl IndexActor { GetMeta { uuid, ret } => { let _ = ret.send(self.handle_get_meta(uuid).await); } - UpdateIndex { uuid, index_settings, ret } => { + UpdateIndex { + uuid, + index_settings, + ret, + } => { let _ = ret.send(self.handle_update_index(uuid, index_settings).await); } } @@ -366,30 +373,34 @@ impl IndexActor { } } - async fn handle_update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { - let index = self.store + async fn handle_update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + let index = self + .store .get(uuid) .await? .ok_or(IndexError::UnexistingIndex)?; - spawn_blocking(move || { - match index_settings.primary_key { - Some(ref primary_key) => { - let mut txn = index.write_txn()?; - if index.primary_key(&txn)?.is_some() { - return Err(IndexError::ExistingPrimaryKey) - } - index.put_primary_key(&mut txn, primary_key)?; - let meta = IndexMeta::new_txn(&index, &txn)?; - txn.commit()?; - Ok(meta) - }, - None => { - let meta = IndexMeta::new(&index)?; - Ok(meta) - }, + spawn_blocking(move || match index_settings.primary_key { + Some(ref primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexError::ExistingPrimaryKey); + } + index.put_primary_key(&mut txn, primary_key)?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) } - }).await + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + } + }) + .await .map_err(|e| IndexError::Error(e.into()))? } } @@ -416,7 +427,8 @@ impl IndexActorHandle { pub async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::CreateIndex { ret, + let msg = IndexMsg::CreateIndex { + ret, uuid, primary_key, }; @@ -502,10 +514,14 @@ impl IndexActorHandle { pub async fn update_index( &self, uuid: Uuid, - index_settings: IndexSettings + index_settings: IndexSettings, ) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::UpdateIndex { uuid, index_settings, ret }; + let msg = IndexMsg::UpdateIndex { + uuid, + index_settings, + ret, + }; let _ = self.read_sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } @@ -571,10 +587,7 @@ impl IndexStore for HeedIndexStore { let index = spawn_blocking(move || open_index(path, index_size)) .await .map_err(|e| IndexError::Error(e.into()))??; - self.index_store - .write() - .await - .insert(uuid, index.clone()); + self.index_store.write().await.insert(uuid, index.clone()); Ok(Some(index)) } } @@ -582,7 +595,8 @@ impl IndexStore for HeedIndexStore { async fn delete(&self, uuid: Uuid) -> Result> { let db_path = self.path.join(format!("index-{}", uuid)); - remove_dir_all(db_path).await + remove_dir_all(db_path) + .await .map_err(|e| IndexError::Error(e.into()))?; let index = self.index_store.write().await.remove(&uuid); Ok(index) @@ -590,11 +604,9 @@ impl IndexStore for HeedIndexStore { } fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path) - .map_err(|e| IndexError::Error(e.into()))?; + create_dir_all(&path).map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path) - .map_err(IndexError::Error)?; + let index = milli::Index::new(options, &path).map_err(IndexError::Error)?; Ok(Index(Arc::new(index))) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index ef8b16340..0db60a41b 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -9,17 +9,17 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use anyhow::bail; use actix_web::web::{Bytes, Payload}; +use anyhow::bail; use futures::stream::StreamExt; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::sleep; -pub use updates::{Processed, Processing, Failed}; -use crate::index::{SearchResult, SearchQuery, Document}; -use crate::index::{UpdateResult, Settings, Facets}; +use crate::index::{Document, SearchQuery, SearchResult}; +use crate::index::{Facets, Settings, UpdateResult}; +pub use updates::{Failed, Processed, Processing}; pub type UpdateStatus = updates::UpdateStatus; @@ -51,7 +51,6 @@ pub struct IndexSettings { pub primary_key: Option, } - pub struct IndexController { uuid_resolver: uuid_resolver::UuidResolverHandle, index_handle: index_actor::IndexActorHandle, @@ -59,11 +58,20 @@ pub struct IndexController { } impl IndexController { - pub fn new(path: impl AsRef, index_size: usize, update_store_size: usize) -> anyhow::Result { + pub fn new( + path: impl AsRef, + index_size: usize, + update_store_size: usize, + ) -> anyhow::Result { let uuid_resolver = uuid_resolver::UuidResolverHandle::new(&path)?; let index_actor = index_actor::IndexActorHandle::new(&path, index_size)?; - let update_handle = update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; - Ok(Self { uuid_resolver, index_handle: index_actor, update_handle }) + let update_handle = + update_actor::UpdateActorHandle::new(index_actor.clone(), &path, update_store_size)?; + Ok(Self { + uuid_resolver, + index_handle: index_actor, + update_handle, + }) } pub async fn add_documents( @@ -75,7 +83,11 @@ impl IndexController { primary_key: Option, ) -> anyhow::Result { let uuid = self.uuid_resolver.get_or_create(uid).await?; - let meta = UpdateMeta::DocumentsAddition { method, format, primary_key }; + let meta = UpdateMeta::DocumentsAddition { + method, + format, + primary_key, + }; let (sender, receiver) = mpsc::channel(10); // It is necessary to spawn a local task to senf the payload to the update handle to @@ -84,10 +96,13 @@ impl IndexController { tokio::task::spawn_local(async move { while let Some(bytes) = payload.next().await { match bytes { - Ok(bytes) => { let _ = sender.send(Ok(bytes)).await; }, + Ok(bytes) => { + let _ = sender.send(Ok(bytes)).await; + } Err(e) => { let error: Box = Box::new(e); - let _ = sender.send(Err(error)).await; }, + let _ = sender.send(Err(error)).await; + } } } }); @@ -105,7 +120,11 @@ impl IndexController { Ok(status) } - pub async fn delete_documents(&self, uid: String, document_ids: Vec) -> anyhow::Result { + pub async fn delete_documents( + &self, + uid: String, + document_ids: Vec, + ) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid).await?; let meta = UpdateMeta::DeleteDocuments; let (sender, receiver) = mpsc::channel(10); @@ -120,7 +139,12 @@ impl IndexController { Ok(status) } - pub async fn update_settings(&self, uid: String, settings: Settings, create: bool) -> anyhow::Result { + pub async fn update_settings( + &self, + uid: String, + settings: Settings, + create: bool, + ) -> anyhow::Result { let uuid = if create { let uuid = self.uuid_resolver.get_or_create(uid).await?; // We need to create the index upfront, since it would otherwise only be created when @@ -143,9 +167,12 @@ impl IndexController { Ok(status) } - pub async fn create_index(&self, index_settings: IndexSettings) -> anyhow::Result { - let IndexSettings { uid: name, primary_key } = index_settings; - let uid = name.unwrap(); + pub async fn create_index( + &self, + index_settings: IndexSettings, + ) -> anyhow::Result { + let IndexSettings { uid, primary_key } = index_settings; + let uid = uid.ok_or_else(|| anyhow::anyhow!("Can't create an index without a uid."))?; let uuid = self.uuid_resolver.create(uid.clone()).await?; let meta = self.index_handle.create_index(uuid, primary_key).await?; let _ = self.update_handle.create(uuid).await?; @@ -155,25 +182,20 @@ impl IndexController { } pub async fn delete_index(&self, uid: String) -> anyhow::Result<()> { - let uuid = self.uuid_resolver - .delete(uid) - .await?; + let uuid = self.uuid_resolver.delete(uid).await?; self.update_handle.delete(uuid).await?; self.index_handle.delete(uuid).await?; Ok(()) } pub async fn update_status(&self, uid: String, id: u64) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid) - .await?; + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.update_handle.update_status(uuid, id).await?; Ok(result) } pub async fn all_update_status(&self, uid: String) -> anyhow::Result> { - let uuid = self.uuid_resolver - .resolve(uid).await?; + let uuid = self.uuid_resolver.resolve(uid).await?; let result = self.update_handle.get_all_updates_status(uuid).await?; Ok(result) } @@ -193,9 +215,7 @@ impl IndexController { } pub async fn settings(&self, uid: String) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid.clone()) - .await?; + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let settings = self.index_handle.settings(uuid).await?; Ok(settings) } @@ -207,10 +227,11 @@ impl IndexController { limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result> { - let uuid = self.uuid_resolver - .resolve(uid.clone()) + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let documents = self + .index_handle + .documents(uuid, offset, limit, attributes_to_retrieve) .await?; - let documents = self.index_handle.documents(uuid, offset, limit, attributes_to_retrieve).await?; Ok(documents) } @@ -220,21 +241,24 @@ impl IndexController { doc_id: String, attributes_to_retrieve: Option>, ) -> anyhow::Result { - let uuid = self.uuid_resolver - .resolve(uid.clone()) + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; + let document = self + .index_handle + .document(uuid, doc_id, attributes_to_retrieve) .await?; - let document = self.index_handle.document(uuid, doc_id, attributes_to_retrieve).await?; Ok(document) } - pub async fn update_index(&self, uid: String, index_settings: IndexSettings) -> anyhow::Result { + pub async fn update_index( + &self, + uid: String, + index_settings: IndexSettings, + ) -> anyhow::Result { if index_settings.uid.is_some() { bail!("Can't change the index uid.") } - let uuid = self.uuid_resolver - .resolve(uid.clone()) - .await?; + let uuid = self.uuid_resolver.resolve(uid.clone()).await?; let meta = self.index_handle.update_index(uuid, index_settings).await?; let meta = IndexMetadata { uid, meta }; Ok(meta) @@ -248,9 +272,7 @@ impl IndexController { pub async fn get_index(&self, uid: String) -> anyhow::Result { let uuid = self.uuid_resolver.resolve(uid.clone()).await?; - let meta = self.index_handle - .get_index_meta(uuid) - .await?; + let meta = self.index_handle.get_index_meta(uuid).await?; let meta = IndexMetadata { uid, meta }; Ok(meta) } diff --git a/meilisearch-http/src/index_controller/update_actor.rs b/meilisearch-http/src/index_controller/update_actor.rs index 2b9d05a4b..1e788ff34 100644 --- a/meilisearch-http/src/index_controller/update_actor.rs +++ b/meilisearch-http/src/index_controller/update_actor.rs @@ -52,7 +52,7 @@ enum UpdateMsg { Create { uuid: Uuid, ret: oneshot::Sender>, - } + }, } struct UpdateActor { @@ -213,7 +213,11 @@ impl UpdateActorHandle where D: AsRef<[u8]> + Sized + 'static + Sync + Send, { - pub fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> anyhow::Result { + pub fn new( + index_handle: IndexActorHandle, + path: impl AsRef, + update_store_size: usize, + ) -> anyhow::Result { let path = path.as_ref().to_owned().join("updates"); let (sender, receiver) = mpsc::channel(100); let store = MapUpdateStoreStore::new(index_handle, &path, update_store_size); @@ -278,7 +282,11 @@ struct MapUpdateStoreStore { } impl MapUpdateStoreStore { - fn new(index_handle: IndexActorHandle, path: impl AsRef, update_store_size: usize) -> Self { + fn new( + index_handle: IndexActorHandle, + path: impl AsRef, + update_store_size: usize, + ) -> Self { let db = Arc::new(RwLock::new(HashMap::new())); let path = path.as_ref().to_owned(); Self { diff --git a/meilisearch-http/src/index_controller/update_handler.rs b/meilisearch-http/src/index_controller/update_handler.rs index 766dfc5f0..17f7107a2 100644 --- a/meilisearch-http/src/index_controller/update_handler.rs +++ b/meilisearch-http/src/index_controller/update_handler.rs @@ -1,14 +1,14 @@ use std::fs::File; +use crate::index::Index; use anyhow::Result; use grenad::CompressionType; use milli::update::UpdateBuilder; -use crate::index::Index; use rayon::ThreadPool; +use crate::index::UpdateResult; use crate::index_controller::updates::{Failed, Processed, Processing}; use crate::index_controller::UpdateMeta; -use crate::index::UpdateResult; use crate::option::IndexerOpts; pub struct UpdateHandler { @@ -23,9 +23,7 @@ pub struct UpdateHandler { } impl UpdateHandler { - pub fn new( - opt: &IndexerOpts, - ) -> anyhow::Result { + pub fn new(opt: &IndexerOpts) -> anyhow::Result { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(opt.indexing_jobs.unwrap_or(0)) .build()?; @@ -59,7 +57,6 @@ impl UpdateHandler { update_builder } - pub fn handle_update( &self, meta: Processing, diff --git a/meilisearch-http/src/index_controller/update_store.rs b/meilisearch-http/src/index_controller/update_store.rs index 9ce2f242b..6de30ab7f 100644 --- a/meilisearch-http/src/index_controller/update_store.rs +++ b/meilisearch-http/src/index_controller/update_store.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{Database, Env, EnvOpenOptions}; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; -use parking_lot::RwLock; use crate::index_controller::updates::*; @@ -252,24 +252,27 @@ where updates.extend(pending); - let aborted = - self.aborted_meta.iter(&rtxn)? + let aborted = self + .aborted_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); updates.extend(aborted); - let processed = - self.processed_meta.iter(&rtxn)? + let processed = self + .processed_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); updates.extend(processed); - let failed = - self.failed_meta.iter(&rtxn)? + let failed = self + .failed_meta + .iter(&rtxn)? .filter_map(Result::ok) .map(|(_, p)| p) .map(UpdateStatus::from); @@ -372,90 +375,90 @@ where //#[cfg(test)] //mod tests { - //use super::*; - //use std::thread; - //use std::time::{Duration, Instant}; +//use super::*; +//use std::thread; +//use std::time::{Duration, Instant}; - //#[test] - //fn simple() { - //let dir = tempfile::tempdir().unwrap(); - //let mut options = EnvOpenOptions::new(); - //options.map_size(4096 * 100); - //let update_store = UpdateStore::open( - //options, - //dir, - //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - //let new_meta = meta.meta().to_string() + " processed"; - //let processed = meta.process(new_meta); - //Ok(processed) - //}, - //) - //.unwrap(); +//#[test] +//fn simple() { +//let dir = tempfile::tempdir().unwrap(); +//let mut options = EnvOpenOptions::new(); +//options.map_size(4096 * 100); +//let update_store = UpdateStore::open( +//options, +//dir, +//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { +//let new_meta = meta.meta().to_string() + " processed"; +//let processed = meta.process(new_meta); +//Ok(processed) +//}, +//) +//.unwrap(); - //let meta = String::from("kiki"); - //let update = update_store.register_update(meta, &[]).unwrap(); - //thread::sleep(Duration::from_millis(100)); - //let meta = update_store.meta(update.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "kiki processed"); - //} else { - //panic!() - //} - //} - - //#[test] - //#[ignore] - //fn long_running_update() { - //let dir = tempfile::tempdir().unwrap(); - //let mut options = EnvOpenOptions::new(); - //options.map_size(4096 * 100); - //let update_store = UpdateStore::open( - //options, - //dir, - //|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - //thread::sleep(Duration::from_millis(400)); - //let new_meta = meta.meta().to_string() + "processed"; - //let processed = meta.process(new_meta); - //Ok(processed) - //}, - //) - //.unwrap(); - - //let before_register = Instant::now(); - - //let meta = String::from("kiki"); - //let update_kiki = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //let meta = String::from("coco"); - //let update_coco = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //let meta = String::from("cucu"); - //let update_cucu = update_store.register_update(meta, &[]).unwrap(); - //assert!(before_register.elapsed() < Duration::from_millis(200)); - - //thread::sleep(Duration::from_millis(400 * 3 + 100)); - - //let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "kiki processed"); - //} else { - //panic!() - //} - - //let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "coco processed"); - //} else { - //panic!() - //} - - //let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - //if let UpdateStatus::Processed(Processed { success, .. }) = meta { - //assert_eq!(success, "cucu processed"); - //} else { - //panic!() - //} - //} +//let meta = String::from("kiki"); +//let update = update_store.register_update(meta, &[]).unwrap(); +//thread::sleep(Duration::from_millis(100)); +//let meta = update_store.meta(update.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "kiki processed"); +//} else { +//panic!() +//} +//} + +//#[test] +//#[ignore] +//fn long_running_update() { +//let dir = tempfile::tempdir().unwrap(); +//let mut options = EnvOpenOptions::new(); +//options.map_size(4096 * 100); +//let update_store = UpdateStore::open( +//options, +//dir, +//|meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { +//thread::sleep(Duration::from_millis(400)); +//let new_meta = meta.meta().to_string() + "processed"; +//let processed = meta.process(new_meta); +//Ok(processed) +//}, +//) +//.unwrap(); + +//let before_register = Instant::now(); + +//let meta = String::from("kiki"); +//let update_kiki = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//let meta = String::from("coco"); +//let update_coco = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//let meta = String::from("cucu"); +//let update_cucu = update_store.register_update(meta, &[]).unwrap(); +//assert!(before_register.elapsed() < Duration::from_millis(200)); + +//thread::sleep(Duration::from_millis(400 * 3 + 100)); + +//let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "kiki processed"); +//} else { +//panic!() +//} + +//let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "coco processed"); +//} else { +//panic!() +//} + +//let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); +//if let UpdateStatus::Processed(Processed { success, .. }) = meta { +//assert_eq!(success, "cucu processed"); +//} else { +//panic!() +//} +//} //} diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs index b2ad54a14..3c70aee71 100644 --- a/meilisearch-http/src/index_controller/updates.rs +++ b/meilisearch-http/src/index_controller/updates.rs @@ -1,5 +1,5 @@ -use chrono::{Utc, DateTime}; -use serde::{Serialize, Deserialize}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] diff --git a/meilisearch-http/src/index_controller/uuid_resolver.rs b/meilisearch-http/src/index_controller/uuid_resolver.rs index 69e9a626b..2ee9c6b17 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver.rs @@ -1,6 +1,9 @@ use std::{fs::create_dir_all, path::Path}; -use heed::{Database, Env, EnvOpenOptions, types::{ByteSlice, Str}}; +use heed::{ + types::{ByteSlice, Str}, + Database, Env, EnvOpenOptions, +}; use log::{info, warn}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; @@ -73,14 +76,14 @@ impl UuidResolverActor { async fn handle_create(&self, uid: String) -> Result { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)) + return Err(UuidError::BadlyFormatted(uid)); } self.store.create_uuid(uid, true).await } async fn handle_get_or_create(&self, uid: String) -> Result { if !is_index_uid_valid(&uid) { - return Err(UuidError::BadlyFormatted(uid)) + return Err(UuidError::BadlyFormatted(uid)); } self.store.create_uuid(uid, false).await } @@ -106,7 +109,8 @@ impl UuidResolverActor { } fn is_index_uid_valid(uid: &str) -> bool { - uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') + uid.chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') } #[derive(Clone)] @@ -235,7 +239,8 @@ impl UuidStore for HeedUuidStore { Ok(uuid) } } - }).await? + }) + .await? } async fn get_uuid(&self, name: String) -> Result> { @@ -250,7 +255,8 @@ impl UuidStore for HeedUuidStore { } None => Ok(None), } - }).await? + }) + .await? } async fn delete(&self, uid: String) -> Result> { @@ -265,9 +271,10 @@ impl UuidStore for HeedUuidStore { txn.commit()?; Ok(Some(uuid)) } - None => Ok(None) + None => Ok(None), } - }).await? + }) + .await? } async fn list(&self) -> Result> { @@ -282,6 +289,7 @@ impl UuidStore for HeedUuidStore { entries.push((name.to_owned(), uuid)) } Ok(entries) - }).await? + }) + .await? } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index da1aee746..cdd874a35 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,63 +1,59 @@ pub mod data; pub mod error; pub mod helpers; -pub mod option; -pub mod routes; mod index; mod index_controller; +pub mod option; +pub mod routes; -pub use option::Opt; pub use self::data::Data; +pub use option::Opt; #[macro_export] macro_rules! create_app { - ($data:expr, $enable_frontend:expr) => { - { - use actix_cors::Cors; - use actix_web::App; - use actix_web::middleware::TrailingSlash; - use actix_web::{web, middleware}; - use meilisearch_http::error::payload_error_handler; - use meilisearch_http::routes::*; + ($data:expr, $enable_frontend:expr) => {{ + use actix_cors::Cors; + use actix_web::middleware::TrailingSlash; + use actix_web::App; + use actix_web::{middleware, web}; + use meilisearch_http::error::payload_error_handler; + use meilisearch_http::routes::*; - let app = App::new() - .data($data.clone()) - .app_data( - web::JsonConfig::default() + let app = App::new() + .data($data.clone()) + .app_data( + web::JsonConfig::default() .limit($data.http_payload_size_limit()) .content_type(|_mime| true) // Accept all mime types .error_handler(|err, _req| payload_error_handler(err).into()), - ) - .app_data( - web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(stop_words::services) - .configure(synonym::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); - let app = if $enable_frontend { - app - .service(load_html) - .service(load_css) - } else { - app - }; - app.wrap( - Cors::default() + ) + .app_data( + web::QueryConfig::default() + .error_handler(|err, _req| payload_error_handler(err).into()), + ) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(stop_words::services) + .configure(synonym::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services); + //.configure(routes::dump::services); + let app = if $enable_frontend { + app.service(load_html).service(load_css) + } else { + app + }; + app.wrap( + Cors::default() .send_wildcard() .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .max_age(86_400) // 24h - ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) - } - }; + .max_age(86_400), // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new(TrailingSlash::Trim)) + }}; } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 36959c5b3..9415f52d5 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -2,7 +2,7 @@ use std::env; use actix_web::HttpServer; use main_error::MainError; -use meilisearch_http::{Data, Opt, create_app}; +use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; //mod analytics; @@ -44,29 +44,30 @@ async fn main() -> Result<(), MainError> { } } "development" => { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .init(); } _ => unreachable!(), } //if let Some(path) = &opt.import_snapshot { - //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; + //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; //} let data = Data::new(opt.clone())?; //if !opt.no_analytics { - //let analytics_data = data.clone(); - //let analytics_opt = opt.clone(); - //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); + //let analytics_data = data.clone(); + //let analytics_opt = opt.clone(); + //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); //} //if let Some(path) = &opt.import_dump { - //dump::import_dump(&data, path, opt.dump_batch_size)?; + //dump::import_dump(&data, path, opt.dump_batch_size)?; //} //if opt.schedule_snapshot { - //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; //} print_launch_resume(&opt, &data); @@ -78,11 +79,14 @@ async fn main() -> Result<(), MainError> { Ok(()) } -async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box> { - +async fn run_http( + data: Data, + opt: Opt, + enable_frontend: bool, +) -> Result<(), Box> { let http_server = HttpServer::new(move || create_app!(&data, enable_frontend)) - // Disable signals allows the server to terminate immediately when a user enter CTRL-C - .disable_signals(); + // Disable signals allows the server to terminate immediately when a user enter CTRL-C + .disable_signals(); if let Some(config) = opt.get_ssl_config()? { http_server @@ -95,7 +99,6 @@ async fn run_http(data: Data, opt: Opt, enable_frontend: bool) -> Result<(), Box Ok(()) } - pub fn print_launch_resume(opt: &Opt, data: &Data) { let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 0a1d494de..82eb75fc1 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -1,15 +1,15 @@ -use std::{error, fs}; use std::io::{BufReader, Read}; use std::path::PathBuf; use std::sync::Arc; +use std::{error, fs}; use byte_unit::Byte; +use grenad::CompressionType; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, RootCertStore, }; -use grenad::CompressionType; use structopt::StructOpt; #[derive(Debug, Clone, StructOpt)] @@ -99,7 +99,11 @@ pub struct Opt { /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt(long, env = "SENTRY_DSN", default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337")] + #[structopt( + long, + env = "SENTRY_DSN", + default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337" + )] pub sentry_dsn: String, /// Disable Sentry error reporting. diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index c53c3cb85..59bc5fe0e 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -7,10 +7,10 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; use serde_json::Value; -use crate::Data; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +use crate::Data; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; @@ -19,7 +19,10 @@ macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { if let Some(content_type) = head.headers.get("Content-Type") { - content_type.to_str().map(|v| v.contains($guard_value)).unwrap_or(false) + content_type + .to_str() + .map(|v| v.contains($guard_value)) + .unwrap_or(false) } else { false } @@ -57,7 +60,10 @@ async fn get_document( ) -> Result { let index = path.index_uid.clone(); let id = path.document_id.clone(); - match data.retrieve_document(index, id, None as Option>).await { + match data + .retrieve_document(index, id, None as Option>) + .await + { Ok(document) => { let json = serde_json::to_string(&document).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -76,7 +82,10 @@ async fn delete_document( data: web::Data, path: web::Path, ) -> Result { - match data.delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]).await { + match data + .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) + .await + { Ok(result) => { let json = serde_json::to_string(&result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -104,16 +113,17 @@ async fn get_all_documents( let attributes_to_retrieve = params .attributes_to_retrieve .as_ref() - .map(|attrs| attrs - .split(',') - .map(String::from) - .collect::>()); + .map(|attrs| attrs.split(',').map(String::from).collect::>()); - match data.retrieve_documents( - path.index_uid.clone(), - params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), - params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), - attributes_to_retrieve).await { + match data + .retrieve_documents( + path.index_uid.clone(), + params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), + params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), + attributes_to_retrieve, + ) + .await + { Ok(docs) => { let json = serde_json::to_string(&docs).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -149,7 +159,8 @@ async fn add_documents_json( UpdateFormat::Json, body, params.primary_key.clone(), - ).await; + ) + .await; match addition_result { Ok(update) => { @@ -163,7 +174,6 @@ async fn add_documents_json( } } - /// Default route for adding documents, this should return an error and redirect to the documentation #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn add_documents_default( @@ -191,7 +201,7 @@ async fn update_documents_default( #[put( "/indexes/{index_uid}/documents", wrap = "Authentication::Private", - guard = "guard_json", + guard = "guard_json" )] async fn update_documents( data: web::Data, @@ -206,7 +216,8 @@ async fn update_documents( UpdateFormat::Json, body, params.primary_key.clone(), - ).await; + ) + .await; match addition_result { Ok(update) => { @@ -231,7 +242,11 @@ async fn delete_documents( ) -> Result { let ids = body .iter() - .map(|v| v.as_str().map(String::from).unwrap_or_else(|| v.to_string())) + .map(|v| { + v.as_str() + .map(String::from) + .unwrap_or_else(|| v.to_string()) + }) .collect(); match data.delete_documents(path.index_uid.clone(), ids).await { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index cb1c08e24..c7c9e521a 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -3,10 +3,10 @@ use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::Data; use crate::error::ResponseError; use crate::helpers::Authentication; use crate::routes::IndexParam; +use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(list_indexes) @@ -18,7 +18,6 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_all_updates_status); } - #[get("/indexes", wrap = "Authentication::Private")] async fn list_indexes(data: web::Data) -> Result { match data.list_indexes().await { @@ -96,7 +95,10 @@ async fn update_index( body: web::Json, ) -> Result { let body = body.into_inner(); - match data.update_index(path.into_inner().index_uid, body.primary_key, body.uid).await { + match data + .update_index(path.into_inner().index_uid, body.primary_key, body.uid) + .await + { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -135,7 +137,9 @@ async fn get_update_status( path: web::Path, ) -> Result { let params = path.into_inner(); - let result = data.get_update_status(params.index_uid, params.update_id).await; + let result = data + .get_update_status(params.index_uid, params.update_id) + .await; match result { Ok(meta) => { let json = serde_json::to_string(&meta).unwrap(); diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index a051ee0f9..b44d747c8 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -1,6 +1,6 @@ +use actix_web::get; use actix_web::web; use actix_web::HttpResponse; -use actix_web::get; use serde::Serialize; use crate::helpers::Authentication; diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 2e718ebe9..c98a5ac51 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -6,9 +6,9 @@ use serde::Deserialize; use crate::error::ResponseError; use crate::helpers::Authentication; +use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -use crate::index::{SearchQuery, DEFAULT_SEARCH_LIMIT}; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(search_with_post).service(search_with_url_query); @@ -80,7 +80,9 @@ async fn search_with_url_query( let query: SearchQuery = match params.into_inner().try_into() { Ok(q) => q, Err(e) => { - return Ok(HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() }))) + return Ok( + HttpResponse::BadRequest().body(serde_json::json!({ "error": e.to_string() })) + ) } }; let search_result = data.search(path.into_inner().index_uid, query).await; @@ -101,7 +103,9 @@ async fn search_with_post( path: web::Path, params: web::Json, ) -> Result { - let search_result = data.search(path.into_inner().index_uid, params.into_inner()).await; + let search_result = data + .search(path.into_inner().index_uid, params.into_inner()) + .await; match search_result { Ok(docs) => { let docs = serde_json::to_string(&docs).unwrap(); diff --git a/meilisearch-http/src/routes/settings/mod.rs b/meilisearch-http/src/routes/settings/mod.rs index df2fe9f15..5a6246f8c 100644 --- a/meilisearch-http/src/routes/settings/mod.rs +++ b/meilisearch-http/src/routes/settings/mod.rs @@ -1,9 +1,9 @@ -use actix_web::{web, HttpResponse, delete, get, post}; +use actix_web::{delete, get, post, web, HttpResponse}; -use crate::Data; use crate::error::ResponseError; -use crate::index::Settings; use crate::helpers::Authentication; +use crate::index::Settings; +use crate::Data; #[macro_export] macro_rules! make_setting_route { @@ -98,15 +98,15 @@ make_setting_route!( ); //make_setting_route!( - //"/indexes/{index_uid}/settings/distinct-attribute", - //String, - //distinct_attribute +//"/indexes/{index_uid}/settings/distinct-attribute", +//String, +//distinct_attribute //); //make_setting_route!( - //"/indexes/{index_uid}/settings/ranking-rules", - //Vec, - //ranking_rules +//"/indexes/{index_uid}/settings/ranking-rules", +//Vec, +//ranking_rules //); macro_rules! create_services { @@ -137,7 +137,10 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.into_inner(), body.into_inner(), true).await { + match data + .update_settings(index_uid.into_inner(), body.into_inner(), true) + .await + { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -170,7 +173,10 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.into_inner(), settings, false).await { + match data + .update_settings(index_uid.into_inner(), settings, false) + .await + { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -180,4 +186,3 @@ async fn delete_all( } } } - diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index cba64194b..eee6536c6 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,8 +1,8 @@ -use std::collections::{HashMap, BTreeMap}; +use std::collections::{BTreeMap, HashMap}; +use actix_web::get; use actix_web::web; use actix_web::HttpResponse; -use actix_web::get; use chrono::{DateTime, Utc}; use serde::Serialize; diff --git a/meilisearch-http/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs index 5b116e829..8f89b6425 100644 --- a/meilisearch-http/src/routes/stop_words.rs +++ b/meilisearch-http/src/routes/stop_words.rs @@ -1,5 +1,5 @@ -use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; +use actix_web::{web, HttpResponse}; use std::collections::BTreeSet; use crate::error::ResponseError; diff --git a/meilisearch-http/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs index 43a0e4aeb..4106e3754 100644 --- a/meilisearch-http/src/routes/synonym.rs +++ b/meilisearch-http/src/routes/synonym.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; -use actix_web::{web, HttpResponse}; use actix_web::{delete, get, post}; +use actix_web::{web, HttpResponse}; use crate::error::ResponseError; use crate::helpers::Authentication; diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index a14769ee1..8fda99ef9 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -19,7 +19,10 @@ impl Index<'_> { pub async fn load_test_set(&self) -> u64 { let url = format!("/indexes/{}/documents", self.uid); - let (response, code) = self.service.post_str(url, include_str!("../assets/test_set.json")).await; + let (response, code) = self + .service + .post_str(url, include_str!("../assets/test_set.json")) + .await; assert_eq!(code, 200); let update_id = response["updateId"].as_i64().unwrap(); self.wait_update_id(update_id as u64).await; @@ -60,7 +63,11 @@ impl Index<'_> { self.service.post(url, documents).await } - pub async fn update_documents(&self, documents: Value, primary_key: Option<&str>) -> (Value, StatusCode) { + pub async fn update_documents( + &self, + documents: Value, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { let url = match primary_key { Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), None => format!("/indexes/{}/documents", self.uid), @@ -95,7 +102,11 @@ impl Index<'_> { self.service.get(url).await } - pub async fn get_document(&self, id: u64, _options: Option) -> (Value, StatusCode) { + pub async fn get_document( + &self, + id: u64, + _options: Option, + ) -> (Value, StatusCode) { let url = format!("/indexes/{}/documents/{}", self.uid, id); self.service.get(url).await } @@ -111,7 +122,10 @@ impl Index<'_> { } if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { - url.push_str(&format!("attributesToRetrieve={}&", attributes_to_retrieve.join(","))); + url.push_str(&format!( + "attributesToRetrieve={}&", + attributes_to_retrieve.join(",") + )); } self.service.get(url).await @@ -129,7 +143,9 @@ impl Index<'_> { pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { let url = format!("/indexes/{}/documents/delete-batch", self.uid); - self.service.post(url, serde_json::to_value(&ids).unwrap()).await + self.service + .post(url, serde_json::to_value(&ids).unwrap()) + .await } pub async fn settings(&self) -> (Value, StatusCode) { diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index fd461e61c..d1874ae84 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -2,8 +2,8 @@ mod index; mod server; mod service; +pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; pub use server::Server; -pub use index::{GetDocumentOptions, GetAllDocumentsOptions}; /// Performs a search test on both post and get routes #[macro_export] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 090524601..6c7225a8f 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -5,7 +5,7 @@ use tempdir::TempDir; use urlencoding::encode; use meilisearch_http::data::Data; -use meilisearch_http::option::{Opt, IndexerOpts}; +use meilisearch_http::option::{IndexerOpts, Opt}; use super::index::Index; use super::service::Service; @@ -55,10 +55,7 @@ impl Server { let data = Data::new(opt).unwrap(); let service = Service(data); - Server { - service, - _dir: dir, - } + Server { service, _dir: dir } } /// Returns a view to an index. There is no guarantee that the index exists. diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index feff49cad..b9bbffc05 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -1,15 +1,14 @@ use actix_web::{http::StatusCode, test}; use serde_json::Value; -use meilisearch_http::data::Data; use meilisearch_http::create_app; +use meilisearch_http::data::Data; pub struct Service(pub Data); impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -24,9 +23,12 @@ impl Service { } /// Send a test post request from a text body, with a `content-type:application/json` header. - pub async fn post_str(&self, url: impl AsRef, body: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + pub async fn post_str( + &self, + url: impl AsRef, + body: impl AsRef, + ) -> (Value, StatusCode) { + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::post() .uri(url.as_ref()) @@ -42,8 +44,7 @@ impl Service { } pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::get().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; @@ -55,8 +56,7 @@ impl Service { } pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::put() .uri(url.as_ref()) @@ -71,8 +71,7 @@ impl Service { } pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { - let mut app = - test::init_service(create_app!(&self.0, true)).await; + let mut app = test::init_service(create_app!(&self.0, true)).await; let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); let res = test::call_service(&mut app, req).await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 37b06f46f..dd10541b5 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -1,7 +1,7 @@ -use serde_json::{json, Value}; use chrono::DateTime; +use serde_json::{json, Value}; -use crate::common::{Server, GetAllDocumentsOptions}; +use crate::common::{GetAllDocumentsOptions, Server}; #[actix_rt::test] async fn add_documents_no_index_creation() { @@ -32,9 +32,12 @@ async fn add_documents_no_index_creation() { assert_eq!(response["updateId"], 0); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 1); - let processed_at = DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); - let enqueued_at = DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); - let started_processing_at = DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); + let processed_at = + DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); + let enqueued_at = + DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); + let started_processing_at = + DateTime::parse_from_rfc3339(response["startedProcessingAt"].as_str().unwrap()).unwrap(); assert!(processed_at > started_processing_at); assert!(started_processing_at > enqueued_at); @@ -71,7 +74,8 @@ async fn document_addition_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.add_documents(documents, Some("primary")).await; assert_eq!(code, 200); + let (_response, code) = index.add_documents(documents, Some("primary")).await; + assert_eq!(code, 200); index.wait_update_id(0).await; @@ -97,7 +101,8 @@ async fn document_update_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.update_documents(documents, Some("primary")).await; assert_eq!(code, 200); + let (_response, code) = index.update_documents(documents, Some("primary")).await; + assert_eq!(code, 200); index.wait_update_id(0).await; @@ -158,7 +163,7 @@ async fn update_documents_with_primary_key_and_primary_key_already_exists() { assert_eq!(code, 200); index.wait_update_id(0).await; -let (response, code) = index.get_update(0).await; + let (response, code) = index.get_update(0).await; assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["updateId"], 0); @@ -263,7 +268,10 @@ async fn update_document() { let (response, code) = index.get_document(1, None).await; assert_eq!(code, 200); - assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); } #[actix_rt::test] @@ -275,7 +283,12 @@ async fn add_larger_dataset() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 77); } @@ -291,7 +304,12 @@ async fn update_larger_dataset() { assert_eq!(code, 200); assert_eq!(response["status"], "processed"); assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 77); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(1000), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 77); } diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index 3dba7e6fd..e7a01acd4 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -1,6 +1,6 @@ use serde_json::json; -use crate::common::{Server, GetAllDocumentsOptions}; +use crate::common::{GetAllDocumentsOptions, Server}; #[actix_rt::test] async fn delete_one_document_unexisting_index() { @@ -24,7 +24,9 @@ async fn delete_one_unexisting_document() { async fn delete_one_document() { let server = Server::new().await; let index = server.index("test"); - index.add_documents(json!([{ "id": 0, "content": "foobar" }]), None).await; + index + .add_documents(json!([{ "id": 0, "content": "foobar" }]), None) + .await; index.wait_update_id(0).await; let (_response, code) = server.index("test").delete_document(0).await; assert_eq!(code, 200); @@ -39,20 +41,26 @@ async fn clear_all_documents_unexisting_index() { let server = Server::new().await; let (_response, code) = server.index("test").clear_all_documents().await; assert_eq!(code, 400); - } #[actix_rt::test] async fn clear_all_documents() { let server = Server::new().await; let index = server.index("test"); - index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), None).await; + index + .add_documents( + json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), + None, + ) + .await; index.wait_update_id(0).await; let (_response, code) = index.clear_all_documents().await; assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -67,7 +75,9 @@ async fn clear_all_documents_empty_index() { assert_eq!(code, 200); let _update = index.wait_update_id(0).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -89,13 +99,14 @@ async fn delete_batch() { assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 1); assert_eq!(response.as_array().unwrap()[0]["id"], 3); } - #[actix_rt::test] async fn delete_no_document_batch() { let server = Server::new().await; @@ -106,7 +117,9 @@ async fn delete_no_document_batch() { assert_eq!(code, 200); let _update = index.wait_update_id(1).await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 3); } diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index c9ed83316..d54234860 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,5 +1,5 @@ -use crate::common::Server; use crate::common::GetAllDocumentsOptions; +use crate::common::Server; use serde_json::json; @@ -8,10 +8,7 @@ use serde_json::json; #[actix_rt::test] async fn get_unexisting_index_single_document() { let server = Server::new().await; - let (_response, code) = server - .index("test") - .get_document(1, None) - .await; + let (_response, code) = server.index("test").get_document(1, None).await; assert_eq!(code, 400); } @@ -20,9 +17,7 @@ async fn get_unexisting_document() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - let (_response, code) = index - .get_document(1, None) - .await; + let (_response, code) = index.get_document(1, None).await; assert_eq!(code, 400); } @@ -40,14 +35,15 @@ async fn get_document() { let (_, code) = index.add_documents(documents, None).await; assert_eq!(code, 200); index.wait_update_id(0).await; - let (response, code) = index - .get_document(0, None) - .await; + let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); - assert_eq!(response, serde_json::json!( { - "id": 0, - "content": "foobar", - })); + assert_eq!( + response, + serde_json::json!( { + "id": 0, + "content": "foobar", + }) + ); } #[actix_rt::test] @@ -67,7 +63,9 @@ async fn get_no_documents() { let (_, code) = index.create(None).await; assert_eq!(code, 200); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert!(response.as_array().unwrap().is_empty()); } @@ -78,7 +76,9 @@ async fn get_all_documents_no_options() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); let arr = response.as_array().unwrap(); assert_eq!(arr.len(), 20); @@ -109,7 +109,12 @@ async fn test_get_all_documents_limit() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { limit: Some(5), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(5), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 5); assert_eq!(response.as_array().unwrap()[0]["id"], 0); @@ -121,7 +126,12 @@ async fn test_get_all_documents_offset() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { offset: Some(5), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + offset: Some(5), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); assert_eq!(response.as_array().unwrap()[0]["id"], 13); @@ -133,35 +143,90 @@ async fn test_get_all_documents_attributes_to_retrieve() { let index = server.index("test"); index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name"]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name"]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); - assert!(response.as_array().unwrap()[0].as_object().unwrap().get("name").is_some()); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("name") + .is_some()); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec![]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec![]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 0); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 0 + ); - let (response, code) = index.get_all_documents(GetAllDocumentsOptions { attributes_to_retrieve: Some(vec!["name", "tags"]), ..Default::default() }).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name", "tags"]), + ..Default::default() + }) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 2); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 2 + ); } #[actix_rt::test] async fn get_documents_displayed_attributes() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["gender"]})).await; + index + .update_settings(json!({"displayedAttributes": ["gender"]})) + .await; index.load_test_set().await; - let (response, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0].as_object().unwrap().keys().count(), 1); - assert!(response.as_array().unwrap()[0].as_object().unwrap().get("gender").is_some()); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("gender") + .is_some()); let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); diff --git a/meilisearch-http/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs index ea0e39c69..a791a596f 100644 --- a/meilisearch-http/tests/documents/mod.rs +++ b/meilisearch-http/tests/documents/mod.rs @@ -1,3 +1,3 @@ mod add_documents; -mod get_documents; mod delete_documents; +mod get_documents; diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 8f7224602..003bbfc58 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,6 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); assert_eq!(response["uid"], "test"); assert!(response.get("createdAt").is_some()); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 9d145899e..39e79daaf 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -6,12 +6,10 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - assert_eq!(code, 200); let (_response, code) = index.delete().await; - assert_eq!(code, 200); assert_eq!(index.get().await.1, 400); diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 1a1815dfe..2ba7b86e4 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -52,6 +52,12 @@ async fn list_multiple_indexes() { assert!(response.is_array()); let arr = response.as_array().unwrap(); assert_eq!(arr.len(), 2); - assert!(arr.iter().find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null).is_some()); - assert!(arr.iter().find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key").is_some()); + assert!(arr + .iter() + .find(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null) + .is_some()); + assert!(arr + .iter() + .find(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key") + .is_some()); } diff --git a/meilisearch-http/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs index 253afcfc4..c9804c160 100644 --- a/meilisearch-http/tests/index/mod.rs +++ b/meilisearch-http/tests/index/mod.rs @@ -1,4 +1,4 @@ mod create_index; +mod delete_index; mod get_index; mod update_index; -mod delete_index; diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs index 7e54d8bbf..7578a8dc1 100644 --- a/meilisearch-http/tests/integration.rs +++ b/meilisearch-http/tests/integration.rs @@ -1,8 +1,8 @@ mod common; +mod documents; mod index; mod search; mod settings; -mod documents; mod updates; // Tests are isolated by features in different modules to allow better readability, test diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index 735a3478f..56ec6439c 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -1,3 +1,2 @@ // This modules contains all the test concerning search. Each particular feture of the search // should be tested in its own module to isolate tests and keep the tests readable. - diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 0a8972108..e4c38cb00 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -21,7 +21,17 @@ async fn get_settings() { assert_eq!(settings["searchableAttributes"], json!(["*"])); println!("{:?}", settings); assert_eq!(settings["attributesForFaceting"], json!({})); - assert_eq!(settings["rankingRules"], json!(["typo", "words", "proximity", "attribute", "wordsPosition", "exactness"])); + assert_eq!( + settings["rankingRules"], + json!([ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ); } #[actix_rt::test] @@ -36,20 +46,24 @@ async fn update_settings_unknown_field() { async fn test_partial_update() { let server = Server::new().await; let index = server.index("test"); - let (_response, _code) = index.update_settings(json!({"displayedAttributes": ["foo"]})).await; + let (_response, _code) = index + .update_settings(json!({"displayedAttributes": ["foo"]})) + .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["*"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); - let (_response, _) = index.update_settings(json!({"searchableAttributes": ["bar"]})).await; + let (_response, _) = index + .update_settings(json!({"searchableAttributes": ["bar"]})) + .await; index.wait_update_id(1).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["bar"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["bar"])); } #[actix_rt::test] @@ -64,20 +78,22 @@ async fn delete_settings_unexisting_index() { async fn reset_all_settings() { let server = Server::new().await; let index = server.index("test"); - index.update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})).await; + index + .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"]})) + .await; index.wait_update_id(0).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["foo"])); - assert_eq!(response["searchableAttributes"],json!(["bar"])); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["bar"])); index.delete_settings().await; index.wait_update_id(1).await; let (response, code) = index.settings().await; assert_eq!(code, 200); - assert_eq!(response["displayedAttributes"],json!(["*"])); - assert_eq!(response["searchableAttributes"],json!(["*"])); + assert_eq!(response["displayedAttributes"], json!(["*"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); } #[actix_rt::test] @@ -149,4 +165,5 @@ macro_rules! test_setting_routes { test_setting_routes!( attributes_for_faceting, displayed_attributes, - searchable_attributes); + searchable_attributes +); diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs index 64b5b560e..6ce5ca381 100644 --- a/meilisearch-http/tests/updates/mod.rs +++ b/meilisearch-http/tests/updates/mod.rs @@ -21,13 +21,15 @@ async fn get_update_status() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - index.add_documents( - serde_json::json!([{ - "id": 1, - "content": "foobar", - }]), - None - ).await; + index + .add_documents( + serde_json::json!([{ + "id": 1, + "content": "foobar", + }]), + None, + ) + .await; let (_response, code) = index.get_update(0).await; assert_eq!(code, 200); // TODO check resonse format, as per #48 @@ -55,10 +57,12 @@ async fn list_updates() { let server = Server::new().await; let index = server.index("test"); index.create(None).await; - index.add_documents( - serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), - None - ).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; let (response, code) = index.list_updates().await; assert_eq!(code, 200); assert_eq!(response.as_array().unwrap().len(), 1);