diff --git a/.dockerignore b/.dockerignore index 364510117..8c6bdbdeb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ target Dockerfile .dockerignore -.git .gitignore diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 601a8dbf4..19c3977fe 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -37,30 +37,6 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-armv7: - name: Publish for ARMv7 - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1.0.0 - - uses: uraimo/run-on-arch-action@v1.0.7 - id: runcmd - with: - architecture: armv7 - distribution: ubuntu18.04 - run: | - apt update - apt install -y curl gcc make - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - source $HOME/.cargo/env - cargo build --release --locked - - name: Upload the binary to release - uses: svenstaro/upload-release-action@v1-release - with: - repo_token: ${{ secrets.PUBLISH_TOKEN }} - file: target/release/meilisearch - asset_name: meilisearch-linux-armv7 - tag: ${{ github.ref }} - publish-armv8: name: Publish for ARMv8 runs-on: ubuntu-18.04 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5a8403d6d..1b361c9bc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,9 +19,16 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-latest] + os: [ubuntu-18.04, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -33,27 +40,18 @@ jobs: command: test args: --locked --release - # We don't run test on Windows since we get the following error: There is not enough space on the disk. - check-on-windows: - name: Cargo check on Windows - runs-on: windows-latest - steps: - - uses: actions/checkout@v2 - - name: Run cargo check without any default features - uses: actions-rs/cargo@v1 - with: - command: check - args: --no-default-features - - name: Run cargo check with all default features - uses: actions-rs/cargo@v1 - with: - command: check - clippy: name: Run Clippy runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -71,6 +69,13 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions-rs/toolchain@v1 with: profile: minimal diff --git a/.gitignore b/.gitignore index 3ae73d6d8..8aa76ff15 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /*.mdb /query-history.txt /data.ms +/snapshots +/dumps diff --git a/Cargo.lock b/Cargo.lock index c131c13d4..43222363e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,6 +620,17 @@ dependencies = [ "vec_map", ] +[[package]] +name = "concat-arrays" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "const_fn" version = "0.4.8" @@ -831,6 +842,26 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "enum-iterator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "env_logger" version = "0.8.4" @@ -1067,12 +1098,37 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "getset" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b328c01a4d71d2d8173daa93562a73ab0fe85616876f02500f53d82948c504" +dependencies = [ + "proc-macro-error", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "gimli" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +[[package]] +name = "git2" +version = "0.13.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.0" @@ -1146,7 +1202,7 @@ dependencies = [ [[package]] name = "heed" version = "0.12.0" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" dependencies = [ "byteorder", "heed-traits", @@ -1164,12 +1220,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" [[package]] name = "heed-types" version = "0.7.2" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" dependencies = [ "bincode", "heed-traits", @@ -1437,6 +1493,30 @@ version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +[[package]] +name = "libgit2-sys" +version = "0.12.21+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -1578,7 +1658,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.3", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1586,7 +1666,6 @@ dependencies = [ "num_cpus", "obkv", "once_cell", - "oxidized-json-checker", "parking_lot", "paste", "pin-project", @@ -1599,6 +1678,7 @@ dependencies = [ "serde", "serde_json", "serde_url_params", + "serdeval", "sha-1 0.9.6", "sha2", "siphasher", @@ -1619,24 +1699,8 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" -version = "0.2.2" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.3" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" +version = "0.2.5" +source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.5#c0b5cf741ed9485147f2cbe523f2214d4fa4c395" dependencies = [ "character_converter", "cow-utils", @@ -1676,12 +1740,13 @@ dependencies = [ [[package]] name = "milli" -version = "0.7.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.1#b4dcdbf00d232f9bc744a4a6ba4ebaf5cb592b56" +version = "0.10.2" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.2#879d5e8799836d93f8995810965b6797be4f69d1" dependencies = [ "bstr", "byteorder", "chrono", + "concat-arrays", "csv", "either", "flate2", @@ -1695,7 +1760,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.2.2", + "meilisearch-tokenizer", "memmap", "obkv", "once_cell", @@ -1857,9 +1922,9 @@ dependencies = [ [[package]] name = "obkv" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" +checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385" [[package]] name = "once_cell" @@ -1888,12 +1953,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "oxidized-json-checker" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" - [[package]] name = "page_size" version = "0.4.2" @@ -2476,15 +2535,6 @@ dependencies = [ "semver 0.11.0", ] -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver 1.0.3", -] - [[package]] name = "rustls" version = "0.19.1" @@ -2498,6 +2548,12 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustversion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" + [[package]] name = "ryu" version = "1.0.5" @@ -2547,12 +2603,6 @@ dependencies = [ "semver-parser 0.10.2", ] -[[package]] -name = "semver" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" - [[package]] name = "semver-parser" version = "0.7.0" @@ -2712,6 +2762,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serdeval" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94023adfd3d548a8bd9a1f09c09f44eaab7080c7a9ab20314bb65154bee62bd0" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -3315,6 +3374,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -3323,13 +3388,18 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.2.0" +version = "5.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" +checksum = "542f37b4798c879409865dde7908e746d836f77839c3a6bea5c8b4e4dcf6620b" dependencies = [ - "bitflags", + "anyhow", + "cfg-if 1.0.0", "chrono", - "rustc_version 0.4.0", + "enum-iterator", + "getset", + "git2", + "rustversion", + "thiserror", ] [[package]] @@ -3470,9 +3540,9 @@ dependencies = [ [[package]] name = "whatlang" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" +checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703" dependencies = [ "hashbrown 0.7.2", ] diff --git a/bors.toml b/bors.toml index e3348c36d..d85949f13 100644 --- a/bors.toml +++ b/bors.toml @@ -1,7 +1,7 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', - 'Cargo check on Windows', + 'Tests on windows-latest', 'Run Clippy', 'Run Rustfmt' ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index bad303643..0f8a5e21f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -18,7 +18,7 @@ hex = { version = "0.4.3", optional = true } reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true } sha-1 = { version = "0.9.4", optional = true } tempfile = { version = "3.1.0", optional = true } -vergen = "3.1.0" +vergen = { version = "5.1.13", default-features = false, features = ["build", "git"] } zip = { version = "0.5.12", optional = true } [dependencies] @@ -42,20 +42,19 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.2" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" -oxidized-json-checker = "0.3.2" parking_lot = "0.11.1" rand = "0.7.3" rayon = "1.5.0" @@ -73,10 +72,11 @@ thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" -obkv = "0.1.1" +obkv = "0.2.0" pin-project = "1.0.7" whoami = { version = "1.1.2", optional = true } reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } +serdeval = "0.1.0" [dependencies.sentry] default-features = false @@ -97,7 +97,7 @@ actix-rt = "2.1.0" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } mockall = "0.9.1" paste = "1.0.5" -serde_url_params = "0.2.0" +serde_url_params = "0.2.1" tempdir = "0.3.7" urlencoding = "1.1.1" @@ -119,5 +119,5 @@ default = ["analytics", "mini-dashboard"] jemallocator = "0.3.2" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip" -sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.4/build.zip" +sha1 = "750e8a8e56cfa61fbf9ead14b08a5f17ad3f3d37" diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 557e04fe7..e33acc892 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -1,12 +1,7 @@ -use vergen::{generate_cargo_keys, ConstantsFlags}; +use vergen::{vergen, Config}; fn main() { - // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag - let mut flags = ConstantsFlags::all(); - flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG); - - // Generate the 'cargo:' key output - generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); + vergen(Config::default()).unwrap(); #[cfg(feature = "mini-dashboard")] mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets"); diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 263e3bd52..3b10a1562 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -101,7 +101,7 @@ impl Index { let index = Self::open(&dst_dir_path, size)?; let mut txn = index.write_txn()?; - let handler = UpdateHandler::new(&indexing_options)?; + let handler = UpdateHandler::new(indexing_options)?; index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 7227f8d35..37fd6fc39 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -6,7 +6,7 @@ use std::path::Path; use std::sync::Arc; use heed::{EnvOpenOptions, RoTxn}; -use milli::obkv_to_json; +use milli::{obkv_to_json, FieldId}; use serde::{de::Deserializer, Deserialize}; use serde_json::{Map, Value}; @@ -62,34 +62,34 @@ impl Index { pub fn settings_txn(&self, txn: &RoTxn) -> Result> { let displayed_attributes = self - .displayed_fields(&txn)? + .displayed_fields(txn)? .map(|fields| fields.into_iter().map(String::from).collect()); let searchable_attributes = self - .searchable_fields(&txn)? + .searchable_fields(txn)? .map(|fields| fields.into_iter().map(String::from).collect()); - let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect(); + let filterable_attributes = self.filterable_fields(txn)?.into_iter().collect(); let criteria = self - .criteria(&txn)? + .criteria(txn)? .into_iter() .map(|c| c.to_string()) .collect(); let stop_words = self - .stop_words(&txn)? + .stop_words(txn)? .map(|stop_words| -> Result> { Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? .unwrap_or_else(BTreeSet::new); - let distinct_field = self.distinct_field(&txn)?.map(String::from); + let distinct_field = self.distinct_field(txn)?.map(String::from); // in milli each word in the synonyms map were split on their separator. Since we lost // this information we are going to put space between words. let synonyms = self - .synonyms(&txn)? + .synonyms(txn)? .iter() .map(|(key, values)| { ( @@ -102,7 +102,7 @@ impl Index { Ok(Settings { displayed_attributes: Some(displayed_attributes), searchable_attributes: Some(searchable_attributes), - filterable_attributes: Some(Some(faceted_attributes)), + filterable_attributes: Some(Some(filterable_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), distinct_attribute: Some(distinct_field), @@ -174,8 +174,8 @@ impl Index { txn: &heed::RoTxn, attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, - ) -> Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + ) -> Result> { + let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 2d8095559..620178773 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -239,7 +239,7 @@ fn compute_matches>( for (key, value) in document { let mut infos = Vec::new(); - compute_value_matches(&mut infos, value, matcher, &analyzer); + compute_value_matches(&mut infos, value, matcher, analyzer); if !infos.is_empty() { matches.insert(key.clone(), infos); } @@ -281,9 +281,9 @@ fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], query_crop_length: usize, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) -> BTreeMap { let mut formatted_options = BTreeMap::new(); @@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options( formatted_options: &mut BTreeMap, attr_to_highlight: &HashSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_highlight { let new_format = FormatOptions { @@ -329,7 +329,7 @@ fn add_highlight_to_formatted_options( break; } - if let Some(id) = fields_ids_map.id(&attr) { + if let Some(id) = fields_ids_map.id(attr) { if displayed_ids.contains(&id) { formatted_options.insert(id, new_format); } @@ -342,7 +342,7 @@ fn add_crop_to_formatted_options( attr_to_crop: &[String], crop_length: usize, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_crop { let mut split = attr.rsplitn(2, ':'); @@ -366,7 +366,7 @@ fn add_crop_to_formatted_options( } } - if let Some(id) = fields_ids_map.id(&attr_name) { + if let Some(id) = fields_ids_map.id(attr_name) { if displayed_ids.contains(&id) { formatted_options .entry(id) @@ -382,7 +382,7 @@ fn add_crop_to_formatted_options( fn add_non_formatted_ids_to_formatted_options( formatted_options: &mut BTreeMap, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, ) { for id in to_retrieve_ids { formatted_options.entry(*id).or_insert(FormatOptions { @@ -395,7 +395,7 @@ fn add_non_formatted_ids_to_formatted_options( fn make_document( attributes_to_retrieve: &BTreeSet, field_ids_map: &FieldsIdsMap, - obkv: obkv::KvReader, + obkv: obkv::KvReaderU16, ) -> Result { let mut document = Document::new(); @@ -418,7 +418,7 @@ fn make_document( fn format_fields>( field_ids_map: &FieldsIdsMap, - obkv: obkv::KvReader, + obkv: obkv::KvReaderU16, formatter: &Formatter, matching_words: &impl Matcher, formatted_options: &BTreeMap, @@ -580,13 +580,23 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { // Matcher::match since the call is expensive. if format_options.highlight && token.is_word() { if let Some(length) = matcher.matches(token.text()) { - if format_options.highlight { - out.push_str(&self.marks.0); - out.push_str(&word[..length]); - out.push_str(&self.marks.1); - out.push_str(&word[length..]); - return out; + match word.get(..length).zip(word.get(length..)) { + Some((head, tail)) => { + out.push_str(&self.marks.0); + out.push_str(head); + out.push_str(&self.marks.1); + out.push_str(tail); + } + // if we are in the middle of a character + // or if all the word should be highlighted, + // we highlight the complete word. + None => { + out.push_str(&self.marks.0); + out.push_str(word); + out.push_str(&self.marks.1); + } } + return out; } } out.push_str(word); @@ -741,6 +751,132 @@ mod test { assert_eq!(value["author"], "J. R. R. Tolkien"); } + /// https://github.com/meilisearch/MeiliSearch/issues/1368 + #[test] + fn formatted_with_highlight_emoji() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Go💼od luck.".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("JacobLey".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + // emojis are deunicoded during tokenization + // TODO Tokenizer should remove spaces after deunicode + matching_words.insert("gobriefcase od", Some(11)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Go💼od luck."); + assert_eq!(value["author"], "JacobLey"); + } + + #[test] + fn formatted_with_highlight_in_unicode_word() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("étoile".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. R. R. Tolkien".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("etoile", Some(1)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "étoile"); + assert_eq!(value["author"], "J. R. R. Tolkien"); + } + #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 09535721f..34fafd143 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -205,7 +205,7 @@ impl Index { // Set the primary key if not set already, ignore if already set. if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { - let mut builder = UpdateBuilder::new(0).settings(txn, &self); + let mut builder = UpdateBuilder::new(0).settings(txn, self); builder.set_primary_key(primary_key.to_string()); builder.execute(|_, _| ())?; } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index a7f1aa8d1..30923584e 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -73,7 +73,7 @@ struct Settings { #[serde(default, deserialize_with = "deserialize_some")] pub synonyms: Option>>>, #[serde(default, deserialize_with = "deserialize_some")] - pub filterable_attributes: Option>>, + pub attributes_for_faceting: Option>>, } fn load_index( @@ -98,7 +98,7 @@ fn load_index( let mut txn = index.write_txn()?; - let handler = UpdateHandler::new(&indexer_options)?; + let handler = UpdateHandler::new(indexer_options)?; index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?; @@ -142,23 +142,19 @@ impl From for index_controller::Settings { // representing the name of the faceted field + the type of the field. Since the type // was not known in the V1 of the dump we are just going to assume everything is a // String - filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + filterable_attributes: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())), // we need to convert the old `Vec` into a `BTreeSet` - ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter(|criterion| { match criterion.as_str() { - "words" | "typo" | "proximity" | "attribute" => Some(criterion), - s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + "words" | "typo" | "proximity" | "attribute" | "exactness" => true, + s if s.starts_with("asc") || s.starts_with("desc") => true, "wordsPosition" => { - warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); - Some(String::from("words")) - } - "exactness" => { - error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); - None + warn!("The criteria `attribute` and `wordsPosition` have been merged into a single criterion `attribute` so `wordsPositon` will be ignored"); + false } s => { error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); - None + false } } }).collect())), @@ -180,3 +176,17 @@ fn import_settings(dir_path: impl AsRef) -> anyhow::Result { Ok(metadata) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn settings_format_regression() { + let settings = Settings::default(); + assert_eq!( + r##"{"rankingRules":null,"distinctAttribute":null,"searchableAttributes":null,"displayedAttributes":null,"stopWords":null,"synonyms":null,"attributesForFaceting":null}"##, + serde_json::to_string(&settings).unwrap() + ); + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index a73740b02..c8aad6815 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -37,7 +37,7 @@ pub trait DumpActorHandle { async fn create_dump(&self) -> Result; /// Return the status of an already created dump - /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status] + /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info] async fn dump_info(&self, uid: String) -> Result; } diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 4085dc0bd..faad75e01 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -40,9 +40,9 @@ impl IndexMeta { } fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { - let created_at = index.created_at(&txn)?; - let updated_at = index.updated_at(&txn)?; - let primary_key = index.primary_key(&txn)?.map(String::from); + let created_at = index.created_at(txn)?; + let updated_at = index.updated_at(txn)?; + let primary_key = index.primary_key(txn)?.map(String::from); Ok(Self { created_at, updated_at, diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 8ba96dad1..59a22910f 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use async_stream::stream; use futures::StreamExt; use log::trace; -use oxidized_json_checker::JsonChecker; +use serdeval::*; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; @@ -180,7 +180,7 @@ where let update_store = self.store.clone(); tokio::task::spawn_blocking(move || { - use std::io::{copy, sink, BufReader, Seek}; + use std::io::{BufReader, Seek}; // If the payload is empty, ignore the check. let update_uuid = if let Some((mut file, uuid)) = file_path { @@ -188,14 +188,10 @@ where file.seek(SeekFrom::Start(0))?; // Check that the json payload is valid: let reader = BufReader::new(&mut file); - let mut checker = JsonChecker::new(reader); + // Validate that the payload is in the correct format. + let _: Seq> = serde_json::from_reader(reader) + .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; - if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { - // The json file is invalid, we use Serde to get a nice error message: - file.seek(SeekFrom::Start(0))?; - let _: serde_json::Value = serde_json::from_reader(file) - .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; - } Some(uuid) } else { None diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 7c46f98fa..79a3cca05 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -59,8 +59,8 @@ impl UpdateStore { let update_files_path = path.as_ref().join(super::UPDATE_DIR); create_dir_all(&update_files_path)?; - self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?; - self.dump_completed(&txn, uuids, &mut dump_data_file)?; + self.dump_pending(txn, uuids, &mut dump_data_file, &path)?; + self.dump_completed(txn, uuids, &mut dump_data_file)?; Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index f4abeae09..e23e05b52 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -162,8 +162,7 @@ impl UpdateStore { // function returns a Result and we must just unlock the loop on Result. 'outer: while timeout(duration, notification_receiver.recv()) .await - .transpose() - .map_or(false, |r| r.is_ok()) + .map_or(true, |o| o.is_some()) { loop { match update_store_weak.upgrade() { @@ -269,13 +268,12 @@ impl UpdateStore { self.pending_queue.remap_key_type::().put( wtxn, &(global_id, index_uuid, enqueued.id()), - &enqueued, + enqueued, )?; } _ => { let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; - self.updates - .put(wtxn, &(index_uuid, update.id()), &update)?; + self.updates.put(wtxn, &(index_uuid, update.id()), update)?; } } Ok(()) @@ -443,13 +441,16 @@ impl UpdateStore { while let Some(Ok(((_, uuid, _), pending))) = pendings.next() { if uuid == index_uuid { - unsafe { - pendings.del_current()?; - } let mut pending = pending.decode()?; if let Some(update_uuid) = pending.content.take() { uuids_to_remove.push(update_uuid); } + + // Invariant check: we can only delete the current entry when we don't hold + // references to it anymore. This must be done after we have retrieved its content. + unsafe { + pendings.del_current()?; + } } } @@ -471,13 +472,6 @@ impl UpdateStore { txn.commit()?; - uuids_to_remove - .iter() - .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) - .for_each(|path| { - let _ = remove_file(path); - }); - // If the currently processing update is from our index, we wait until it is // finished before returning. This ensure that no write to the index occurs after we delete it. if let State::Processing(uuid, _) = *self.state.read() { @@ -487,6 +481,16 @@ impl UpdateStore { } } + // Finally, remove any outstanding update files. This must be done after waiting for the + // last update to ensure that the update files are not deleted before the update needs + // them. + uuids_to_remove + .iter() + .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) + .for_each(|path| { + let _ = remove_file(path); + }); + Ok(()) } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 0eb61f84c..9df131365 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,3 +1,43 @@ +//! # MeiliSearch +//! Hello there, future contributors. If you are here and see this code, it's probably because you want to add a super new fancy feature in MeiliSearch or fix a bug and first of all, thank you for that! +//! +//! To help you in this task, we'll try to do a little overview of the project. +//! ## Milli +//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch. It's where we actually index documents and perform searches. Its purpose is to do these two tasks as fast as possible. You can give an update to milli, and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only uses one core). +//! As you can see, we're missing quite a lot of features here; milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implement dumps or snapshots, etc... +//! +//! ## `Index` module +//! The [index] module is what encapsulates one milli index. It abstracts over its transaction and isolates a task that can be run into a thread. This is the unit of interaction with milli. +//! If you add a feature to milli, you'll probably need to add it in this module too before exposing it to the rest of meilisearch. +//! +//! ## `IndexController` module +//! To handle multiple indexes, we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes, forward asynchronous updates to its indexes, and provide an API to search in its indexes synchronously. +//! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model). +//! +//! ### The actor model +//! Every actor is composed of at least three files: +//! - `mod.rs` declare and import all the files used by the actor. We also describe the interface (= all the methods) used to interact with the actor. If you are not modifying anything inside of an actor, this is usually all you need to see. +//! - `handle_impl.rs` implements the interface described in the `mod.rs`; in reality, there is no code logic in this file. Every method is only wrapping its parameters in a structure that is sent to the actor. This is useful for test and futureproofing. +//! - `message.rs` contains an enum that describes all the interactions you can have with the actor. +//! - `actor.rs` is used to create and execute the actor. It's where we'll write the loop looking for new messages and actually perform the tasks. +//! +//! MeiliSearch currently uses four actors: +//! - [`uuid_resolver`](index_controller/uuid_resolver/index.html) hold the association between the user-provided indexes name and the internal [`uuid`](https://en.wikipedia.org/wiki/Universally_unique_identifier) representation we use. +//! - [`index_actor`](index_controller::index_actor) is our representation of multiples indexes. Any request made to MeiliSearch that needs to talk to milli will pass through this actor. +//! - [`update_actor`](index_controller/update_actor/index.html) is in charge of indexes updates. Since updates can take a long time to receive and process, we need to: +//! 1. Store them as fast as possible so we can continue to receive other updates even if nothing has been processed +//! 2. Feed the `index_actor` with a new update every time it finished its current job. +//! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening. +//! +//! ## Data module +//! The [data] module provide a unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance +//! +//! ## HTTP server +//! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module. +//! Currently, the configuration of actix-web is made in the [lib.rs](crate). +//! Most of the routes use [extractors] to handle the authentication. + +#![allow(rustdoc::private_intra_doc_links)] pub mod data; #[macro_use] pub mod error; @@ -106,20 +146,13 @@ macro_rules! create_app { use actix_web::middleware::TrailingSlash; use actix_web::App; use actix_web::{middleware, web}; - use meilisearch_http::routes::*; + use meilisearch_http::routes; use meilisearch_http::{configure_auth, configure_data, dashboard}; App::new() .configure(|s| configure_data(s, $data.clone())) .configure(|s| configure_auth(s, &$data)) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services) - .configure(dump::services) + .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) .wrap( Cors::default() diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 5638c453f..78dacd640 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -49,12 +49,12 @@ async fn main() -> Result<(), MainError> { release: sentry::release_name!(), dsn: Some(SENTRY_DSN.parse()?), before_send: Some(Arc::new(|event| { - event - .message - .as_ref() - .map(|msg| msg.to_lowercase().contains("no space left on device")) - .unwrap_or(false) - .then(|| event) + if matches!(event.message, Some(ref msg) if msg.to_lowercase().contains("no space left on device")) + { + None + } else { + Some(event) + } })), ..Default::default() }); @@ -105,11 +105,11 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box pub fn print_launch_resume(opt: &Opt, data: &Data) { let commit_sha = match option_env!("COMMIT_SHA") { - Some("") | None => env!("VERGEN_SHA"), + Some("") | None => env!("VERGEN_GIT_SHA"), Some(commit_sha) => commit_sha, }; let commit_date = match option_env!("COMMIT_DATE") { - Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"), Some(commit_date) => commit_date, }; @@ -145,7 +145,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { " Thank you for using MeiliSearch! -We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html Anonymous telemetry: \"Enabled\"" ); diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 1f987a588..72bc55986 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -6,12 +6,12 @@ use crate::error::ResponseError; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/dumps").route(web::post().to(create_dump))) - .service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status))); +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("").route(web::post().to(create_dump))) + .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status))); } -async fn create_dump(data: GuardedData) -> Result { +pub async fn create_dump(data: GuardedData) -> Result { let res = data.create_dump().await?; debug!("returns: {:?}", res); diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs deleted file mode 100644 index 54237de1a..000000000 --- a/meilisearch-http/src/routes/health.rs +++ /dev/null @@ -1,11 +0,0 @@ -use actix_web::{web, HttpResponse}; - -use crate::error::ResponseError; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/health").route(web::get().to(get_health))); -} - -async fn get_health() -> Result { - Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) -} diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/indexes/documents.rs similarity index 84% rename from meilisearch-http/src/routes/document.rs rename to meilisearch-http/src/routes/indexes/documents.rs index 112467f67..a4bf465b5 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -49,32 +49,29 @@ fn guard_json(head: &actix_web::dev::RequestHead) -> bool { } #[derive(Deserialize)] -struct DocumentParam { +pub struct DocumentParam { index_uid: String, document_id: String, } -pub fn services(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::scope("/indexes/{index_uid}/documents") - .service( - web::resource("") - .route(web::get().to(get_all_documents)) - .route(web::post().guard(guard_json).to(add_documents)) - .route(web::put().guard(guard_json).to(update_documents)) - .route(web::delete().to(clear_all_documents)), - ) - // this route needs to be before the /documents/{document_id} to match properly - .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) - .service( - web::resource("/{document_id}") - .route(web::get().to(get_document)) - .route(web::delete().to(delete_document)), - ), + web::resource("") + .route(web::get().to(get_all_documents)) + .route(web::post().guard(guard_json).to(add_documents)) + .route(web::put().guard(guard_json).to(update_documents)) + .route(web::delete().to(clear_all_documents)), + ) + // this route needs to be before the /documents/{document_id} to match properly + .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) + .service( + web::resource("/{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), ); } -async fn get_document( +pub async fn get_document( data: GuardedData, path: web::Path, ) -> Result { @@ -87,7 +84,7 @@ async fn get_document( Ok(HttpResponse::Ok().json(document)) } -async fn delete_document( +pub async fn delete_document( data: GuardedData, path: web::Path, ) -> Result { @@ -100,13 +97,13 @@ async fn delete_document( #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct BrowseQuery { +pub struct BrowseQuery { offset: Option, limit: Option, attributes_to_retrieve: Option, } -async fn get_all_documents( +pub async fn get_all_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -137,13 +134,13 @@ async fn get_all_documents( #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct UpdateDocumentsQuery { +pub struct UpdateDocumentsQuery { primary_key: Option, } /// Route used when the payload type is "application/json" /// Used to add or replace documents -async fn add_documents( +pub async fn add_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -166,7 +163,7 @@ async fn add_documents( /// Route used when the payload type is "application/json" /// Used to add or replace documents -async fn update_documents( +pub async fn update_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -187,7 +184,7 @@ async fn update_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } -async fn delete_documents( +pub async fn delete_documents( data: GuardedData, path: web::Path, body: web::Json>, @@ -207,7 +204,7 @@ async fn delete_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -async fn clear_all_documents( +pub async fn clear_all_documents( data: GuardedData, path: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/indexes/mod.rs similarity index 57% rename from meilisearch-http/src/routes/index.rs rename to meilisearch-http/src/routes/indexes/mod.rs index badbdcc10..8314bf032 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -3,42 +3,63 @@ use chrono::{DateTime, Utc}; use log::debug; use serde::{Deserialize, Serialize}; -use super::{IndexParam, UpdateStatusResponse}; use crate::error::ResponseError; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::routes::IndexParam; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { +pub mod documents; +pub mod search; +pub mod settings; +pub mod updates; + +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::resource("indexes") + web::resource("") .route(web::get().to(list_indexes)) .route(web::post().to(create_index)), ) .service( - web::resource("/indexes/{index_uid}") - .route(web::get().to(get_index)) - .route(web::put().to(update_index)) - .route(web::delete().to(delete_index)), - ) - .service( - web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)), - ) - .service( - web::resource("/indexes/{index_uid}/updates/{update_id}") - .route(web::get().to(get_update_status)), + web::scope("/{index_uid}") + .service( + web::resource("") + .route(web::get().to(get_index)) + .route(web::put().to(update_index)) + .route(web::delete().to(delete_index)), + ) + .service(web::resource("/stats").route(web::get().to(get_index_stats))) + .service(web::scope("/documents").configure(documents::configure)) + .service(web::scope("/search").configure(search::configure)) + .service(web::scope("/updates").configure(updates::configure)) + .service(web::scope("/settings").configure(settings::configure)), ); } +pub async fn list_indexes(data: GuardedData) -> Result { + let indexes = data.list_indexes().await?; + debug!("returns: {:?}", indexes); + Ok(HttpResponse::Ok().json(indexes)) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct IndexCreateRequest { +pub struct IndexCreateRequest { uid: String, primary_key: Option, } +pub async fn create_index( + data: GuardedData, + body: web::Json, +) -> Result { + let body = body.into_inner(); + let meta = data.create_index(body.uid, body.primary_key).await?; + Ok(HttpResponse::Created().json(meta)) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct UpdateIndexRequest { +pub struct UpdateIndexRequest { uid: Option, primary_key: Option, } @@ -53,22 +74,7 @@ pub struct UpdateIndexResponse { primary_key: Option, } -async fn list_indexes(data: GuardedData) -> Result { - let indexes = data.list_indexes().await?; - debug!("returns: {:?}", indexes); - Ok(HttpResponse::Ok().json(indexes)) -} - -async fn create_index( - data: GuardedData, - body: web::Json, -) -> Result { - let body = body.into_inner(); - let meta = data.create_index(body.uid, body.primary_key).await?; - Ok(HttpResponse::Ok().json(meta)) -} - -async fn get_index( +pub async fn get_index( data: GuardedData, path: web::Path, ) -> Result { @@ -77,7 +83,7 @@ async fn get_index( Ok(HttpResponse::Ok().json(meta)) } -async fn update_index( +pub async fn update_index( data: GuardedData, path: web::Path, body: web::Json, @@ -91,7 +97,7 @@ async fn update_index( Ok(HttpResponse::Ok().json(meta)) } -async fn delete_index( +pub async fn delete_index( data: GuardedData, path: web::Path, ) -> Result { @@ -99,35 +105,12 @@ async fn delete_index( Ok(HttpResponse::NoContent().finish()) } -#[derive(Deserialize)] -struct UpdateParam { - index_uid: String, - update_id: u64, -} - -async fn get_update_status( - data: GuardedData, - path: web::Path, -) -> Result { - let params = path.into_inner(); - let meta = data - .get_update_status(params.index_uid, params.update_id) - .await?; - let meta = UpdateStatusResponse::from(meta); - debug!("returns: {:?}", meta); - Ok(HttpResponse::Ok().json(meta)) -} - -async fn get_all_updates_status( +pub async fn get_index_stats( data: GuardedData, path: web::Path, ) -> Result { - let metas = data.get_updates_status(path.into_inner().index_uid).await?; - let metas = metas - .into_iter() - .map(UpdateStatusResponse::from) - .collect::>(); + let response = data.get_index_stats(path.index_uid.clone()).await?; - debug!("returns: {:?}", metas); - Ok(HttpResponse::Ok().json(metas)) + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/indexes/search.rs similarity index 88% rename from meilisearch-http/src/routes/search.rs rename to meilisearch-http/src/routes/indexes/search.rs index 31a7dbd03..8a2f4ae1d 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -11,9 +11,9 @@ use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::resource("/indexes/{index_uid}/search") + web::resource("") .route(web::get().to(search_with_url_query)) .route(web::post().to(search_with_post)), ); @@ -77,19 +77,24 @@ impl From for SearchQuery { } } -async fn search_with_url_query( - data: GuardedData, +pub async fn search_with_url_query( + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { debug!("called with params: {:?}", params); let query = params.into_inner().into(); let search_result = data.search(path.into_inner().index_uid, query).await?; + + // Tests that the nb_hits is always set to false + #[cfg(test)] + assert!(!search_result.exhaustive_nb_hits); + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } -async fn search_with_post( +pub async fn search_with_post( data: GuardedData, path: web::Path, params: web::Json, @@ -98,6 +103,11 @@ async fn search_with_post( let search_result = data .search(path.into_inner().index_uid, params.into_inner()) .await?; + + // Tests that the nb_hits is always set to false + #[cfg(test)] + assert!(!search_result.exhaustive_nb_hits); + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs similarity index 82% rename from meilisearch-http/src/routes/settings.rs rename to meilisearch-http/src/routes/indexes/settings.rs index 812e37b58..47f18a68a 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -9,7 +9,7 @@ use crate::{error::ResponseError, index::Unchecked}; #[macro_export] macro_rules! make_setting_route { ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { - mod $attr { + pub mod $attr { use log::debug; use actix_web::{web, HttpResponse, Resource}; @@ -18,7 +18,7 @@ macro_rules! make_setting_route { use crate::index::Settings; use crate::extractors::authentication::{GuardedData, policies::*}; - async fn delete( + pub async fn delete( data: GuardedData, index_uid: web::Path, ) -> Result { @@ -32,7 +32,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - async fn update( + pub async fn update( data: GuardedData, index_uid: actix_web::web::Path, body: actix_web::web::Json>, @@ -47,7 +47,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - async fn get( + pub async fn get( data: GuardedData, index_uid: actix_web::web::Path, ) -> std::result::Result { @@ -69,68 +69,63 @@ macro_rules! make_setting_route { } make_setting_route!( - "/indexes/{index_uid}/settings/filterable-attributes", + "/filterable-attributes", std::collections::HashSet, filterable_attributes, "filterableAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/displayed-attributes", + "/displayed-attributes", Vec, displayed_attributes, "displayedAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/searchable-attributes", + "/searchable-attributes", Vec, searchable_attributes, "searchableAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/stop-words", + "/stop-words", std::collections::BTreeSet, stop_words, "stopWords" ); make_setting_route!( - "/indexes/{index_uid}/settings/synonyms", + "/synonyms", std::collections::BTreeMap>, synonyms, "synonyms" ); make_setting_route!( - "/indexes/{index_uid}/settings/distinct-attribute", + "/distinct-attribute", String, distinct_attribute, "distinctAttribute" ); -make_setting_route!( - "/indexes/{index_uid}/settings/ranking-rules", - Vec, - ranking_rules, - "rankingRules" -); +make_setting_route!("/ranking-rules", Vec, ranking_rules, "rankingRules"); -macro_rules! create_services { +macro_rules! generate_configure { ($($mod:ident),*) => { - pub fn services(cfg: &mut web::ServiceConfig) { - cfg - .service(web::resource("/indexes/{index_uid}/settings") - .route(web::post().to(update_all)) - .route(web::get().to(get_all)) - .route(web::delete().to(delete_all))) + pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service( + web::resource("") + .route(web::post().to(update_all)) + .route(web::get().to(get_all)) + .route(web::delete().to(delete_all))) $(.service($mod::resources()))*; } }; } -create_services!( +generate_configure!( filterable_attributes, displayed_attributes, searchable_attributes, @@ -140,7 +135,7 @@ create_services!( ranking_rules ); -async fn update_all( +pub async fn update_all( data: GuardedData, index_uid: web::Path, body: web::Json>, @@ -154,7 +149,7 @@ async fn update_all( Ok(HttpResponse::Accepted().json(json)) } -async fn get_all( +pub async fn get_all( data: GuardedData, index_uid: web::Path, ) -> Result { @@ -163,7 +158,7 @@ async fn get_all( Ok(HttpResponse::Ok().json(settings)) } -async fn delete_all( +pub async fn delete_all( data: GuardedData, index_uid: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/indexes/updates.rs b/meilisearch-http/src/routes/indexes/updates.rs new file mode 100644 index 000000000..471636abf --- /dev/null +++ b/meilisearch-http/src/routes/indexes/updates.rs @@ -0,0 +1,64 @@ +use actix_web::{web, HttpResponse}; +use chrono::{DateTime, Utc}; +use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::routes::{IndexParam, UpdateStatusResponse}; +use crate::Data; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("").route(web::get().to(get_all_updates_status))) + .service(web::resource("{update_id}").route(web::get().to(get_update_status))); +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateIndexRequest { + uid: Option, + primary_key: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateIndexResponse { + name: String, + uid: String, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + +#[derive(Deserialize)] +pub struct UpdateParam { + index_uid: String, + update_id: u64, +} + +pub async fn get_update_status( + data: GuardedData, + path: web::Path, +) -> Result { + let params = path.into_inner(); + let meta = data + .get_update_status(params.index_uid, params.update_id) + .await?; + let meta = UpdateStatusResponse::from(meta); + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) +} + +pub async fn get_all_updates_status( + data: GuardedData, + path: web::Path, +) -> Result { + let metas = data.get_updates_status(path.into_inner().index_uid).await?; + let metas = metas + .into_iter() + .map(UpdateStatusResponse::from) + .collect::>(); + + debug!("returns: {:?}", metas); + Ok(HttpResponse::Ok().json(metas)) +} diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs deleted file mode 100644 index d47e264be..000000000 --- a/meilisearch-http/src/routes/key.rs +++ /dev/null @@ -1,23 +0,0 @@ -use actix_web::{web, HttpResponse}; -use serde::Serialize; - -use crate::extractors::authentication::{policies::*, GuardedData}; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/keys").route(web::get().to(list))); -} - -#[derive(Serialize)] -struct KeysResponse { - private: Option, - public: Option, -} - -async fn list(data: GuardedData) -> HttpResponse { - let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(&KeysResponse { - private: api_keys.private, - public: api_keys.public, - }) -} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 520949cd8..184616892 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,21 +1,27 @@ use std::time::Duration; -use actix_web::HttpResponse; +use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; +use log::debug; use serde::{Deserialize, Serialize}; use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::index::{Settings, Unchecked}; use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; +use crate::Data; -pub mod document; -pub mod dump; -pub mod health; -pub mod index; -pub mod key; -pub mod search; -pub mod settings; -pub mod stats; +mod dump; +mod indexes; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("/health").route(web::get().to(get_health))) + .service(web::scope("/dumps").configure(dump::configure)) + .service(web::resource("/keys").route(web::get().to(list_keys))) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))) + .service(web::scope("/indexes").configure(indexes::configure)); +} #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(clippy::large_enum_variant)] @@ -43,7 +49,6 @@ pub enum UpdateType { impl From<&UpdateStatus> for UpdateType { fn from(other: &UpdateStatus) -> Self { use milli::update::IndexDocumentsMethod::*; - match other.meta() { UpdateMeta::DocumentsAddition { method, .. } => { let number = match other { @@ -223,3 +228,147 @@ impl IndexUpdateResponse { pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) } + +async fn get_stats(data: GuardedData) -> Result { + let response = data.get_all_stats().await?; + + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct VersionResponse { + commit_sha: String, + commit_date: String, + pkg_version: String, +} + +async fn get_version(_data: GuardedData) -> HttpResponse { + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_GIT_SHA"), + Some(commit_sha) => commit_sha, + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"), + Some(commit_date) => commit_date, + }; + + HttpResponse::Ok().json(VersionResponse { + commit_sha: commit_sha.to_string(), + commit_date: commit_date.to_string(), + pkg_version: env!("CARGO_PKG_VERSION").to_string(), + }) +} + +#[derive(Serialize)] +struct KeysResponse { + private: Option, + public: Option, +} + +pub async fn list_keys(data: GuardedData) -> HttpResponse { + let api_keys = data.api_keys.clone(); + HttpResponse::Ok().json(&KeysResponse { + private: api_keys.private, + public: api_keys.public, + }) +} + +pub async fn get_health() -> Result { + Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::data::Data; + use crate::extractors::authentication::GuardedData; + + /// A type implemented for a route that uses a authentication policy `Policy`. + /// + /// This trait is used for regression testing of route authenticaton policies. + trait Is {} + + macro_rules! impl_is_policy { + ($($param:ident)*) => { + impl Is for Func + where Func: Fn(GuardedData, $($param,)*) -> Res {} + + }; + } + + impl_is_policy! {} + impl_is_policy! {A} + impl_is_policy! {A B} + impl_is_policy! {A B C} + impl_is_policy! {A B C D} + + /// Emits a compile error if a route doesn't have the correct authentication policy. + /// + /// This works by trying to cast the route function into a Is type, where Policy it + /// the authentication policy defined for the route. + macro_rules! test_auth_routes { + ($($policy:ident => { $($route:expr,)*})*) => { + #[test] + fn test_auth() { + $($(let _: &dyn Is<$policy, _> = &$route;)*)* + } + }; + } + + test_auth_routes! { + Public => { + indexes::search::search_with_url_query, + indexes::search::search_with_post, + + indexes::documents::get_document, + indexes::documents::get_all_documents, + } + Private => { + get_stats, + get_version, + + indexes::create_index, + indexes::list_indexes, + indexes::get_index_stats, + indexes::delete_index, + indexes::update_index, + indexes::get_index, + + dump::create_dump, + + indexes::settings::filterable_attributes::get, + indexes::settings::displayed_attributes::get, + indexes::settings::searchable_attributes::get, + indexes::settings::stop_words::get, + indexes::settings::synonyms::get, + indexes::settings::distinct_attribute::get, + indexes::settings::filterable_attributes::update, + indexes::settings::displayed_attributes::update, + indexes::settings::searchable_attributes::update, + indexes::settings::stop_words::update, + indexes::settings::synonyms::update, + indexes::settings::distinct_attribute::update, + indexes::settings::filterable_attributes::delete, + indexes::settings::displayed_attributes::delete, + indexes::settings::searchable_attributes::delete, + indexes::settings::stop_words::delete, + indexes::settings::synonyms::delete, + indexes::settings::distinct_attribute::delete, + indexes::settings::delete_all, + indexes::settings::get_all, + indexes::settings::update_all, + + indexes::documents::clear_all_documents, + indexes::documents::delete_documents, + indexes::documents::update_documents, + indexes::documents::add_documents, + indexes::documents::delete_document, + + indexes::updates::get_all_updates_status, + indexes::updates::get_update_status, + } + Admin => { list_keys, } + } +} diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs deleted file mode 100644 index a0078d76a..000000000 --- a/meilisearch-http/src/routes/stats.rs +++ /dev/null @@ -1,56 +0,0 @@ -use actix_web::{web, HttpResponse}; -use log::debug; -use serde::Serialize; - -use crate::error::ResponseError; -use crate::extractors::authentication::{policies::*, GuardedData}; -use crate::routes::IndexParam; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats))) - .service(web::resource("/stats").route(web::get().to(get_stats))) - .service(web::resource("/version").route(web::get().to(get_version))); -} - -async fn get_index_stats( - data: GuardedData, - path: web::Path, -) -> Result { - let response = data.get_index_stats(path.index_uid.clone()).await?; - - debug!("returns: {:?}", response); - Ok(HttpResponse::Ok().json(response)) -} - -async fn get_stats(data: GuardedData) -> Result { - let response = data.get_all_stats().await?; - - debug!("returns: {:?}", response); - Ok(HttpResponse::Ok().json(response)) -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct VersionResponse { - commit_sha: String, - commit_date: String, - pkg_version: String, -} - -async fn get_version(_data: GuardedData) -> HttpResponse { - let commit_sha = match option_env!("COMMIT_SHA") { - Some("") | None => env!("VERGEN_SHA"), - Some(commit_sha) => commit_sha, - }; - let commit_date = match option_env!("COMMIT_DATE") { - Some("") | None => env!("VERGEN_COMMIT_DATE"), - Some(commit_date) => commit_date, - }; - - HttpResponse::Ok().json(VersionResponse { - commit_sha: commit_sha.to_string(), - commit_date: commit_date.to_string(), - pkg_version: env!("CARGO_PKG_VERSION").to_string(), - }) -} diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 7d98d0733..cb790ba70 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,4 +1,7 @@ -use std::time::Duration; +use std::{ + panic::{catch_unwind, resume_unwind, UnwindSafe}, + time::Duration, +}; use actix_web::http::StatusCode; use paste::paste; @@ -185,6 +188,37 @@ impl Index<'_> { self.service.get(url).await } + /// Performs both GET and POST search queries + pub async fn search( + &self, + query: Value, + test: impl Fn(Value, StatusCode) + UnwindSafe + Clone, + ) { + let (response, code) = self.search_post(query.clone()).await; + let t = test.clone(); + if let Err(e) = catch_unwind(move || t(response, code)) { + eprintln!("Error with post search"); + resume_unwind(e); + } + + let (response, code) = self.search_get(query).await; + if let Err(e) = catch_unwind(move || test(response, code)) { + eprintln!("Error with get search"); + resume_unwind(e); + } + } + + pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/search", self.uid); + self.service.post(url, query).await + } + + pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { + let params = serde_url_params::to_string(&query).unwrap(); + let url = format!("/indexes/{}/search?{}", self.uid, params); + self.service.get(url).await + } + make_settings_test_routes!(distinct_attribute); } diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 945bd6b5c..14344db35 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -61,7 +61,7 @@ async fn get_no_documents() { let server = Server::new().await; let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index .get_all_documents(GetAllDocumentsOptions::default()) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index e65908cb2..3c081f0ed 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,7 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); assert_eq!(response["uid"], "test"); assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); @@ -23,7 +23,7 @@ async fn create_index_with_primary_key() { let index = server.index("test"); let (response, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); assert_eq!(response["uid"], "test"); assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); @@ -41,7 +41,7 @@ async fn create_existing_index() { let index = server.index("test"); let (_, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_response, code) = index.create(Some("primary")).await; assert_eq!(code, 400); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index b0f067b24..10a6c0282 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -1,3 +1,5 @@ +use serde_json::json; + use crate::common::Server; #[actix_rt::test] @@ -6,7 +8,7 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_response, code) = index.delete().await; @@ -23,3 +25,16 @@ async fn delete_unexisting_index() { assert_eq!(code, 404); } + +#[actix_rt::test] +async fn loop_delete_add_documents() { + let server = Server::new().await; + let index = server.index("test"); + let documents = json!([{"id": 1, "field1": "hello"}]); + for _ in 0..50 { + let (response, code) = index.add_documents(documents.clone(), None).await; + assert_eq!(code, 202, "{}", response); + let (response, code) = index.delete().await; + assert_eq!(code, 204, "{}", response); + } +} diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index a6b22509e..ba26a8e3a 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -7,7 +7,7 @@ async fn create_and_get_index() { let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.get().await; diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index 8494bbae3..3599e1605 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -8,7 +8,7 @@ async fn stats() { let index = server.index("test"); let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.stats().await; diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index c7d910b59..454b55fce 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -7,7 +7,7 @@ async fn update_primary_key() { let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.update(Some("primary")).await; @@ -31,7 +31,7 @@ async fn update_nothing() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (update, code) = index.update(None).await; @@ -47,7 +47,7 @@ async fn update_existing_primary_key() { let index = server.index("test"); let (_response, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_update, code) = index.update(Some("primary2")).await; diff --git a/meilisearch-http/tests/search/errors.rs b/meilisearch-http/tests/search/errors.rs new file mode 100644 index 000000000..a09e8f76e --- /dev/null +++ b/meilisearch-http/tests/search/errors.rs @@ -0,0 +1,28 @@ +use crate::common::Server; +use serde_json::json; + +#[actix_rt::test] +async fn search_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + + index + .search(json!({"q": "hello"}), |response, code| { + assert_eq!(code, 404, "{}", response); + assert_eq!(response["errorCode"], "index_not_found"); + }) + .await; +} + +#[actix_rt::test] +async fn search_unexisting_parameter() { + let server = Server::new().await; + let index = server.index("test"); + + index + .search(json!({"marin": "hello"}), |response, code| { + assert_eq!(code, 400, "{}", response); + assert_eq!(response["errorCode"], "bad_request"); + }) + .await; +} diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index 56ec6439c..b2a8f760c 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -1,2 +1,195 @@ // This modules contains all the test concerning search. Each particular feture of the search // should be tested in its own module to isolate tests and keep the tests readable. + +mod errors; + +use crate::common::Server; +use once_cell::sync::Lazy; +use serde_json::{json, Value}; + +static DOCUMENTS: Lazy = Lazy::new(|| { + json!([ + { + "title": "Shazam!", + "id": "287947" + }, + { + "title": "Captain Marvel", + "id": "299537" + }, + { + "title": "Escape Room", + "id": "522681" + }, + { "title": "How to Train Your Dragon: The Hidden World", "id": "166428" + }, + { + "title": "Glass", + "id": "450465" + } + ]) +}); + +#[actix_rt::test] +async fn simple_placeholder_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search(json!({}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 5); + }) + .await; +} + +#[actix_rt::test] +async fn simple_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search(json!({"q": "glass"}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }) + .await; +} + +#[actix_rt::test] +async fn search_multiple_params() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search( + json!({ + "q": "glass", + "attributesToCrop": ["title:2"], + "attributesToHighlight": ["title"], + "limit": 1, + "offset": 0, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_with_filter_string_notation() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + index + .search( + json!({ + "filter": "title = Glass" + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_with_filter_array_notation() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + let (response, code) = index + .search_post(json!({ + "filter": ["title = Glass"] + })) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + + let (response, code) = index + .search_post(json!({ + "filter": [["title = Glass", "title = \"Shazam!\"", "title = \"Escape Room\""]] + })) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); +} + +#[actix_rt::test] +async fn search_facet_distribution() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + index + .search( + json!({ + "facetsDistribution": ["title"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + let dist = response["facetsDistribution"].as_object().unwrap(); + assert_eq!(dist.len(), 1); + assert!(dist.get("title").is_some()); + }, + ) + .await; +} + +#[actix_rt::test] +async fn displayed_attributes() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({ "displayedAttributes": ["title"] })) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + let (response, code) = index + .search_post(json!({ "attributesToRetrieve": ["title", "id"] })) + .await; + assert_eq!(code, 200, "{}", response); + assert!(response["hits"].get("title").is_none()); +} diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 0b523eef3..837c90cc1 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -206,7 +206,7 @@ macro_rules! test_setting_routes { let server = Server::new().await; let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 201, "{}", response); let url = format!("/indexes/test/settings/{}", stringify!($setting) .chars() diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index aba860256..39c59fb27 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -28,7 +28,7 @@ async fn stats() { let index = server.index("test"); let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = server.stats().await;