diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml new file mode 100644 index 000000000..33a8d7cb8 --- /dev/null +++ b/.github/release-draft-template.yml @@ -0,0 +1,13 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +version-template: '0.21.0-alpha.$PATCH' +exclude-labels: + - 'skip-changelog' +template: | + ## Changes + + $CHANGES +no-changes-template: 'Changes are coming soon 😎' +sort-direction: 'ascending' +version-resolver: + default: patch diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml new file mode 100644 index 000000000..570bc532e --- /dev/null +++ b/.github/workflows/flaky.yml @@ -0,0 +1,15 @@ +name: Look for flaky tests +on: + schedule: + - cron: "0 12 * * FRI" # every friday at 12:00PM + +jobs: + flaky: + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + - name: Install cargo-flaky + run: cargo install cargo-flaky + - name: Run cargo flaky 100 times + run: cargo flaky -i 100 --release diff --git a/.github/workflows/publish-docker-latest.yml b/.github/workflows/publish-docker-latest.yml index 967248a07..f887f5e2e 100644 --- a/.github/workflows/publish-docker-latest.yml +++ b/.github/workflows/publish-docker-latest.yml @@ -13,6 +13,9 @@ jobs: - name: Check if current release is latest run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)" id: release + - name: Set COMMIT_DATE env variable + run: | + echo "COMMIT_DATE=$( git log --pretty=format:'%ad' -n1 --date=short )" >> $GITHUB_ENV - name: Publish to Registry if: steps.release.outputs.is_latest == 'true' uses: elgohr/Publish-Docker-Github-Action@master @@ -20,3 +23,5 @@ jobs: name: getmeili/meilisearch username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + tag_names: true + buildargs: COMMIT_SHA,COMMIT_DATE diff --git a/.github/workflows/publish-docker-tag.yml b/.github/workflows/publish-docker-tag.yml index d607d4286..be540927a 100644 --- a/.github/workflows/publish-docker-tag.yml +++ b/.github/workflows/publish-docker-tag.yml @@ -11,10 +11,16 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 + - name: Set COMMIT_DATE env variable + run: | + echo "COMMIT_DATE=$( git log --pretty=format:'%ad' -n1 --date=short )" >> $GITHUB_ENV - name: Publish to Registry uses: elgohr/Publish-Docker-Github-Action@master + env: + COMMIT_SHA: ${{ github.sha }} with: name: getmeili/meilisearch username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} tag_names: true + buildargs: COMMIT_SHA,COMMIT_DATE diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 000000000..9ec8b9d64 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,16 @@ +name: Release Drafter + +on: + push: + branches: + - main + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v5 + with: + config-name: release-draft-template.yml + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..5a8403d6d --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,81 @@ +name: Rust + +on: + workflow_dispatch: + pull_request: + push: + # trying and staging branches are for Bors config + branches: + - trying + - staging + +env: + CARGO_TERM_COLOR: always + +jobs: + tests: + name: Tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, macos-latest] + steps: + - uses: actions/checkout@v2 + - name: Run cargo check without any default features + uses: actions-rs/cargo@v1 + with: + command: build + args: --locked --release --no-default-features + - name: Run cargo test + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --release + + # We don't run test on Windows since we get the following error: There is not enough space on the disk. + check-on-windows: + name: Cargo check on Windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Run cargo check without any default features + uses: actions-rs/cargo@v1 + with: + command: check + args: --no-default-features + - name: Run cargo check with all default features + uses: actions-rs/cargo@v1 + with: + command: check + + clippy: + name: Run Clippy + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: clippy + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets -- --deny warnings + + fmt: + name: Run Rustfmt + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt + - name: Run cargo fmt + run: cargo fmt --all -- --check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index bdb20fc70..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -on: - push: - branches: - - release-v* - - trying - - staging - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' # this only concerns tags on stable - -name: Test binaries with cargo test - -jobs: - check: - name: Test on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04, macos-latest] - steps: - - uses: actions/checkout@v1 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: clippy - - name: Run cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --release - - name: Run cargo clippy - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --all-targets - - build-image: - name: Test the build of Docker image - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - run: docker build . --file Dockerfile -t meilisearch - name: Docker build - - ## A push occurred on a release branch, a prerelease is created and assets are generated - prerelease: - name: create prerelease - needs: [check, build-image] - if: ${{ contains(github.ref, 'release-') && github.event_name == 'push' }} - runs-on: ubuntu-18.04 - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Get version number - id: version-number - run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')" - - name: Get commit count - id: commit-count - run: echo "##[set-output name=count;]$(git rev-list remotes/origin/master..remotes/origin/release-${{ steps.version-number.outputs.number }} --count)" - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # Personal Access Token - with: - tag_name: ${{ steps.version-number.outputs.number }}rc${{ steps.commit-count.outputs.count }} - release_name: Pre-release ${{ steps.version-number.outputs.number }}-rc${{ steps.commit-count.outputs.count }} - prerelease: true - - ## If a tag is pushed, a release is created for this tag, and assets will be generated - release: - name: create release - needs: [check, build-image] - if: ${{ contains(github.ref, 'tags/v') }} - runs-on: ubuntu-18.04 - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Get version number - id: version-number - run: echo "##[set-output name=number;]$(echo ${{ github.ref }} | sed 's/.*\(v.*\)/\1/')" - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.PUBLISH_TOKEN }} # PAT - with: - tag_name: ${{ steps.version-number.outputs.number }} - release_name: Meilisearch ${{ steps.version-number.outputs.number }} - prerelease: false diff --git a/.gitignore b/.gitignore index e1f56a99c..3ae73d6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /target -meilisearch-core/target **/*.csv **/*.json_lines **/*.rs.bk diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 6807a76f0..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,126 +0,0 @@ -## v0.20.0 - 2021-03-22 - - - Fix build on mac M1 (#1280) - - Server root returns 200 in production (#1292) - - Healthcheck returns 200 (#1291) - - Snapshot temporary files are not created in /tmp anymore (#1238) - -## v0.19.0 - 2021-02-09 - - - The snapshots are now created and then renamed in atomically (#1172) - - Fix a race condition when an update and a document addition are processed immediately one after the other (#1176) - - Latin synonyms are normalized during indexation (#1174) - -## v0.18.1 - 2021-01-14 - - - Fix unexpected CORS error (#1185) - -## v0.18.0 - 2021-01-11 - - - Integration with the new tokenizer (#1091) - - Fix setting consistency bug (#1128) - - Fix attributes to retrieve bug (#1131) - - Increase default payload size (#1147) - - Improvements to code quality (#1167, #1165, #1126, #1151) - -## v0.17.0 - 2020-11-30 - - Fix corrupted data during placeholder search (#1089) - - Remove maintenance error from http (#1082) - - Disable frontend in production (#1097) - - Update nbHits count with filtered documents (#849) - - Remove update changelog ci check (#1090) - - Add deploy on Platform.sh option to README (#1087) - - Change movie gifs in README (#1077) - - Remove some clippy warnings (#1100) - - Improve script `download-latest.sh` (#1054) - - Bump dependencies version (#1056, #1057, #1059) - -## v0.16.0 - 2020-11-02 - - - Automatically create index on document push if index doesn't exist (#914) - - Sort displayedAttributes and facetDistribution (#946) - -## v0.15.0 - 2020-09-30 - - - Update actix-web dependency to 3.0.0 (#963) - - Consider an empty query to be a placeholder search (#916) - -## v0.14.1 - - - Fix version mismatch in snapshot importation (#959) - -## v0.14.0 - - - Sort displayedAttributes (#943) - - Fix facet distribution case (#797) - - Snapshotting (#839) - - Fix bucket-sort unwrap bug (#915) - -## v0.13.0 - - - placeholder search (#771) - - Add database version mismatch check (#794) - - Displayed and searchable attributes wildcard (#846) - - Remove sys-info route (#810) - - Check database version mismatch (#794) - - Fix unique docid bug (#841) - - Error codes in updates (#792) - - Sentry disable argument (#813) - - Log analytics if enabled (#825) - - Fix default values displayed on web interface (#874) - -## v0.12.0 - - - Fix long documents not being indexed completely bug (#816) - - Fix distinct attribute returning id instead of name (#800) - - error code rename (#805) - -## v0.11.1 - - - Fix facet cache on document update (#789) - - Improvements on settings consistency (#778) - -## v0.11.0 - - - Change the HTTP framework, moving from tide to actix-web (#601) - - Bump sentry version to 0.18.1 (#690) - - Enable max payload size override (#684) - - Disable sentry in debug (#681) - - Better terminal greeting (#680) - - Fix highlight misalignment (#679) - - Add support for facet count (#676) - - Add support for faceted search (#631) - - Add support for configuring the lmdb map size (#646, #647) - - Add exposed port for Dockerfile (#654) - - Add sentry probe (#664) - - Fix url trailing slash and double slash issues (#659) - - Fix accept all Content-Type by default (#653) - - Return the error message from Serde when a deserialization error is encountered (#661) - - Fix NormalizePath middleware to make the dashboard accessible (#695) - - Update sentry features to remove openssl (#702) - - Add SSL support (#669) - - Rename fieldsFrequency into fieldsDistribution in stats (#719) - - Add support for error code reporting (#703) - - Allow the dashboard to query private servers (#732) - - Add telemetry (#720) - - Add post route for search (#735) - -## v0.10.1 - - - Add support for floating points in filters (#640) - - Add '@' character as tokenizer separator (#607) - - Add support for filtering on arrays of strings (#611) - -## v0.10.0 - - - Refined filtering (#592) - - Add the number of hits in search result (#541) - - Add support for aligned crop in search result (#543) - - Sanitize the content displayed in the web interface (#539) - - Add support of nested null, boolean and seq values (#571 and #568, #574) - - Fixed the core benchmark (#576) - - Publish an ARMv7 and ARMv8 binaries on releases (#540 and #581) - - Fixed a bug where the result of the update status after the first update was empty (#542) - - Fixed a bug where stop words were not handled correctly (#594) - - Fix CORS issues (#602) - - Support wildcard on attributes to retrieve, highlight, and crop (#549, #565, and #598) diff --git a/Cargo.lock b/Cargo.lock index 70df337f3..fb00dc762 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,49 +1,29 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +checksum = "1d5dbeb2d9e51344cb83ca7cc170f1217f9fe25bfc50160e6e200b5c31c1019a" dependencies = [ "bitflags", - "bytes 0.5.6", + "bytes 1.0.1", "futures-core", "futures-sink", "log", - "pin-project 0.4.27", + "pin-project-lite", "tokio", "tokio-util", ] -[[package]] -name = "actix-connect" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" -dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "derive_more", - "either", - "futures-util", - "http", - "log", - "rustls 0.18.1", - "tokio-rustls", - "trust-dns-proto", - "trust-dns-resolver", - "webpki", -] - [[package]] name = "actix-cors" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b133d8026a9f209a9aeeeacd028e7451bcca975f592881b305d37983f303d7" +version = "0.6.0-beta.1" +source = "git+https://github.com/MarinPostma/actix-extras.git?rev=2dac1a4#2dac1a421619bf7b386dea63d3ae25a3bc4abc43" dependencies = [ + "actix-service", "actix-web", "derive_more", "futures-util", @@ -54,67 +34,63 @@ dependencies = [ [[package]] name = "actix-http" -version = "2.2.0" +version = "3.0.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +checksum = "59d51c2ba06062e698a5d212d860e9fb2afc931c285ede687aaae896c8150347" dependencies = [ "actix-codec", - "actix-connect", "actix-rt", "actix-service", - "actix-threadpool", "actix-tls", "actix-utils", - "base64 0.13.0", + "ahash 0.7.4", + "base64", "bitflags", "brotli2", - "bytes 0.5.6", - "cookie", - "copyless", + "bytes 1.0.1", + "bytestring", "derive_more", - "either", "encoding_rs", "flate2", - "futures-channel", "futures-core", "futures-util", - "fxhash", "h2", "http", "httparse", - "indexmap", "itoa", "language-tags", - "lazy_static", + "local-channel", "log", "mime", + "once_cell", + "paste", "percent-encoding", - "pin-project 1.0.4", - "rand 0.7.3", + "pin-project", + "pin-project-lite", + "rand 0.8.4", "regex", "serde", - "serde_json", - "serde_urlencoded", - "sha-1 0.9.2", - "slab", - "time 0.2.24", + "sha-1 0.9.6", + "smallvec", + "time 0.2.27", + "tokio", ] [[package]] name = "actix-macros" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "actix-router" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8be584b3b6c705a18eabc11c4059cf83b255bdd8511673d1d569f4ce40c69de" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", @@ -125,119 +101,77 @@ dependencies = [ [[package]] name = "actix-rt" -version = "1.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" dependencies = [ "actix-macros", - "actix-threadpool", - "copyless", - "futures-channel", - "futures-util", - "smallvec", + "futures-core", "tokio", ] [[package]] name = "actix-server" -version = "1.0.4" +version = "2.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87" dependencies = [ - "actix-codec", "actix-rt", "actix-service", "actix-utils", - "futures-channel", - "futures-util", + "futures-core", "log", "mio", - "mio-uds", "num_cpus", "slab", - "socket2", + "tokio", ] [[package]] name = "actix-service" -version = "1.0.6" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +checksum = "77f5f9d66a8730d0fae62c26f3424f5751e5518086628a40b7ab6fca4a705034" dependencies = [ - "futures-util", - "pin-project 0.4.27", -] - -[[package]] -name = "actix-testing" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" -dependencies = [ - "actix-macros", - "actix-rt", - "actix-server", - "actix-service", - "log", - "socket2", -] - -[[package]] -name = "actix-threadpool" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" -dependencies = [ - "derive_more", - "futures-channel", - "lazy_static", - "log", - "num_cpus", - "parking_lot", - "threadpool", + "futures-core", + "paste", + "pin-project-lite", ] [[package]] name = "actix-tls" -version = "2.0.0" +version = "3.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" -dependencies = [ - "actix-codec", - "actix-service", - "actix-utils", - "futures-util", - "rustls 0.18.1", - "tokio-rustls", - "webpki", - "webpki-roots 0.20.0", -] - -[[package]] -name = "actix-utils" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +checksum = "65b7bb60840962ef0332f7ea01a57d73a24d2cb663708511ff800250bbfef569" dependencies = [ "actix-codec", "actix-rt", "actix-service", - "bitflags", - "bytes 0.5.6", - "either", - "futures-channel", - "futures-sink", - "futures-util", + "actix-utils", + "derive_more", + "futures-core", + "http", "log", - "pin-project 0.4.27", - "slab", + "tokio-rustls", + "tokio-util", + "webpki-roots", +] + +[[package]] +name = "actix-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" +dependencies = [ + "local-waker", + "pin-project-lite", ] [[package]] name = "actix-web" -version = "3.3.2" +version = "4.0.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +checksum = "ff12e933051557d700b0fcad20fe25b9ca38395cc87bbc5aeaddaef17b937a2f" dependencies = [ "actix-codec", "actix-http", @@ -246,58 +180,72 @@ dependencies = [ "actix-rt", "actix-server", "actix-service", - "actix-testing", - "actix-threadpool", "actix-tls", "actix-utils", "actix-web-codegen", - "awc", - "bytes 0.5.6", + "ahash 0.7.4", + "bytes 1.0.1", + "cookie", "derive_more", + "either", "encoding_rs", - "futures-channel", "futures-core", "futures-util", - "fxhash", + "itoa", + "language-tags", "log", "mime", - "pin-project 1.0.4", + "once_cell", + "pin-project", "regex", - "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", + "smallvec", "socket2", - "time 0.2.24", - "tinyvec", + "time 0.2.27", "url", ] [[package]] name = "actix-web-codegen" -version = "0.4.0" +version = "0.5.0-beta.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +checksum = "0d048c6986743105c1e8e9729fbc8d5d1667f2f62393a58be8d85a7d9a5a6c8d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + +[[package]] +name = "actix-web-static-files" +version = "3.0.5" +source = "git+https://github.com/MarinPostma/actix-web-static-files.git?rev=6db8c3e#6db8c3e2940d61659581492b5e9c9b9062567613" +dependencies = [ + "actix-service", + "actix-web", + "change-detection", + "derive_more", + "futures", + "mime_guess", + "path-slash", ] [[package]] name = "addr2line" -version = "0.14.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" dependencies = [ "gimli", ] [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" @@ -307,15 +255,20 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.4.7" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom 0.2.3", + "once_cell", + "version_check", +] [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -326,14 +279,20 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] -name = "arc-swap" -version = "1.2.0" +name = "anyhow" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73" +checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61" + +[[package]] +name = "arc-swap" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" [[package]] name = "assert-json-diff" @@ -345,20 +304,35 @@ dependencies = [ ] [[package]] -name = "assert_matches" -version = "1.4.0" +name = "async-stream" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "695579f0f2520f3774bb40461e5adb066459d4e0af4d59d20175484fb8e9edf1" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] @@ -369,7 +343,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -378,38 +352,14 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "awc" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" -dependencies = [ - "actix-codec", - "actix-http", - "actix-rt", - "actix-service", - "base64 0.13.0", - "bytes 0.5.6", - "cfg-if 1.0.0", - "derive_more", - "futures-core", - "log", - "mime", - "percent-encoding", - "rand 0.7.3", - "rustls 0.18.1", - "serde", - "serde_json", - "serde_urlencoded", -] - [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "b7815ea54e4d821e791162e078acbebfd6d8c8939cd559c9335dceb1c8ca7282" dependencies = [ "addr2line", + "cc", "cfg-if 1.0.0", "libc", "miniz_oxide", @@ -423,12 +373,6 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -437,11 +381,10 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bincode" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "byteorder", "serde", ] @@ -451,15 +394,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.7.3" @@ -469,7 +403,7 @@ dependencies = [ "block-padding", "byte-tools", "byteorder", - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -512,9 +446,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" dependencies = [ "lazy_static", "memchr", @@ -524,9 +458,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.4.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byte-tools" @@ -535,16 +469,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] -name = "byteorder" -version = "1.4.2" +name = "byte-unit" +version = "4.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "063197e6eb4b775b64160dedde7a0986bb2836cce140e9492e9e96f28e18bcd8" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "bytemuck" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966d2ab714d0f785dbac0a0396251a35280aeb42413281617d0209ab4898435" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "0.5.6" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" [[package]] name = "bytes" @@ -562,19 +511,45 @@ dependencies = [ ] [[package]] -name = "cast" -version = "0.2.3" +name = "bzip2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" dependencies = [ - "rustc_version", + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cargo_toml" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3596addfb02dcdc06f5252ddda9f3785f9230f5827fb4284645240fa05ad92" +dependencies = [ + "serde", + "serde_derive", + "toml", ] [[package]] name = "cc" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" +dependencies = [ + "jobserver", +] [[package]] name = "cedarwood" @@ -597,6 +572,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "change-detection" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "159fa412eae48a1d94d0b9ecdb85c97ce56eb2a347c62394d3fdbf221adabc1a" +dependencies = [ + "path-matchers", + "path-slash", +] + [[package]] name = "character_converter" version = "1.0.0" @@ -617,15 +602,9 @@ dependencies = [ "num-traits", "serde", "time 0.1.44", - "winapi 0.3.9", + "winapi", ] -[[package]] -name = "chunked_transfer" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" - [[package]] name = "clap" version = "2.33.3" @@ -642,34 +621,28 @@ dependencies = [ ] [[package]] -name = "compact_arena" -version = "0.4.1" +name = "const_fn" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5242c6ffe360608bbe43daef80990a7824c8b588e8db617f4e13054df3e6ef08" +checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" [[package]] -name = "const_fn" -version = "0.4.5" +name = "convert_case" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.14.3" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627" dependencies = [ "percent-encoding", - "time 0.2.24", + "time 0.2.27", "version_check", ] -[[package]] -name = "copyless" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" - [[package]] name = "cow-utils" version = "0.1.2" @@ -677,10 +650,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +dependencies = [ + "libc", +] [[package]] name = "crc32fast" @@ -691,50 +667,14 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "criterion" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8" -dependencies = [ - "atty", - "cast", - "clap", - "criterion-plot", - "csv", - "itertools 0.9.0", - "lazy_static", - "num-traits", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_cbor", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" -dependencies = [ - "cast", - "itertools 0.9.0", -] - [[package]] name = "crossbeam-channel" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", ] [[package]] @@ -745,18 +685,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", "lazy_static", "memoffset", "scopeguard", @@ -783,20 +722,19 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg", "cfg-if 1.0.0", "lazy_static", ] [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -826,20 +764,27 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.11" +version = "0.99.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" dependencies = [ - "proc-macro2", - "quote", - "syn", + "convert_case", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "deunicode" -version = "1.1.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" +checksum = "7f37775d639f64aa16389eede0cbe6a70f56df4609d50d8b6858690d5d7bf8f2" + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" [[package]] name = "digest" @@ -847,7 +792,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array 0.12.3", + "generic-array 0.12.4", ] [[package]] @@ -865,6 +810,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + [[package]] name = "either" version = "1.6.1" @@ -873,73 +824,26 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "enum-as-inner" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "env_logger" -version = "0.7.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ "atty", - "humantime 1.3.0", + "humantime", "log", "regex", "termcolor", ] -[[package]] -name = "env_logger" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" -dependencies = [ - "atty", - "humantime 2.0.1", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -948,21 +852,21 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", - "winapi 0.3.9", + "redox_syscall", + "winapi", ] [[package]] name = "flate2" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -970,6 +874,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -978,23 +891,19 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", ] [[package]] -name = "fs2" -version = "0.4.3" +name = "fragile" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi 0.3.9", -] +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" [[package]] name = "fs_extra" @@ -1004,9 +913,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "fst" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" [[package]] name = "fuchsia-cprng" @@ -1014,27 +923,11 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309f13e3f4be6d5917178c84db67c0b9a09177ac16d4f9a7313a767a68adaa77" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -1047,9 +940,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3b03bd32f6ec7885edeb99acd1e47e20e34fd4dfd3c6deed6fcac8a9d28f6a" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -1057,15 +950,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed8aeae2b6ab243ebabe6f54cd4cf53054d98883d5d326128af7d57a9ca5cd3d" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-executor" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f7836b36b7533d16fd5937311d98ba8965ab81030de8b0024c299dd5d51fb9b" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -1074,43 +967,42 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41234e71d5e8ca73d01563974ef6f50e516d71e18f1a2f1184742e31f5d469f" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-macro" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3520e0eb4e704e88d771b92d51273ee212997f0d8282f17f5d8ff1cb39104e42" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "futures-sink" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c72d188479368953c6c8c7140e40d7a4401674ab3b98a41e60e515d6cbdbe5de" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08944cea9021170d383287169859c0ca8147d9ec285978393109954448f33cc7" -dependencies = [ - "once_cell", -] +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-util" -version = "0.3.10" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd206efbe2ca683b2ce138ccdf61e1b0a63f5816dcedc9d8654c500ba0cea6" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures-channel", "futures-core", "futures-io", @@ -1118,7 +1010,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.4", + "pin-project-lite", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1136,9 +1028,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" dependencies = [ "typenum", ] @@ -1166,9 +1058,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1177,17 +1069,37 @@ dependencies = [ [[package]] name = "gimli" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "grenad" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe" +dependencies = [ + "byteorder", + "flate2", + "log", + "nix", + "snap", + "tempfile", + "zstd", +] [[package]] name = "h2" -version = "0.2.7" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "futures-core", "futures-sink", @@ -1198,15 +1110,8 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "tracing-futures", ] -[[package]] -name = "half" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" - [[package]] name = "hashbrown" version = "0.7.2" @@ -1222,25 +1127,26 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" -dependencies = [ - "ahash 0.4.7", - "serde", -] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "heck" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] [[package]] name = "heed" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" +version = "0.12.0" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" dependencies = [ "byteorder", "heed-traits", @@ -1258,14 +1164,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" [[package]] name = "heed-types" version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" dependencies = [ "bincode", "heed-traits", @@ -1276,13 +1180,19 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hostname" version = "0.3.1" @@ -1291,14 +1201,14 @@ checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ "libc", "match_cfg", - "winapi 0.3.9", + "winapi", ] [[package]] name = "http" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes 1.0.1", "fnv", @@ -1307,19 +1217,20 @@ dependencies = [ [[package]] name = "http-body" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "http", + "pin-project-lite", ] [[package]] name = "httparse" -version = "1.3.4" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" @@ -1328,27 +1239,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] -name = "humantime" -version = "1.3.0" +name = "httpdate" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] +name = "human_format" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.9" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", @@ -1356,9 +1270,9 @@ dependencies = [ "http", "http-body", "httparse", - "httpdate", + "httpdate 1.0.1", "itoa", - "pin-project 1.0.4", + "pin-project-lite", "socket2", "tokio", "tower-service", @@ -1368,15 +1282,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.21.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ - "bytes 0.5.6", "futures-util", "hyper", "log", - "rustls 0.18.1", + "rustls", "tokio", "tokio-rustls", "webpki", @@ -1384,34 +1297,20 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", "unicode-normalization", ] -[[package]] -name = "im" -version = "14.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696059c87b83c5a258817ecd67c3af915e3ed141891fc35a1e79908801cf0ce7" -dependencies = [ - "bitmaps", - "rand_core 0.5.1", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown 0.9.1", @@ -1427,41 +1326,11 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "intervaltree" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566d5aa3b5cc5c5809cc1a9c9588d917a634248bfc58f7ea14e354e71595a32c" -dependencies = [ - "smallvec", -] - -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - -[[package]] -name = "ipconfig" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" -dependencies = [ - "socket2", - "widestring", - "winapi 0.3.9", - "winreg 0.6.2", -] - [[package]] name = "ipnet" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" [[package]] name = "itertools" @@ -1474,9 +1343,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" dependencies = [ "either", ] @@ -1510,13 +1379,13 @@ dependencies = [ [[package]] name = "jieba-rs" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fbdeee8786790f4a99fa30ff5c5f88aa5183f7583693e3788d17fc8a48f33a" +checksum = "fea3b3172a80f9958abc3b9a637e4e311cd696dc6813440e5cc929b8a5311055" dependencies = [ "cedarwood", "fxhash", - "hashbrown 0.9.1", + "hashbrown 0.11.2", "lazy_static", "phf", "phf_codegen", @@ -1524,22 +1393,21 @@ dependencies = [ ] [[package]] -name = "js-sys" -version = "0.3.46" +name = "jobserver" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" dependencies = [ - "wasm-bindgen", + "libc", ] [[package]] -name = "kernel32-sys" -version = "0.2.2" +name = "js-sys" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "wasm-bindgen", ] [[package]] @@ -1556,18 +1424,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "levenshtein_automata" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" dependencies = [ "fst", ] [[package]] name = "libc" -version = "0.2.82" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" +checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" [[package]] name = "linked-hash-map" @@ -1577,9 +1445,8 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lmdb-rkv-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5" +version = "0.15.0" +source = "git+https://github.com/meilisearch/lmdb-rs#d0b50d02938ee84e4e4372697ea991fe2a4cae3b" dependencies = [ "cc", "libc", @@ -1587,30 +1454,61 @@ dependencies = [ ] [[package]] -name = "lock_api" -version = "0.4.2" +name = "local-channel" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" + +[[package]] +name = "lock_api" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] -name = "lru-cache" -version = "0.1.2" +name = "logging_timer" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" dependencies = [ - "linked-hash-map", + "log", + "logging_timer_proc_macros", +] + +[[package]] +name = "logging_timer_proc_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" +dependencies = [ + "log", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", ] [[package]] @@ -1637,94 +1535,71 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "meilisearch-core" -version = "0.20.0" -dependencies = [ - "arc-swap", - "assert_matches", - "bincode", - "byteorder", - "chrono", - "compact_arena", - "cow-utils", - "criterion", - "crossbeam-channel", - "csv", - "deunicode", - "either", - "env_logger 0.8.2", - "fst", - "hashbrown 0.9.1", - "heed", - "indexmap", - "intervaltree", - "itertools 0.10.0", - "jemallocator", - "levenshtein_automata", - "log", - "meilisearch-error", - "meilisearch-schema", - "meilisearch-tokenizer", - "meilisearch-types", - "once_cell", - "ordered-float", - "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", - "pest_derive", - "regex", - "rustyline", - "sdset", - "serde", - "serde_json", - "slice-group-by", - "structopt", - "tempfile", - "termcolor", - "unicase", - "zerocopy", -] - [[package]] name = "meilisearch-error" -version = "0.20.0" +version = "0.21.0" dependencies = [ "actix-http", ] [[package]] name = "meilisearch-http" -version = "0.20.0" +version = "0.21.0" dependencies = [ "actix-cors", "actix-http", "actix-rt", "actix-service", "actix-web", + "actix-web-static-files", + "anyhow", + "arc-swap", "assert-json-diff", - "bytes 1.0.1", + "async-stream", + "async-trait", + "byte-unit", + "bytes 0.6.0", + "cargo_toml", "chrono", "crossbeam-channel", - "env_logger 0.8.2", + "either", + "env_logger", "flate2", + "fst", "futures", + "futures-util", + "grenad", + "heed", + "hex", "http", "indexmap", + "itertools 0.10.1", "jemallocator", "log", "main_error", - "meilisearch-core", "meilisearch-error", - "meilisearch-schema", + "meilisearch-tokenizer 0.2.3", + "memmap", + "milli", "mime", + "mockall", + "num_cpus", + "obkv", "once_cell", - "rand 0.8.2", + "oxidized-json-checker", + "parking_lot", + "paste", + "pin-project", + "rand 0.7.3", + "rayon", "regex", - "rustls 0.18.1", + "reqwest", + "rustls", "sentry", "serde", "serde_json", - "serde_qs", "serde_url_params", + "sha-1 0.9.6", "sha2", "siphasher", "slice-group-by", @@ -1732,29 +1607,20 @@ dependencies = [ "tar", "tempdir", "tempfile", + "thiserror", "tokio", - "ureq", + "urlencoding", "uuid", "vergen", "walkdir", "whoami", -] - -[[package]] -name = "meilisearch-schema" -version = "0.20.0" -dependencies = [ - "indexmap", - "meilisearch-error", - "serde", - "serde_json", - "zerocopy", + "zip", ] [[package]] name = "meilisearch-tokenizer" -version = "0.1.1" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.1.3#d3fe5311a66c1f31682a297df8a8b6b8916f4252" +version = "0.2.2" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f" dependencies = [ "character_converter", "cow-utils", @@ -1768,28 +1634,87 @@ dependencies = [ ] [[package]] -name = "meilisearch-types" -version = "0.20.0" +name = "meilisearch-tokenizer" +version = "0.2.3" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" dependencies = [ - "serde", - "zerocopy", + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", ] [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "memoffset" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] +[[package]] +name = "milli" +version = "0.7.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.0#9dbc8b2dd06e12f0ee551d877261223fa4968110" +dependencies = [ + "bstr", + "byteorder", + "chrono", + "csv", + "either", + "flate2", + "fst", + "fxhash", + "grenad", + "heed", + "human_format", + "itertools 0.10.1", + "levenshtein_automata", + "linked-hash-map", + "log", + "logging_timer", + "meilisearch-tokenizer 0.2.2", + "memmap", + "obkv", + "once_cell", + "ordered-float", + "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", + "pest_derive", + "rayon", + "regex", + "roaring", + "serde", + "serde_json", + "slice-group-by", + "smallstr", + "smallvec", + "tempfile", + "tinytemplate", + "uuid", +] + [[package]] name = "mime" version = "0.3.16" @@ -1808,9 +1733,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", @@ -1818,55 +1743,51 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.23" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", "libc", "log", "miow", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio", + "ntapi", + "winapi", ] [[package]] name = "miow" -version = "0.2.2" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "winapi", ] [[package]] -name = "net2" -version = "0.2.37" +name = "mockall" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +checksum = "18d614ad23f9bb59119b8b5670a85c7ba92c5e9adf4385c81ea00c51c8be33d5" dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd4234635bca06fc96c7368d038061e0aae1b00a764dc817e900dc974e3deea" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] @@ -1881,6 +1802,21 @@ dependencies = [ "libc", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1912,21 +1848,24 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a38f2be3697a57b4060074ff41b44c16870d916ad7877c17696e063257482bc7" +dependencies = [ + "memchr", +] + +[[package]] +name = "obkv" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" [[package]] name = "once_cell" -version = "1.5.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" - -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" [[package]] name = "opaque-debug" @@ -1942,14 +1881,19 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "ordered-float" -version = "2.0.1" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacdec97876ef3ede8c50efc429220641a0b11ba0048b4b0c357bccbc47c5204" +checksum = "f100fcfb41e5385e0991f74981732049f9b896821542a219420491046baafdc2" dependencies = [ "num-traits", - "serde", ] +[[package]] +name = "oxidized-json-checker" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" + [[package]] name = "page_size" version = "0.4.2" @@ -1957,7 +1901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1973,18 +1917,39 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", - "winapi 0.3.9", + "winapi", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + +[[package]] +name = "path-matchers" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36cd9b72a47679ec193a5f0229d9ab686b7bd45e1fbc59ccf953c9f3d83f7b2b" +dependencies = [ + "glob", +] + +[[package]] +name = "path-slash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cacbb3c4ff353b534a67fb8d7524d00229da4cb1dc8c79f4db96e375ab5b619" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2026,9 +1991,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] @@ -2082,55 +2047,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.27" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ - "pin-project-internal 0.4.27", -] - -[[package]] -name = "pin-project" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" -dependencies = [ - "pin-project-internal 1.0.4", + "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.27" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "pin-project-lite" -version = "0.1.11" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" - -[[package]] -name = "pin-project-lite" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2144,24 +2083,41 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" -[[package]] -name = "plotters" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" -dependencies = [ - "js-sys", - "num-traits", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "ppv-lite86" version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "predicates" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2169,9 +2125,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", "version_check", ] @@ -2181,8 +2137,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.27", + "quote 1.0.9", "version_check", ] @@ -2194,32 +2150,44 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" -version = "1.0.24" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" dependencies = [ - "unicode-xid", + "unicode-xid 0.1.0", ] [[package]] -name = "quick-error" -version = "1.2.3" +name = "proc-macro2" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +dependencies = [ + "unicode-xid 0.2.2", +] [[package]] name = "quote" -version = "1.0.8" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" dependencies = [ - "proc-macro2", + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2 1.0.27", ] [[package]] @@ -2232,7 +2200,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2251,14 +2219,14 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", - "rand_chacha 0.3.0", - "rand_core 0.6.1", - "rand_hc 0.3.0", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", ] [[package]] @@ -2273,12 +2241,12 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.1", + "rand_core 0.6.3", ] [[package]] @@ -2307,11 +2275,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.1", + "getrandom 0.2.3", ] [[package]] @@ -2325,11 +2293,11 @@ dependencies = [ [[package]] name = "rand_hc" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core 0.6.1", + "rand_core 0.6.3", ] [[package]] @@ -2341,20 +2309,11 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_xoshiro" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ "autocfg", "crossbeam-deque", @@ -2364,13 +2323,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", "lazy_static", "num_cpus", ] @@ -2386,45 +2345,35 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_syscall" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-automata" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", -] +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -2432,17 +2381,17 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] name = "reqwest" -version = "0.10.10" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", + "base64", + "bytes 1.0.1", "encoding_rs", "futures-core", "futures-util", @@ -2455,10 +2404,9 @@ dependencies = [ "lazy_static", "log", "mime", - "mime_guess", "percent-encoding", - "pin-project-lite 0.2.4", - "rustls 0.18.1", + "pin-project-lite", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -2468,25 +2416,21 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.20.0", - "winreg 0.7.0", + "webpki-roots", + "winreg", ] [[package]] -name = "resolv-conf" -version = "0.7.0" +name = "retain_mut" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", -] +checksum = "e9c17925a9027d298a4603d286befe3f9dc0e8ed02523141914eb628798d6e5b" [[package]] name = "ring" -version = "0.16.19" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", @@ -2494,14 +2438,25 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", +] + +[[package]] +name = "roaring" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536cfa885fc388b8ae69edf96d7970849b7d9c1395da1b8330f17715babf8a09" +dependencies = [ + "bytemuck", + "byteorder", + "retain_mut", ] [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" [[package]] name = "rustc_version" @@ -2509,55 +2464,40 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.3", ] [[package]] name = "rustls" -version = "0.18.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64 0.12.3", + "base64", "log", "ring", "sct", "webpki", ] -[[package]] -name = "rustls" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustyline" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8227301bfc717136f0ecbd3d064ba8199e44497a0bdd46bb01ede4387cfd2cec" -dependencies = [ - "bitflags", - "cfg-if 1.0.0", - "fs2", - "libc", - "log", - "memchr", - "nix", - "scopeguard", - "unicode-segmentation", - "unicode-width", - "utf8parse", - "winapi 0.3.9", -] - [[package]] name = "ryu" version = "1.0.5" @@ -2581,29 +2521,38 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "sct" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ "ring", "untrusted", ] -[[package]] -name = "sdset" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb21fe0588557792176c89bc7b943027b14f346d03c6be6a199c2860277d93a" - [[package]] name = "semver" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", ] +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", +] + +[[package]] +name = "semver" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" + [[package]] name = "semver-parser" version = "0.7.0" @@ -2611,79 +2560,129 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] -name = "sentry" -version = "0.18.1" +name = "semver-parser" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sentry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27c425b07c7186018e2ef9ac3a25b01dae78b05a7ef604d07f216b9f59b42b4" +dependencies = [ + "httpdate 0.3.2", + "reqwest", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-log", + "sentry-panic", +] + +[[package]] +name = "sentry-backtrace" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80a5b9d9be0a0e25b2aaa5f3e9815d7fc6b8904f800c41800e5583652b5ca733" dependencies = [ "backtrace", - "env_logger 0.7.1", - "failure", + "lazy_static", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2410b212de9b2eb7427d2bf9a1f4f5cb2aa14359863d982066ead00d6de9bce0" +dependencies = [ "hostname", - "httpdate", - "im", "lazy_static", "libc", - "log", - "rand 0.7.3", "regex", - "reqwest", - "rustc_version", - "sentry-types", + "rustc_version 0.3.3", + "sentry-core", "uname", - "url", +] + +[[package]] +name = "sentry-core" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbbe485e384cb5540940e65d729820ffcbedc0c902fcb27081e44dacfe6a0c34" +dependencies = [ + "lazy_static", + "rand 0.8.4", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-log" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647143f672410ae5f242acd40f9f8f39729aff5ac7e856d91450fdfc30c2e960" +dependencies = [ + "log", + "sentry-core", +] + +[[package]] +name = "sentry-panic" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cf195cff04a50b90e6b9ac8b4874dc63b8e0a466f193702801ef98baa9bd90" +dependencies = [ + "sentry-backtrace", + "sentry-core", ] [[package]] name = "sentry-types" -version = "0.14.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" +checksum = "bc5e777cff85b44538ac766a9604676b7180d01d2566e76b2ac41426c734498c" dependencies = [ "chrono", "debugid", - "failure", "serde", "serde_json", + "thiserror", "url", "uuid", ] [[package]] name = "serde" -version = "1.0.119" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bdd36f49e35b61d49efd8aa7fc068fd295961fd2286d0b2ee9a4c7a14e99cc3" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" -dependencies = [ - "half", - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.119" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552954ce79a059ddd5fd68c271592374bd15cab2274970380c000118aeffe1cd" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "serde_json" -version = "1.0.61" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -2691,22 +2690,11 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5cb0f0564a84554436c4ceff5c896308d4e09d0eb4bd0215b8f698f88084601" -dependencies = [ - "percent-encoding", - "serde", - "thiserror", -] - [[package]] name = "serde_url_params" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24680ccd1ad7cdee9e8affa70f37d081b3d14d3800d33a28f474d0f7a55f305" +checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" dependencies = [ "serde", "url", @@ -2738,13 +2726,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", ] @@ -2757,47 +2745,37 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", ] [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] [[package]] name = "siphasher" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" - -[[package]] -name = "sized-chunks" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59044ea371ad781ff976f7b06480b9f0180e834eda94114f2afb4afc12b7718" -dependencies = [ - "bitmaps", - "typenum", -] +checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "slice-group-by" @@ -2805,6 +2783,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f7474f0b646d228360ab62ed974744617bc869d959eac8403bfa3665931a7fb" +[[package]] +name = "smallstr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e922794d168678729ffc7e07182721a14219c65814e66e91b839a272fe5ae4f" +dependencies = [ + "serde", + "smallvec", +] + [[package]] name = "smallvec" version = "1.6.1" @@ -2812,14 +2800,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] -name = "socket2" -version = "0.3.19" +name = "snap" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" + +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ - "cfg-if 1.0.0", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2830,9 +2823,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66a8cff4fa24853fdf6b51f75c6d7f8206d7c75cab4e467bcd7f25c2b1febe0" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" dependencies = [ "version_check", ] @@ -2844,7 +2837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -2857,11 +2850,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.27", + "quote 1.0.9", "serde", "serde_derive", - "syn", + "syn 1.0.73", ] [[package]] @@ -2871,13 +2864,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.27", + "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.73", ] [[package]] @@ -2911,20 +2904,31 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "syn" -version = "1.0.58" +version = "0.15.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "unicode-xid 0.2.2", ] [[package]] @@ -2942,21 +2946,20 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", + "unicode-xid 0.2.2", ] [[package]] name = "tar" -version = "0.4.30" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489997b7557e9a43e192c527face4feacc78bfbe6eed67fd55c4c9e381cba290" +checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -2978,10 +2981,10 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.2", - "redox_syscall 0.2.4", + "rand 0.8.4", + "redox_syscall", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3004,40 +3007,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] @@ -3048,14 +3033,14 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", + "winapi", ] [[package]] name = "time" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273d3ed44dca264b0d6b3665e8d48fb515042d42466fad93d2a45b90ec4058f7" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" dependencies = [ "const_fn", "libc", @@ -3063,7 +3048,7 @@ dependencies = [ "stdweb", "time-macros", "version_check", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3078,22 +3063,22 @@ dependencies = [ [[package]] name = "time-macros-impl" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.27", + "quote 1.0.9", "standback", - "syn", + "syn 1.0.73", ] [[package]] name = "tinytemplate" -version = "1.2.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ada8616fad06a2d0c455adc530de4ef57605a8120cc65da9653e0e9623ca74" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" dependencies = [ "serde", "serde_json", @@ -3101,9 +3086,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" dependencies = [ "tinyvec_macros", ] @@ -3116,140 +3101,100 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.24" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2" dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "iovec", - "lazy_static", + "autocfg", + "bytes 1.0.1", "libc", "memchr", "mio", - "mio-uds", "num_cpus", - "pin-project-lite 0.1.11", + "once_cell", + "parking_lot", + "pin-project-lite", "signal-hook-registry", - "slab", "tokio-macros", - "winapi 0.3.9", + "winapi", ] [[package]] name = "tokio-macros" -version = "0.2.6" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", ] [[package]] name = "tokio-rustls" -version = "0.14.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "futures-core", - "rustls 0.18.1", + "rustls", "tokio", "webpki", ] [[package]] name = "tokio-util" -version = "0.3.1" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures-core", "futures-sink", "log", - "pin-project-lite 0.1.11", + "pin-project-lite", "tokio", ] [[package]] -name = "tower-service" -version = "0.3.0" +name = "toml" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", - "log", - "pin-project-lite 0.2.4", + "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] [[package]] -name = "tracing-futures" -version = "0.2.4" +name = "treeline" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" -dependencies = [ - "pin-project 0.4.27", - "tracing", -] - -[[package]] -name = "trust-dns-proto" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" -dependencies = [ - "async-trait", - "backtrace", - "enum-as-inner", - "futures", - "idna", - "lazy_static", - "log", - "rand 0.7.3", - "smallvec", - "thiserror", - "tokio", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" -dependencies = [ - "backtrace", - "cfg-if 0.1.10", - "futures", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "resolv-conf", - "smallvec", - "thiserror", - "tokio", - "trust-dns-proto", -] +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "try-lock" @@ -3259,9 +3204,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "ucd-trie" @@ -3289,18 +3234,18 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" dependencies = [ "matches", ] [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" dependencies = [ "tinyvec", ] @@ -3319,9 +3264,15 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "untrusted" @@ -3329,27 +3280,11 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc" -dependencies = [ - "base64 0.13.0", - "chunked_transfer", - "log", - "once_cell", - "rustls 0.19.0", - "url", - "webpki", - "webpki-roots 0.21.0", -] - [[package]] name = "url" -version = "2.2.0" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", @@ -3359,10 +3294,16 @@ dependencies = [ ] [[package]] -name = "utf8parse" -version = "0.2.0" +name = "urlencoding" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" + +[[package]] +name = "utf8-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b" [[package]] name = "uuid" @@ -3370,7 +3311,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.1", + "getrandom 0.2.3", "serde", ] @@ -3382,28 +3323,29 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" +checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", + "rustc_version 0.4.0", ] [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "walkdir" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", - "winapi 0.3.9", + "winapi", "winapi-util", ] @@ -3431,9 +3373,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3443,24 +3385,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3470,38 +3412,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ - "quote", + "quote 1.0.9", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -3519,18 +3461,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" -dependencies = [ - "webpki", -] - -[[package]] -name = "webpki-roots" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" dependencies = [ "webpki", ] @@ -3546,26 +3479,14 @@ dependencies = [ [[package]] name = "whoami" -version = "1.0.3" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d595b2e146f36183d6a590b8d41568e2bc84c922267f43baf61c956330eeb436" +checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" dependencies = [ "wasm-bindgen", "web-sys", ] -[[package]] -name = "widestring" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -3576,12 +3497,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -3594,7 +3509,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3603,32 +3518,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "winreg" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "winreg" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] @@ -3656,7 +3552,52 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.27", + "syn 1.0.73", "synstructure", ] + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time 0.1.44", +] + +[[package]] +name = "zstd" +version = "0.5.4+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "2.0.6+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.4.18+zstd.1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +dependencies = [ + "cc", + "glob", + "itertools 0.9.0", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 913ab34c8..a1dca038e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,7 @@ [workspace] members = [ - "meilisearch-core", "meilisearch-http", - "meilisearch-schema", - "meilisearch-types", + "meilisearch-error", ] [profile.release] diff --git a/Dockerfile b/Dockerfile index 9898d02db..811be5c14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Compile -FROM alpine:3.10 AS compiler +FROM alpine:3.14 AS compiler RUN apk update --quiet RUN apk add curl @@ -9,14 +9,30 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y WORKDIR /meilisearch -COPY . . +COPY Cargo.lock . +COPY Cargo.toml . + +COPY meilisearch-error/Cargo.toml meilisearch-error/ +COPY meilisearch-http/Cargo.toml meilisearch-http/ ENV RUSTFLAGS="-C target-feature=-crt-static" +# Create dummy main.rs files for each workspace member to be able to compile all the dependencies +RUN find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;' +# Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too +RUN $HOME/.cargo/bin/cargo build --release +# Cleanup dummy main.rs files +RUN find . -path "*/src/main.rs" -delete + +ARG COMMIT_SHA +ARG COMMIT_DATE +ENV COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} + +COPY . . RUN $HOME/.cargo/bin/cargo build --release # Run -FROM alpine:3.10 +FROM alpine:3.14 RUN apk add -q --no-cache libgcc tini diff --git a/bors.toml b/bors.toml index 4a3bf8c38..e3348c36d 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,9 @@ -status = ["Test on macos-latest", "Test on ubuntu-18.04"] -# 4 hours timeout -timeout-sec = 14400 +status = [ + 'Tests on ubuntu-18.04', + 'Tests on macos-latest', + 'Cargo check on Windows', + 'Run Clippy', + 'Run Rustfmt' +] +# 3 hours timeout +timeout-sec = 10800 diff --git a/bump.sh b/bump.sh deleted file mode 100755 index 32797e830..000000000 --- a/bump.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/bash - -NEW_VERSION=$1 - -if [ -z "$NEW_VERSION" ] -then - echo "error: a version number must be provided" - exit 1 -fi - -# find current version -CURRENT_VERSION=$(cat **/*.toml | grep meilisearch | grep version | sed 's/.*\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/' | sed "1q;d") - -# bump all version in .toml -echo "bumping from version $CURRENT_VERSION to version $NEW_VERSION" -while true -do - read -r -p "Continue (y/n)?" choice - case "$choice" in - y|Y ) break;; - n|N ) echo "aborting bump" && exit 0;; - * ) echo "invalid choice";; - esac -done -# update all crate version -sed -i "s/version = \"$CURRENT_VERSION\"/version = \"$NEW_VERSION\"/" **/*.toml - -printf "running cargo check: " - -CARGO_CHECK=$(cargo check 2>&1) - -if [ $? != "0" ] -then - printf "\033[31;1m FAIL \033[0m\n" - printf "$CARGO_CHECK" - exit 1 -fi -printf "\033[32;1m OK \033[0m\n" diff --git a/meilisearch-core/Cargo.toml b/meilisearch-core/Cargo.toml deleted file mode 100644 index a60cac84d..000000000 --- a/meilisearch-core/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -[package] -name = "meilisearch-core" -version = "0.20.0" -license = "MIT" -authors = ["Kerollmops "] -edition = "2018" - -[dependencies] -arc-swap = "1.2.0" -bincode = "1.3.1" -byteorder = "1.3.4" -chrono = { version = "0.4.19", features = ["serde"] } -compact_arena = "0.4.1" -cow-utils = "0.1.2" -crossbeam-channel = "0.5.0" -deunicode = "1.1.1" -either = "1.6.1" -env_logger = "0.8.2" -fst = "0.4.5" -hashbrown = { version = "0.9.1", features = ["serde"] } -heed = "0.10.6" -indexmap = { version = "1.6.1", features = ["serde-1"] } -intervaltree = "0.2.6" -itertools = "0.10.0" -levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } -log = "0.4.11" -meilisearch-error = { path = "../meilisearch-error", version = "0.20.0" } -meilisearch-schema = { path = "../meilisearch-schema", version = "0.20.0" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.1.3" } -meilisearch-types = { path = "../meilisearch-types", version = "0.20.0" } -once_cell = "1.5.2" -ordered-float = { version = "2.0.1", features = ["serde"] } -pest = { git = "https://github.com/pest-parser/pest.git", rev = "51fd1d49f1041f7839975664ef71fe15c7dcaf67" } -pest_derive = "2.1.0" -regex = "1.4.2" -sdset = "0.4.0" -serde = { version = "1.0.118", features = ["derive"] } -serde_json = { version = "1.0.61", features = ["preserve_order"] } -slice-group-by = "0.2.6" -unicase = "2.6.0" -zerocopy = "0.3.0" - -[dev-dependencies] -assert_matches = "1.4.0" -criterion = "0.3.3" -csv = "1.1.5" -rustyline = { version = "7.1.0", default-features = false } -structopt = "0.3.21" -tempfile = "3.1.0" -termcolor = "1.1.2" - -[target.'cfg(unix)'.dev-dependencies] -jemallocator = "0.3.2" diff --git a/meilisearch-core/examples/from_file.rs b/meilisearch-core/examples/from_file.rs deleted file mode 100644 index beb1028e2..000000000 --- a/meilisearch-core/examples/from_file.rs +++ /dev/null @@ -1,473 +0,0 @@ -use std::collections::HashSet; -use std::collections::btree_map::{BTreeMap, Entry}; -use std::error::Error; -use std::io::{Read, Write}; -use std::iter::FromIterator; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; -use std::{fs, io, sync::mpsc}; - -use rustyline::{Config, Editor}; -use serde::{Deserialize, Serialize}; -use structopt::StructOpt; -use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; - -use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult}; -use meilisearch_core::settings::Settings; -use meilisearch_schema::FieldId; - -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[derive(Debug, StructOpt)] -struct IndexCommand { - /// The destination where the database must be created. - #[structopt(parse(from_os_str))] - database_path: PathBuf, - - #[structopt(long, default_value = "default")] - index_uid: String, - - /// The csv file path to index, you can also use `-` to specify the standard input. - #[structopt(parse(from_os_str))] - csv_data_path: PathBuf, - - /// The path to the settings. - #[structopt(long, parse(from_os_str))] - settings: PathBuf, - - #[structopt(long)] - update_group_size: Option, - - #[structopt(long, parse(from_os_str))] - compact_to_path: Option, -} - -#[derive(Debug, StructOpt)] -struct SearchCommand { - /// The path of the database to work with. - #[structopt(parse(from_os_str))] - database_path: PathBuf, - - #[structopt(long, default_value = "default")] - index_uid: String, - - /// Timeout after which the search will return results. - #[structopt(long)] - fetch_timeout_ms: Option, - - /// The number of returned results - #[structopt(short, long, default_value = "10")] - number_results: usize, - - /// The number of characters before and after the first match - #[structopt(short = "C", long, default_value = "35")] - char_context: usize, - - /// A filter string that can be `!adult` or `adult` to - /// filter documents on this specfied field - #[structopt(short, long)] - filter: Option, - - /// Fields that must be displayed. - displayed_fields: Vec, -} - -#[derive(Debug, StructOpt)] -struct ShowUpdatesCommand { - /// The path of the database to work with. - #[structopt(parse(from_os_str))] - database_path: PathBuf, - - #[structopt(long, default_value = "default")] - index_uid: String, -} - -#[derive(Debug, StructOpt)] -enum Command { - Index(IndexCommand), - Search(SearchCommand), - ShowUpdates(ShowUpdatesCommand), -} - -impl Command { - fn path(&self) -> &Path { - match self { - Command::Index(command) => &command.database_path, - Command::Search(command) => &command.database_path, - Command::ShowUpdates(command) => &command.database_path, - } - } -} - -#[derive(Serialize, Deserialize)] -#[serde(transparent)] -struct Document(indexmap::IndexMap); - -fn index_command(command: IndexCommand, database: Database) -> Result<(), Box> { - let start = Instant::now(); - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = - move |_name: &str, update: ProcessedUpdateResult| sender.send(update.update_id).unwrap(); - let index = match database.open_index(&command.index_uid) { - Some(index) => index, - None => database.create_index(&command.index_uid).unwrap(), - }; - - database.set_update_callback(Box::new(update_fn)); - - let db = &database; - - let settings = { - let string = fs::read_to_string(&command.settings)?; - let settings: Settings = serde_json::from_str(&string).unwrap(); - settings.to_update().unwrap() - }; - - db.update_write(|w| index.settings_update(w, settings))?; - - let mut rdr = if command.csv_data_path.as_os_str() == "-" { - csv::Reader::from_reader(Box::new(io::stdin()) as Box) - } else { - let file = std::fs::File::open(command.csv_data_path)?; - csv::Reader::from_reader(Box::new(file) as Box) - }; - - let mut raw_record = csv::StringRecord::new(); - let headers = rdr.headers()?.clone(); - - let mut max_update_id = 0; - let mut i = 0; - let mut end_of_file = false; - - while !end_of_file { - let mut additions = index.documents_addition(); - - loop { - end_of_file = !rdr.read_record(&mut raw_record)?; - if end_of_file { - break; - } - - let document: Document = match raw_record.deserialize(Some(&headers)) { - Ok(document) => document, - Err(e) => { - eprintln!("{:?}", e); - continue; - } - }; - - additions.update_document(document); - - print!("\rindexing document {}", i); - i += 1; - - if let Some(group_size) = command.update_group_size { - if i % group_size == 0 { - break; - } - } - } - - println!(); - - let update_id = db.update_write(|w| additions.finalize(w))?; - - println!("committing update..."); - max_update_id = max_update_id.max(update_id); - println!("committed update {}", update_id); - } - - println!("Waiting for update {}", max_update_id); - for id in receiver { - if id == max_update_id { - break; - } - } - - println!( - "database created in {:.2?} at: {:?}", - start.elapsed(), - command.database_path - ); - - if let Some(path) = command.compact_to_path { - fs::create_dir_all(&path)?; - let start = Instant::now(); - let _file = database.copy_and_compact_to_path(path.join("data.mdb"))?; - println!( - "database compacted in {:.2?} at: {:?}", - start.elapsed(), - path - ); - } - - Ok(()) -} - -fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> { - let mut stdout = StandardStream::stdout(ColorChoice::Always); - let mut highlighted = false; - - for range in ranges.windows(2) { - let [start, end] = match range { - [start, end] => [*start, *end], - _ => unreachable!(), - }; - if highlighted { - stdout.set_color( - ColorSpec::new() - .set_fg(Some(Color::Yellow)) - .set_underline(true), - )?; - } - write!(&mut stdout, "{}", &text[start..end])?; - stdout.reset()?; - highlighted = !highlighted; - } - - Ok(()) -} - -fn char_to_byte_range(index: usize, length: usize, text: &str) -> (usize, usize) { - let mut byte_index = 0; - let mut byte_length = 0; - - for (n, (i, c)) in text.char_indices().enumerate() { - if n == index { - byte_index = i; - } - - if n + 1 == index + length { - byte_length = i - byte_index + c.len_utf8(); - break; - } - } - - (byte_index, byte_length) -} - -fn create_highlight_areas(text: &str, highlights: &[Highlight]) -> Vec { - let mut byte_indexes = BTreeMap::new(); - - for highlight in highlights { - let char_index = highlight.char_index as usize; - let char_length = highlight.char_length as usize; - let (byte_index, byte_length) = char_to_byte_range(char_index, char_length, text); - - match byte_indexes.entry(byte_index) { - Entry::Vacant(entry) => { - entry.insert(byte_length); - } - Entry::Occupied(mut entry) => { - if *entry.get() < byte_length { - entry.insert(byte_length); - } - } - } - } - - let mut title_areas = Vec::new(); - title_areas.push(0); - for (byte_index, length) in byte_indexes { - title_areas.push(byte_index); - title_areas.push(byte_index + length); - } - title_areas.push(text.len()); - title_areas.sort_unstable(); - title_areas -} - -/// note: matches must have been sorted by `char_index` and `char_length` before being passed. -/// -/// ```no_run -/// matches.sort_unstable_by_key(|m| (m.char_index, m.char_length)); -/// -/// let matches = matches.matches.iter().filter(|m| SchemaAttr::new(m.attribute) == attr).cloned(); -/// -/// let (text, matches) = crop_text(&text, matches, 35); -/// ``` -fn crop_text( - text: &str, - highlights: impl IntoIterator, - context: usize, -) -> (String, Vec) { - let mut highlights = highlights.into_iter().peekable(); - - let char_index = highlights - .peek() - .map(|m| m.char_index as usize) - .unwrap_or(0); - let start = char_index.saturating_sub(context); - let text = text.chars().skip(start).take(context * 2).collect(); - - let highlights = highlights - .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + (context * 2)) - .map(|highlight| Highlight { - char_index: highlight.char_index - start as u16, - ..highlight - }) - .collect(); - - (text, highlights) -} - -fn search_command(command: SearchCommand, database: Database) -> Result<(), Box> { - let db = &database; - let index = database - .open_index(&command.index_uid) - .expect("Could not find index"); - - let reader = db.main_read_txn().unwrap(); - let schema = index.main.schema(&reader)?; - reader.abort().unwrap(); - - let schema = schema.ok_or(meilisearch_core::Error::SchemaMissing)?; - - let fields = command - .displayed_fields - .iter() - .map(String::as_str) - .collect::>(); - - let config = Config::builder().auto_add_history(true).build(); - let mut readline = Editor::<()>::with_config(config); - let _ = readline.load_history("query-history.txt"); - - for result in readline.iter("Searching for: ") { - match result { - Ok(query) => { - let start_total = Instant::now(); - - let reader = db.main_read_txn().unwrap(); - let ref_index = &index; - let ref_reader = &reader; - - let mut builder = index.query_builder(); - if let Some(timeout) = command.fetch_timeout_ms { - builder.with_fetch_timeout(Duration::from_millis(timeout)); - } - - if let Some(ref filter) = command.filter { - let filter = filter.as_str(); - let (positive, filter) = if let Some(stripped) = filter.strip_prefix('!') { - (false, stripped) - } else { - (true, filter) - }; - - let attr = schema - .id(filter) - .expect("Could not find filtered attribute"); - - builder.with_filter(move |document_id| { - let string: String = ref_index - .document_attribute(ref_reader, document_id, attr) - .unwrap() - .unwrap(); - (string == "true") == positive - }); - } - - let result = builder.query(ref_reader, Some(&query), 0..command.number_results)?; - - let mut retrieve_duration = Duration::default(); - - let number_of_documents = result.documents.len(); - for mut doc in result.documents { - doc.highlights - .sort_unstable_by_key(|m| (m.char_index, m.char_length)); - - let start_retrieve = Instant::now(); - let result = index.document::(&reader, Some(&fields), doc.id); - retrieve_duration += start_retrieve.elapsed(); - - match result { - Ok(Some(document)) => { - println!("raw-id: {:?}", doc.id); - for (name, text) in document.0 { - print!("{}: ", name); - - let attr = schema.id(&name).unwrap(); - let highlights = doc - .highlights - .iter() - .filter(|m| FieldId::new(m.attribute) == attr) - .cloned(); - let (text, highlights) = - crop_text(&text, highlights, command.char_context); - let areas = create_highlight_areas(&text, &highlights); - display_highlights(&text, &areas)?; - println!(); - } - } - Ok(None) => eprintln!("missing document"), - Err(e) => eprintln!("{}", e), - } - - let mut matching_attributes = HashSet::new(); - for highlight in doc.highlights { - let attr = FieldId::new(highlight.attribute); - let name = schema.name(attr); - matching_attributes.insert(name); - } - - let matching_attributes = Vec::from_iter(matching_attributes); - println!("matching in: {:?}", matching_attributes); - - println!(); - } - - eprintln!( - "whole documents fields retrieve took {:.2?}", - retrieve_duration - ); - eprintln!( - "===== Found {} results in {:.2?} =====", - number_of_documents, - start_total.elapsed() - ); - } - Err(err) => { - println!("Error: {:?}", err); - break; - } - } - } - - readline.save_history("query-history.txt").unwrap(); - - Ok(()) -} - -fn show_updates_command( - command: ShowUpdatesCommand, - database: Database, -) -> Result<(), Box> { - let db = &database; - let index = database - .open_index(&command.index_uid) - .expect("Could not find index"); - - let reader = db.update_read_txn().unwrap(); - let updates = index.all_updates_status(&reader)?; - println!("{:#?}", updates); - reader.abort().unwrap(); - - Ok(()) -} - -fn main() -> Result<(), Box> { - env_logger::init(); - - let opt = Command::from_args(); - let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?; - - match opt { - Command::Index(command) => index_command(command, database), - Command::Search(command) => search_command(command, database), - Command::ShowUpdates(command) => show_updates_command(command, database), - } -} diff --git a/meilisearch-core/src/automaton/dfa.rs b/meilisearch-core/src/automaton/dfa.rs deleted file mode 100644 index da1a6eb39..000000000 --- a/meilisearch-core/src/automaton/dfa.rs +++ /dev/null @@ -1,53 +0,0 @@ -use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA}; -use once_cell::sync::OnceCell; - -static LEVDIST0: OnceCell = OnceCell::new(); -static LEVDIST1: OnceCell = OnceCell::new(); -static LEVDIST2: OnceCell = OnceCell::new(); - -#[derive(Copy, Clone)] -enum PrefixSetting { - Prefix, - NoPrefix, -} - -fn build_dfa_with_setting(query: &str, setting: PrefixSetting) -> DFA { - use PrefixSetting::{NoPrefix, Prefix}; - - match query.len() { - 0..=4 => { - let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true)); - match setting { - Prefix => builder.build_prefix_dfa(query), - NoPrefix => builder.build_dfa(query), - } - } - 5..=8 => { - let builder = LEVDIST1.get_or_init(|| LevBuilder::new(1, true)); - match setting { - Prefix => builder.build_prefix_dfa(query), - NoPrefix => builder.build_dfa(query), - } - } - _ => { - let builder = LEVDIST2.get_or_init(|| LevBuilder::new(2, true)); - match setting { - Prefix => builder.build_prefix_dfa(query), - NoPrefix => builder.build_dfa(query), - } - } - } -} - -pub fn build_prefix_dfa(query: &str) -> DFA { - build_dfa_with_setting(query, PrefixSetting::Prefix) -} - -pub fn build_dfa(query: &str) -> DFA { - build_dfa_with_setting(query, PrefixSetting::NoPrefix) -} - -pub fn build_exact_dfa(query: &str) -> DFA { - let builder = LEVDIST0.get_or_init(|| LevBuilder::new(0, true)); - builder.build_dfa(query) -} diff --git a/meilisearch-core/src/automaton/mod.rs b/meilisearch-core/src/automaton/mod.rs deleted file mode 100644 index f31d0f0a5..000000000 --- a/meilisearch-core/src/automaton/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod dfa; - -pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa}; - diff --git a/meilisearch-core/src/bucket_sort.rs b/meilisearch-core/src/bucket_sort.rs deleted file mode 100644 index 57e50b87f..000000000 --- a/meilisearch-core/src/bucket_sort.rs +++ /dev/null @@ -1,679 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::mem; -use std::ops::Deref; -use std::ops::Range; -use std::rc::Rc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Instant; -use std::fmt; - -use compact_arena::{SmallArena, Idx32, mk_arena}; -use log::{debug, error}; -use sdset::{Set, SetBuf, exponential_search, SetOperation, Counter, duo::OpBuilder}; -use slice_group_by::{GroupBy, GroupByMut}; - -use meilisearch_types::DocIndex; - -use crate::criterion::{Criteria, Context, ContextMut}; -use crate::distinct_map::{BufferedDistinctMap, DistinctMap}; -use crate::raw_document::RawDocument; -use crate::{database::MainT, reordered_attrs::ReorderedAttrs}; -use crate::{store, Document, DocumentId, MResult, Index, RankedMap, MainReader, Error}; -use crate::query_tree::{create_query_tree, traverse_query_tree}; -use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey}; -use crate::query_tree::Context as QTContext; - -#[derive(Debug, Default)] -pub struct SortResult { - pub documents: Vec, - pub nb_hits: usize, - pub exhaustive_nb_hit: bool, - pub facets: Option>>, - pub exhaustive_facets_count: Option, -} - -#[allow(clippy::too_many_arguments)] -pub fn bucket_sort<'c, FI>( - reader: &heed::RoTxn, - query: &str, - range: Range, - facets_docids: Option>, - facet_count_docids: Option>)>>>, - filter: Option, - criteria: Criteria<'c>, - searchable_attrs: Option, - index: &Index, -) -> MResult -where - FI: Fn(DocumentId) -> bool, -{ - // We delegate the filter work to the distinct query builder, - // specifying a distinct rule that has no effect. - if filter.is_some() { - let distinct = |_| None; - let distinct_size = 1; - return bucket_sort_with_distinct( - reader, - query, - range, - facets_docids, - facet_count_docids, - filter, - distinct, - distinct_size, - criteria, - searchable_attrs, - index, - ); - } - - let mut result = SortResult::default(); - - let words_set = index.main.words_fst(reader)?; - let stop_words = index.main.stop_words_fst(reader)?; - - let context = QTContext { - words_set, - stop_words, - synonyms: index.synonyms, - postings_lists: index.postings_lists, - prefix_postings_lists: index.prefix_postings_lists_cache, - }; - - let (operation, mapping) = create_query_tree(reader, &context, query)?; - debug!("operation:\n{:?}", operation); - debug!("mapping:\n{:?}", mapping); - - fn recurs_operation<'o>(map: &mut HashMap, operation: &'o Operation) { - match operation { - Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)), - Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)), - Operation::Query(query) => { map.insert(query.id, &query.kind); }, - } - } - - let mut queries_kinds = HashMap::new(); - recurs_operation(&mut queries_kinds, &operation); - - let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?; - debug!("found {} documents", docids.len()); - debug!("number of postings {:?}", queries.len()); - - if let Some(facets_docids) = facets_docids { - let intersection = sdset::duo::OpBuilder::new(docids.as_ref(), facets_docids.as_set()) - .intersection() - .into_set_buf(); - docids = Cow::Owned(intersection); - } - - if let Some(f) = facet_count_docids { - // hardcoded value, until approximation optimization - result.exhaustive_facets_count = Some(true); - result.facets = Some(facet_count(f, &docids)); - } - - let before = Instant::now(); - mk_arena!(arena); - let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries); - debug!("matches cleaned in {:.02?}", before.elapsed()); - - let before_bucket_sort = Instant::now(); - - let before_raw_documents_building = Instant::now(); - let mut raw_documents = Vec::new(); - for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) { - let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref()); - raw_documents.push(raw_document); - } - debug!("creating {} candidates documents took {:.02?}", - raw_documents.len(), - before_raw_documents_building.elapsed(), - ); - - let before_criterion_loop = Instant::now(); - let proximity_count = AtomicUsize::new(0); - - let mut groups = vec![raw_documents.as_mut_slice()]; - - 'criteria: for criterion in criteria.as_ref() { - let tmp_groups = mem::replace(&mut groups, Vec::new()); - let mut documents_seen = 0; - - for mut group in tmp_groups { - let before_criterion_preparation = Instant::now(); - - let ctx = ContextMut { - reader, - postings_lists: &mut arena, - query_mapping: &mapping, - documents_fields_counts_store: index.documents_fields_counts, - }; - - criterion.prepare(ctx, &mut group)?; - debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed()); - - let ctx = Context { - postings_lists: &arena, - query_mapping: &mapping, - }; - - let before_criterion_sort = Instant::now(); - group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b)); - debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed()); - - for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) { - debug!("{:?} produced a group of size {}", criterion.name(), group.len()); - - documents_seen += group.len(); - groups.push(group); - - // we have sort enough documents if the last document sorted is after - // the end of the requested range, we can continue to the next criterion - if documents_seen >= range.end { - continue 'criteria; - } - } - } - } - - debug!("criterion loop took {:.02?}", before_criterion_loop.elapsed()); - debug!("proximity evaluation called {} times", proximity_count.load(Ordering::Relaxed)); - - let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?; - let iter = raw_documents.into_iter().skip(range.start).take(range.len()); - let iter = iter.map(|rd| Document::from_raw(rd, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema)); - let documents = iter.collect(); - - debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed()); - - result.documents = documents; - result.nb_hits = docids.len(); - - Ok(result) -} - -#[allow(clippy::too_many_arguments)] -pub fn bucket_sort_with_distinct<'c, FI, FD>( - reader: &heed::RoTxn, - query: &str, - range: Range, - facets_docids: Option>, - facet_count_docids: Option>)>>>, - filter: Option, - distinct: FD, - distinct_size: usize, - criteria: Criteria<'c>, - searchable_attrs: Option, - index: &Index, -) -> MResult -where - FI: Fn(DocumentId) -> bool, - FD: Fn(DocumentId) -> Option, -{ - let mut result = SortResult::default(); - let mut filtered_count = 0; - - let words_set = index.main.words_fst(reader)?; - let stop_words = index.main.stop_words_fst(reader)?; - - let context = QTContext { - words_set, - stop_words, - synonyms: index.synonyms, - postings_lists: index.postings_lists, - prefix_postings_lists: index.prefix_postings_lists_cache, - }; - - let (operation, mapping) = create_query_tree(reader, &context, query)?; - debug!("operation:\n{:?}", operation); - debug!("mapping:\n{:?}", mapping); - - fn recurs_operation<'o>(map: &mut HashMap, operation: &'o Operation) { - match operation { - Operation::And(ops) => ops.iter().for_each(|op| recurs_operation(map, op)), - Operation::Or(ops) => ops.iter().for_each(|op| recurs_operation(map, op)), - Operation::Query(query) => { map.insert(query.id, &query.kind); }, - } - } - - let mut queries_kinds = HashMap::new(); - recurs_operation(&mut queries_kinds, &operation); - - let QueryResult { mut docids, queries } = traverse_query_tree(reader, &context, &operation)?; - debug!("found {} documents", docids.len()); - debug!("number of postings {:?}", queries.len()); - - if let Some(facets_docids) = facets_docids { - let intersection = OpBuilder::new(docids.as_ref(), facets_docids.as_set()) - .intersection() - .into_set_buf(); - docids = Cow::Owned(intersection); - } - - if let Some(f) = facet_count_docids { - // hardcoded value, until approximation optimization - result.exhaustive_facets_count = Some(true); - result.facets = Some(facet_count(f, &docids)); - } - - let before = Instant::now(); - mk_arena!(arena); - let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries); - debug!("matches cleaned in {:.02?}", before.elapsed()); - - let before_raw_documents_building = Instant::now(); - let mut raw_documents = Vec::new(); - for bare_matches in bare_matches.linear_group_by_key_mut(|sm| sm.document_id) { - let raw_document = RawDocument::new(bare_matches, &mut arena, searchable_attrs.as_ref()); - raw_documents.push(raw_document); - } - debug!("creating {} candidates documents took {:.02?}", - raw_documents.len(), - before_raw_documents_building.elapsed(), - ); - - let mut groups = vec![raw_documents.as_mut_slice()]; - let mut key_cache = HashMap::new(); - - let mut filter_map = HashMap::new(); - // these two variables informs on the current distinct map and - // on the raw offset of the start of the group where the - // range.start bound is located according to the distinct function - let mut distinct_map = DistinctMap::new(distinct_size); - let mut distinct_raw_offset = 0; - - 'criteria: for criterion in criteria.as_ref() { - let tmp_groups = mem::replace(&mut groups, Vec::new()); - let mut buf_distinct = BufferedDistinctMap::new(&mut distinct_map); - let mut documents_seen = 0; - - for mut group in tmp_groups { - // if this group does not overlap with the requested range, - // push it without sorting and splitting it - if documents_seen + group.len() < distinct_raw_offset { - documents_seen += group.len(); - groups.push(group); - continue; - } - - let ctx = ContextMut { - reader, - postings_lists: &mut arena, - query_mapping: &mapping, - documents_fields_counts_store: index.documents_fields_counts, - }; - - let before_criterion_preparation = Instant::now(); - criterion.prepare(ctx, &mut group)?; - debug!("{:?} preparation took {:.02?}", criterion.name(), before_criterion_preparation.elapsed()); - - let ctx = Context { - postings_lists: &arena, - query_mapping: &mapping, - }; - - let before_criterion_sort = Instant::now(); - group.sort_unstable_by(|a, b| criterion.evaluate(&ctx, a, b)); - debug!("{:?} evaluation took {:.02?}", criterion.name(), before_criterion_sort.elapsed()); - - for group in group.binary_group_by_mut(|a, b| criterion.eq(&ctx, a, b)) { - // we must compute the real distinguished len of this sub-group - for document in group.iter() { - let filter_accepted = match &filter { - Some(filter) => { - let entry = filter_map.entry(document.id); - *entry.or_insert_with(|| { - let accepted = (filter)(document.id); - // we only want to count it out the first time we see it - if !accepted { - filtered_count += 1; - } - accepted - }) - } - None => true, - }; - - if filter_accepted { - let entry = key_cache.entry(document.id); - let mut seen = true; - let key = entry.or_insert_with(|| { - seen = false; - (distinct)(document.id).map(Rc::new) - }); - - let distinct = match key.clone() { - Some(key) => buf_distinct.register(key), - None => buf_distinct.register_without_key(), - }; - - // we only want to count the document if it is the first time we see it and - // if it wasn't accepted by distinct - if !seen && !distinct { - filtered_count += 1; - } - } - - // the requested range end is reached: stop computing distinct - if buf_distinct.len() >= range.end { - break; - } - } - - documents_seen += group.len(); - groups.push(group); - - // if this sub-group does not overlap with the requested range - // we must update the distinct map and its start index - if buf_distinct.len() < range.start { - buf_distinct.transfert_to_internal(); - distinct_raw_offset = documents_seen; - } - - // we have sort enough documents if the last document sorted is after - // the end of the requested range, we can continue to the next criterion - if buf_distinct.len() >= range.end { - continue 'criteria; - } - } - } - } - - // once we classified the documents related to the current - // automatons we save that as the next valid result - let mut seen = BufferedDistinctMap::new(&mut distinct_map); - let schema = index.main.schema(reader)?.ok_or(Error::SchemaMissing)?; - - let mut documents = Vec::with_capacity(range.len()); - for raw_document in raw_documents.into_iter().skip(distinct_raw_offset) { - let filter_accepted = match &filter { - Some(_) => filter_map.remove(&raw_document.id).unwrap_or_else(|| { - error!("error during filtering: expected value for document id {}", &raw_document.id.0); - Default::default() - }), - None => true, - }; - - if filter_accepted { - let key = key_cache.remove(&raw_document.id).unwrap_or_else(|| { - error!("error during distinct: expected value for document id {}", &raw_document.id.0); - Default::default() - }); - let distinct_accepted = match key { - Some(key) => seen.register(key), - None => seen.register_without_key(), - }; - - if distinct_accepted && seen.len() > range.start { - documents.push(Document::from_raw(raw_document, &queries_kinds, &arena, searchable_attrs.as_ref(), &schema)); - if documents.len() == range.len() { - break; - } - } - } - } - result.documents = documents; - result.nb_hits = docids.len() - filtered_count; - - Ok(result) -} - -fn cleanup_bare_matches<'tag, 'txn>( - arena: &mut SmallArena<'tag, PostingsListView<'txn>>, - docids: &Set, - queries: HashMap>>, -) -> Vec> -{ - let docidslen = docids.len() as f32; - let mut bare_matches = Vec::new(); - - for (PostingsKey { query, input, distance, is_exact }, matches) in queries { - let postings_list_view = PostingsListView::original(Rc::from(input), Rc::new(matches)); - let pllen = postings_list_view.len() as f32; - - if docidslen / pllen >= 0.8 { - let mut offset = 0; - for matches in postings_list_view.linear_group_by_key(|m| m.document_id) { - let document_id = matches[0].document_id; - if docids.contains(&document_id) { - let range = postings_list_view.range(offset, matches.len()); - let posting_list_index = arena.add(range); - - let bare_match = BareMatch { - document_id, - query_index: query.id, - distance, - is_exact, - postings_list: posting_list_index, - }; - - bare_matches.push(bare_match); - } - - offset += matches.len(); - } - - } else { - let mut offset = 0; - for id in docids.as_slice() { - let di = DocIndex { document_id: *id, ..DocIndex::default() }; - let pos = exponential_search(&postings_list_view[offset..], &di).unwrap_or_else(|x| x); - - offset += pos; - - let group = postings_list_view[offset..] - .linear_group_by_key(|m| m.document_id) - .next() - .filter(|matches| matches[0].document_id == *id); - - if let Some(matches) = group { - let range = postings_list_view.range(offset, matches.len()); - let posting_list_index = arena.add(range); - - let bare_match = BareMatch { - document_id: *id, - query_index: query.id, - distance, - is_exact, - postings_list: posting_list_index, - }; - - bare_matches.push(bare_match); - } - } - } - } - - let before_raw_documents_presort = Instant::now(); - bare_matches.sort_unstable_by_key(|sm| sm.document_id); - debug!("sort by documents ids took {:.02?}", before_raw_documents_presort.elapsed()); - - bare_matches -} - -pub struct BareMatch<'tag> { - pub document_id: DocumentId, - pub query_index: usize, - pub distance: u8, - pub is_exact: bool, - pub postings_list: Idx32<'tag>, -} - -impl fmt::Debug for BareMatch<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BareMatch") - .field("document_id", &self.document_id) - .field("query_index", &self.query_index) - .field("distance", &self.distance) - .field("is_exact", &self.is_exact) - .finish() - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct SimpleMatch { - pub query_index: usize, - pub distance: u8, - pub attribute: u16, - pub word_index: u16, - pub is_exact: bool, -} - -#[derive(Clone)] -pub enum PostingsListView<'txn> { - Original { - input: Rc<[u8]>, - postings_list: Rc>>, - offset: usize, - len: usize, - }, - Rewritten { - input: Rc<[u8]>, - postings_list: SetBuf, - }, -} - -impl fmt::Debug for PostingsListView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PostingsListView") - .field("input", &std::str::from_utf8(&self.input()).unwrap()) - .field("postings_list", &self.as_ref()) - .finish() - } -} - -impl<'txn> PostingsListView<'txn> { - pub fn original(input: Rc<[u8]>, postings_list: Rc>>) -> PostingsListView<'txn> { - let len = postings_list.len(); - PostingsListView::Original { input, postings_list, offset: 0, len } - } - - pub fn rewritten(input: Rc<[u8]>, postings_list: SetBuf) -> PostingsListView<'txn> { - PostingsListView::Rewritten { input, postings_list } - } - - pub fn rewrite_with(&mut self, postings_list: SetBuf) { - let input = match self { - PostingsListView::Original { input, .. } => input.clone(), - PostingsListView::Rewritten { input, .. } => input.clone(), - }; - *self = PostingsListView::rewritten(input, postings_list); - } - - pub fn len(&self) -> usize { - match self { - PostingsListView::Original { len, .. } => *len, - PostingsListView::Rewritten { postings_list, .. } => postings_list.len(), - } - } - - pub fn input(&self) -> &[u8] { - match self { - PostingsListView::Original { ref input, .. } => input, - PostingsListView::Rewritten { ref input, .. } => input, - } - } - - pub fn range(&self, range_offset: usize, range_len: usize) -> PostingsListView<'txn> { - match self { - PostingsListView::Original { input, postings_list, offset, len } => { - assert!(range_offset + range_len <= *len); - PostingsListView::Original { - input: input.clone(), - postings_list: postings_list.clone(), - offset: offset + range_offset, - len: range_len, - } - }, - PostingsListView::Rewritten { .. } => { - panic!("Cannot create a range on a rewritten postings list view"); - } - } - } -} - -impl AsRef> for PostingsListView<'_> { - fn as_ref(&self) -> &Set { - self - } -} - -impl Deref for PostingsListView<'_> { - type Target = Set; - - fn deref(&self) -> &Set { - match *self { - PostingsListView::Original { ref postings_list, offset, len, .. } => { - Set::new_unchecked(&postings_list[offset..offset + len]) - }, - PostingsListView::Rewritten { ref postings_list, .. } => postings_list, - } - } -} - -/// sorts documents ids according to user defined ranking rules. -pub fn placeholder_document_sort( - document_ids: &mut [DocumentId], - index: &store::Index, - reader: &MainReader, - ranked_map: &RankedMap -) -> MResult<()> { - use crate::settings::RankingRule; - use std::cmp::Ordering; - - enum SortOrder { - Asc, - Desc, - } - - if let Some(ranking_rules) = index.main.ranking_rules(reader)? { - let schema = index.main.schema(reader)? - .ok_or(Error::SchemaMissing)?; - - // Select custom rules from ranking rules, and map them to custom rules - // containing a field_id - let ranking_rules = ranking_rules.iter().filter_map(|r| - match r { - RankingRule::Asc(name) => schema.id(name).map(|f| (f, SortOrder::Asc)), - RankingRule::Desc(name) => schema.id(name).map(|f| (f, SortOrder::Desc)), - _ => None, - }).collect::>(); - - document_ids.sort_unstable_by(|a, b| { - for (field_id, order) in &ranking_rules { - let a_value = ranked_map.get(*a, *field_id); - let b_value = ranked_map.get(*b, *field_id); - let (a, b) = match order { - SortOrder::Asc => (a_value, b_value), - SortOrder::Desc => (b_value, a_value), - }; - match a.cmp(&b) { - Ordering::Equal => continue, - ordering => return ordering, - } - } - Ordering::Equal - }); - } - Ok(()) -} - -/// For each entry in facet_docids, calculates the number of documents in the intersection with candidate_docids. -pub fn facet_count( - facet_docids: HashMap>)>>, - candidate_docids: &Set, -) -> HashMap> { - let mut facets_counts = HashMap::with_capacity(facet_docids.len()); - for (key, doc_map) in facet_docids { - let mut count_map = HashMap::with_capacity(doc_map.len()); - for (_, (value, docids)) in doc_map { - let mut counter = Counter::new(); - let op = OpBuilder::new(docids.as_ref(), candidate_docids).intersection(); - SetOperation::::extend_collection(op, &mut counter); - count_map.insert(value.to_string(), counter.0); - } - facets_counts.insert(key, count_map); - } - facets_counts -} diff --git a/meilisearch-core/src/criterion/attribute.rs b/meilisearch-core/src/criterion/attribute.rs deleted file mode 100644 index bf28330d2..000000000 --- a/meilisearch-core/src/criterion/attribute.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::cmp::Ordering; -use slice_group_by::GroupBy; -use crate::{RawDocument, MResult}; -use crate::bucket_sort::SimpleMatch; -use super::{Criterion, Context, ContextMut, prepare_bare_matches}; - -pub struct Attribute; - -impl Criterion for Attribute { - fn name(&self) -> &str { "attribute" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping); - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - #[inline] - fn sum_of_attribute(matches: &[SimpleMatch]) -> usize { - let mut sum_of_attribute = 0; - for group in matches.linear_group_by_key(|bm| bm.query_index) { - sum_of_attribute += group[0].attribute as usize; - } - sum_of_attribute - } - - let lhs = sum_of_attribute(&lhs.processed_matches); - let rhs = sum_of_attribute(&rhs.processed_matches); - - lhs.cmp(&rhs) - } -} diff --git a/meilisearch-core/src/criterion/document_id.rs b/meilisearch-core/src/criterion/document_id.rs deleted file mode 100644 index 2795423f2..000000000 --- a/meilisearch-core/src/criterion/document_id.rs +++ /dev/null @@ -1,16 +0,0 @@ -use std::cmp::Ordering; -use crate::RawDocument; -use super::{Criterion, Context}; - -pub struct DocumentId; - -impl Criterion for DocumentId { - fn name(&self) -> &str { "stable document id" } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - let lhs = &lhs.id; - let rhs = &rhs.id; - - lhs.cmp(rhs) - } -} diff --git a/meilisearch-core/src/criterion/exactness.rs b/meilisearch-core/src/criterion/exactness.rs deleted file mode 100644 index 9b2d7c188..000000000 --- a/meilisearch-core/src/criterion/exactness.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::cmp::{Ordering, Reverse}; -use std::collections::hash_map::{HashMap, Entry}; -use meilisearch_schema::IndexedPos; -use slice_group_by::GroupBy; -use crate::{RawDocument, MResult}; -use crate::bucket_sort::BareMatch; -use super::{Criterion, Context, ContextMut}; - -pub struct Exactness; - -impl Criterion for Exactness { - fn name(&self) -> &str { "exactness" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - let store = ctx.documents_fields_counts_store; - let reader = ctx.reader; - - 'documents: for doc in documents { - doc.bare_matches.sort_unstable_by_key(|bm| (bm.query_index, Reverse(bm.is_exact))); - - // mark the document if we find a "one word field" that matches - let mut fields_counts = HashMap::new(); - for group in doc.bare_matches.linear_group_by_key(|bm| bm.query_index) { - for group in group.linear_group_by_key(|bm| bm.is_exact) { - if !group[0].is_exact { break } - - for bm in group { - for di in ctx.postings_lists[bm.postings_list].as_ref() { - - let attr = IndexedPos(di.attribute); - let count = match fields_counts.entry(attr) { - Entry::Occupied(entry) => *entry.get(), - Entry::Vacant(entry) => { - let count = store.document_field_count(reader, doc.id, attr)?; - *entry.insert(count) - }, - }; - - if count == Some(1) { - doc.contains_one_word_field = true; - continue 'documents - } - } - } - } - } - } - - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - #[inline] - fn sum_exact_query_words(matches: &[BareMatch]) -> usize { - let mut sum_exact_query_words = 0; - - for group in matches.linear_group_by_key(|bm| bm.query_index) { - sum_exact_query_words += group[0].is_exact as usize; - } - - sum_exact_query_words - } - - // does it contains a "one word field" - lhs.contains_one_word_field.cmp(&rhs.contains_one_word_field).reverse() - // if not, with document contains the more exact words - .then_with(|| { - let lhs = sum_exact_query_words(&lhs.bare_matches); - let rhs = sum_exact_query_words(&rhs.bare_matches); - lhs.cmp(&rhs).reverse() - }) - } -} diff --git a/meilisearch-core/src/criterion/mod.rs b/meilisearch-core/src/criterion/mod.rs deleted file mode 100644 index 3fe77115a..000000000 --- a/meilisearch-core/src/criterion/mod.rs +++ /dev/null @@ -1,292 +0,0 @@ -use std::cmp::{self, Ordering}; -use std::collections::HashMap; -use std::ops::Range; - -use compact_arena::SmallArena; -use sdset::SetBuf; -use slice_group_by::GroupBy; - -use crate::bucket_sort::{SimpleMatch, PostingsListView}; -use crate::database::MainT; -use crate::query_tree::QueryId; -use crate::{store, RawDocument, MResult}; - -mod typo; -mod words; -mod proximity; -mod attribute; -mod words_position; -mod exactness; -mod document_id; -mod sort_by_attr; - -pub use self::typo::Typo; -pub use self::words::Words; -pub use self::proximity::Proximity; -pub use self::attribute::Attribute; -pub use self::words_position::WordsPosition; -pub use self::exactness::Exactness; -pub use self::document_id::DocumentId; -pub use self::sort_by_attr::SortByAttr; - -pub trait Criterion { - fn name(&self) -> &str; - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - _ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - _documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - Ok(()) - } - - fn evaluate<'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: &Context<'p, 'tag, 'txn, 'q>, - lhs: &RawDocument<'r, 'tag>, - rhs: &RawDocument<'r, 'tag>, - ) -> Ordering; - - #[inline] - fn eq<'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: &Context<'p, 'tag, 'txn, 'q>, - lhs: &RawDocument<'r, 'tag>, - rhs: &RawDocument<'r, 'tag>, - ) -> bool - { - self.evaluate(ctx, lhs, rhs) == Ordering::Equal - } -} - -pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> { - pub reader: &'h heed::RoTxn<'h, MainT>, - pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>, - pub query_mapping: &'q HashMap>, - pub documents_fields_counts_store: store::DocumentsFieldsCounts, -} - -pub struct Context<'p, 'tag, 'txn, 'q> { - pub postings_lists: &'p SmallArena<'tag, PostingsListView<'txn>>, - pub query_mapping: &'q HashMap>, -} - -#[derive(Default)] -pub struct CriteriaBuilder<'a> { - inner: Vec>, -} - -impl<'a> CriteriaBuilder<'a> { - pub fn new() -> CriteriaBuilder<'a> { - CriteriaBuilder { inner: Vec::new() } - } - - pub fn with_capacity(capacity: usize) -> CriteriaBuilder<'a> { - CriteriaBuilder { - inner: Vec::with_capacity(capacity), - } - } - - pub fn reserve(&mut self, additional: usize) { - self.inner.reserve(additional) - } - - #[allow(clippy::should_implement_trait)] - pub fn add(mut self, criterion: C) -> CriteriaBuilder<'a> - where - C: Criterion, - { - self.push(criterion); - self - } - - pub fn push(&mut self, criterion: C) - where - C: Criterion, - { - self.inner.push(Box::new(criterion)); - } - - pub fn build(self) -> Criteria<'a> { - Criteria { inner: self.inner } - } -} - -pub struct Criteria<'a> { - inner: Vec>, -} - -impl<'a> Default for Criteria<'a> { - fn default() -> Self { - CriteriaBuilder::with_capacity(7) - .add(Typo) - .add(Words) - .add(Proximity) - .add(Attribute) - .add(WordsPosition) - .add(Exactness) - .add(DocumentId) - .build() - } -} - -impl<'a> AsRef<[Box]> for Criteria<'a> { - fn as_ref(&self) -> &[Box] { - &self.inner - } -} - -fn prepare_query_distances<'a, 'tag, 'txn>( - documents: &mut [RawDocument<'a, 'tag>], - query_mapping: &HashMap>, - postings_lists: &SmallArena<'tag, PostingsListView<'txn>>, -) { - for document in documents { - if !document.processed_distances.is_empty() { continue } - - let mut processed = Vec::new(); - for m in document.bare_matches.iter() { - if postings_lists[m.postings_list].is_empty() { continue } - - let range = query_mapping[&(m.query_index as usize)].clone(); - let new_len = cmp::max(range.end as usize, processed.len()); - processed.resize(new_len, None); - - for index in range { - let index = index as usize; - processed[index] = match processed[index] { - Some(distance) if distance > m.distance => Some(m.distance), - Some(distance) => Some(distance), - None => Some(m.distance), - }; - } - } - - document.processed_distances = processed; - } -} - -fn prepare_bare_matches<'a, 'tag, 'txn>( - documents: &mut [RawDocument<'a, 'tag>], - postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>, - query_mapping: &HashMap>, -) { - for document in documents { - if !document.processed_matches.is_empty() { continue } - - let mut processed = Vec::new(); - for m in document.bare_matches.iter() { - let postings_list = &postings_lists[m.postings_list]; - processed.reserve(postings_list.len()); - for di in postings_list.as_ref() { - let simple_match = SimpleMatch { - query_index: m.query_index, - distance: m.distance, - attribute: di.attribute, - word_index: di.word_index, - is_exact: m.is_exact, - }; - processed.push(simple_match); - } - } - - let processed = multiword_rewrite_matches(&mut processed, query_mapping); - document.processed_matches = processed.into_vec(); - } -} - -fn multiword_rewrite_matches( - matches: &mut [SimpleMatch], - query_mapping: &HashMap>, -) -> SetBuf -{ - matches.sort_unstable_by_key(|m| (m.attribute, m.word_index)); - - let mut padded_matches = Vec::with_capacity(matches.len()); - - // let before_padding = Instant::now(); - // for each attribute of each document - for same_document_attribute in matches.linear_group_by_key(|m| m.attribute) { - // padding will only be applied - // to word indices in the same attribute - let mut padding = 0; - let mut iter = same_document_attribute.linear_group_by_key(|m| m.word_index); - - // for each match at the same position - // in this document attribute - while let Some(same_word_index) = iter.next() { - // find the biggest padding - let mut biggest = 0; - for match_ in same_word_index { - let mut replacement = query_mapping[&(match_.query_index as usize)].clone(); - let replacement_len = replacement.len(); - let nexts = iter.remainder().linear_group_by_key(|m| m.word_index); - - if let Some(query_index) = replacement.next() { - let word_index = match_.word_index + padding as u16; - let match_ = SimpleMatch { query_index, word_index, ..*match_ }; - padded_matches.push(match_); - } - - let mut found = false; - - // look ahead and if there already is a match - // corresponding to this padding word, abort the padding - 'padding: for (x, next_group) in nexts.enumerate() { - for (i, query_index) in replacement.clone().enumerate().skip(x) { - let word_index = match_.word_index + padding as u16 + (i + 1) as u16; - let padmatch = SimpleMatch { query_index, word_index, ..*match_ }; - - for nmatch_ in next_group { - let mut rep = query_mapping[&(nmatch_.query_index as usize)].clone(); - let query_index = rep.next().unwrap(); - if query_index == padmatch.query_index { - if !found { - // if we find a corresponding padding for the - // first time we must push preceding paddings - for (i, query_index) in replacement.clone().enumerate().take(i) { - let word_index = match_.word_index + padding as u16 + (i + 1) as u16; - let match_ = SimpleMatch { query_index, word_index, ..*match_ }; - padded_matches.push(match_); - biggest = biggest.max(i + 1); - } - } - - padded_matches.push(padmatch); - found = true; - continue 'padding; - } - } - } - - // if we do not find a corresponding padding in the - // next groups so stop here and pad what was found - break; - } - - if !found { - // if no padding was found in the following matches - // we must insert the entire padding - for (i, query_index) in replacement.enumerate() { - let word_index = match_.word_index + padding as u16 + (i + 1) as u16; - let match_ = SimpleMatch { query_index, word_index, ..*match_ }; - padded_matches.push(match_); - } - - biggest = biggest.max(replacement_len - 1); - } - } - - padding += biggest; - } - } - - // debug!("padding matches took {:.02?}", before_padding.elapsed()); - - // With this check we can see that the loop above takes something - // like 43% of the search time even when no rewrite is needed. - // assert_eq!(before_matches, padded_matches); - - SetBuf::from_dirty(padded_matches) -} diff --git a/meilisearch-core/src/criterion/proximity.rs b/meilisearch-core/src/criterion/proximity.rs deleted file mode 100644 index c6a606d56..000000000 --- a/meilisearch-core/src/criterion/proximity.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::cmp::{self, Ordering}; -use slice_group_by::GroupBy; -use crate::bucket_sort::{SimpleMatch}; -use crate::{RawDocument, MResult}; -use super::{Criterion, Context, ContextMut, prepare_bare_matches}; - -const MAX_DISTANCE: u16 = 8; - -pub struct Proximity; - -impl Criterion for Proximity { - fn name(&self) -> &str { "proximity" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping); - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - fn index_proximity(lhs: u16, rhs: u16) -> u16 { - if lhs < rhs { - cmp::min(rhs - lhs, MAX_DISTANCE) - } else { - cmp::min(lhs - rhs, MAX_DISTANCE) + 1 - } - } - - fn attribute_proximity(lhs: SimpleMatch, rhs: SimpleMatch) -> u16 { - if lhs.attribute != rhs.attribute { MAX_DISTANCE } - else { index_proximity(lhs.word_index, rhs.word_index) } - } - - fn min_proximity(lhs: &[SimpleMatch], rhs: &[SimpleMatch]) -> u16 { - let mut min_prox = u16::max_value(); - for a in lhs { - for b in rhs { - let prox = attribute_proximity(*a, *b); - min_prox = cmp::min(min_prox, prox); - } - } - min_prox - } - - fn matches_proximity(matches: &[SimpleMatch],) -> u16 { - let mut proximity = 0; - let mut iter = matches.linear_group_by_key(|m| m.query_index); - - // iterate over groups by windows of size 2 - let mut last = iter.next(); - while let (Some(lhs), Some(rhs)) = (last, iter.next()) { - proximity += min_proximity(lhs, rhs); - last = Some(rhs); - } - - proximity - } - - let lhs = matches_proximity(&lhs.processed_matches); - let rhs = matches_proximity(&rhs.processed_matches); - - lhs.cmp(&rhs) - } -} diff --git a/meilisearch-core/src/criterion/sort_by_attr.rs b/meilisearch-core/src/criterion/sort_by_attr.rs deleted file mode 100644 index 453aba655..000000000 --- a/meilisearch-core/src/criterion/sort_by_attr.rs +++ /dev/null @@ -1,129 +0,0 @@ -use std::cmp::Ordering; -use std::error::Error; -use std::fmt; -use meilisearch_schema::{Schema, FieldId}; -use crate::{RankedMap, RawDocument}; -use super::{Criterion, Context}; - -/// An helper struct that permit to sort documents by -/// some of their stored attributes. -/// -/// # Note -/// -/// If a document cannot be deserialized it will be considered [`None`][]. -/// -/// Deserialized documents are compared like `Some(doc0).cmp(&Some(doc1))`, -/// so you must check the [`Ord`] of `Option` implementation. -/// -/// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None -/// [`Ord`]: https://doc.rust-lang.org/std/option/enum.Option.html#impl-Ord -/// -/// # Example -/// -/// ```ignore -/// use serde_derive::Deserialize; -/// use meilisearch::rank::criterion::*; -/// -/// let custom_ranking = SortByAttr::lower_is_better(&ranked_map, &schema, "published_at")?; -/// -/// let builder = CriteriaBuilder::with_capacity(8) -/// .add(Typo) -/// .add(Words) -/// .add(Proximity) -/// .add(Attribute) -/// .add(WordsPosition) -/// .add(Exactness) -/// .add(custom_ranking) -/// .add(DocumentId); -/// -/// let criterion = builder.build(); -/// -/// ``` -pub struct SortByAttr<'a> { - ranked_map: &'a RankedMap, - field_id: FieldId, - reversed: bool, -} - -impl<'a> SortByAttr<'a> { - pub fn lower_is_better( - ranked_map: &'a RankedMap, - schema: &Schema, - attr_name: &str, - ) -> Result, SortByAttrError> { - SortByAttr::new(ranked_map, schema, attr_name, false) - } - - pub fn higher_is_better( - ranked_map: &'a RankedMap, - schema: &Schema, - attr_name: &str, - ) -> Result, SortByAttrError> { - SortByAttr::new(ranked_map, schema, attr_name, true) - } - - fn new( - ranked_map: &'a RankedMap, - schema: &Schema, - attr_name: &str, - reversed: bool, - ) -> Result, SortByAttrError> { - let field_id = match schema.id(attr_name) { - Some(field_id) => field_id, - None => return Err(SortByAttrError::AttributeNotFound), - }; - - if !schema.is_ranked(field_id) { - return Err(SortByAttrError::AttributeNotRegisteredForRanking); - } - - Ok(SortByAttr { - ranked_map, - field_id, - reversed, - }) - } -} - -impl Criterion for SortByAttr<'_> { - fn name(&self) -> &str { - "sort by attribute" - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - let lhs = self.ranked_map.get(lhs.id, self.field_id); - let rhs = self.ranked_map.get(rhs.id, self.field_id); - - match (lhs, rhs) { - (Some(lhs), Some(rhs)) => { - let order = lhs.cmp(&rhs); - if self.reversed { - order.reverse() - } else { - order - } - } - (None, Some(_)) => Ordering::Greater, - (Some(_), None) => Ordering::Less, - (None, None) => Ordering::Equal, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum SortByAttrError { - AttributeNotFound, - AttributeNotRegisteredForRanking, -} - -impl fmt::Display for SortByAttrError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use SortByAttrError::*; - match self { - AttributeNotFound => f.write_str("attribute not found in the schema"), - AttributeNotRegisteredForRanking => f.write_str("attribute not registered for ranking"), - } - } -} - -impl Error for SortByAttrError {} diff --git a/meilisearch-core/src/criterion/typo.rs b/meilisearch-core/src/criterion/typo.rs deleted file mode 100644 index 4b9fdea1d..000000000 --- a/meilisearch-core/src/criterion/typo.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::cmp::Ordering; -use crate::{RawDocument, MResult}; -use super::{Criterion, Context, ContextMut, prepare_query_distances}; - -pub struct Typo; - -impl Criterion for Typo { - fn name(&self) -> &str { "typo" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists); - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - // This function is a wrong logarithmic 10 function. - // It is safe to panic on input number higher than 3, - // the number of typos is never bigger than that. - #[inline] - #[allow(clippy::approx_constant)] - fn custom_log10(n: u8) -> f32 { - match n { - 0 => 0.0, // log(1) - 1 => 0.30102, // log(2) - 2 => 0.47712, // log(3) - 3 => 0.60205, // log(4) - _ => panic!("invalid number"), - } - } - - #[inline] - fn compute_typos(distances: &[Option]) -> usize { - let mut number_words: usize = 0; - let mut sum_typos = 0.0; - - for distance in distances { - if let Some(distance) = distance { - sum_typos += custom_log10(*distance); - number_words += 1; - } - } - - (number_words as f32 / (sum_typos + 1.0) * 1000.0) as usize - } - - let lhs = compute_typos(&lhs.processed_distances); - let rhs = compute_typos(&rhs.processed_distances); - - lhs.cmp(&rhs).reverse() - } -} diff --git a/meilisearch-core/src/criterion/words.rs b/meilisearch-core/src/criterion/words.rs deleted file mode 100644 index 1a171ee1e..000000000 --- a/meilisearch-core/src/criterion/words.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::cmp::Ordering; -use crate::{RawDocument, MResult}; -use super::{Criterion, Context, ContextMut, prepare_query_distances}; - -pub struct Words; - -impl Criterion for Words { - fn name(&self) -> &str { "words" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - prepare_query_distances(documents, ctx.query_mapping, ctx.postings_lists); - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - #[inline] - fn number_of_query_words(distances: &[Option]) -> usize { - distances.iter().cloned().filter(Option::is_some).count() - } - - let lhs = number_of_query_words(&lhs.processed_distances); - let rhs = number_of_query_words(&rhs.processed_distances); - - lhs.cmp(&rhs).reverse() - } -} diff --git a/meilisearch-core/src/criterion/words_position.rs b/meilisearch-core/src/criterion/words_position.rs deleted file mode 100644 index 037e14de6..000000000 --- a/meilisearch-core/src/criterion/words_position.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::cmp::Ordering; -use slice_group_by::GroupBy; -use crate::bucket_sort::SimpleMatch; -use crate::{RawDocument, MResult}; -use super::{Criterion, Context, ContextMut, prepare_bare_matches}; - -pub struct WordsPosition; - -impl Criterion for WordsPosition { - fn name(&self) -> &str { "words position" } - - fn prepare<'h, 'p, 'tag, 'txn, 'q, 'r>( - &self, - ctx: ContextMut<'h, 'p, 'tag, 'txn, 'q>, - documents: &mut [RawDocument<'r, 'tag>], - ) -> MResult<()> - { - prepare_bare_matches(documents, ctx.postings_lists, ctx.query_mapping); - Ok(()) - } - - fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - #[inline] - fn sum_words_position(matches: &[SimpleMatch]) -> usize { - let mut sum_words_position = 0; - for group in matches.linear_group_by_key(|bm| bm.query_index) { - sum_words_position += group[0].word_index as usize; - } - sum_words_position - } - - let lhs = sum_words_position(&lhs.processed_matches); - let rhs = sum_words_position(&rhs.processed_matches); - - lhs.cmp(&rhs) - } -} diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs deleted file mode 100644 index da8d44d6a..000000000 --- a/meilisearch-core/src/database.rs +++ /dev/null @@ -1,1302 +0,0 @@ -use std::collections::hash_map::{Entry, HashMap}; -use std::collections::BTreeMap; -use std::fs::File; -use std::path::Path; -use std::sync::{Arc, RwLock}; -use std::{fs, thread}; -use std::io::{Read, Write, ErrorKind}; - -use chrono::{DateTime, Utc}; -use crossbeam_channel::{Receiver, Sender}; -use heed::CompactionOption; -use heed::types::{Str, Unit, SerdeBincode}; -use log::{debug, error}; -use meilisearch_schema::Schema; -use regex::Regex; - -use crate::{store, update, Index, MResult, Error}; - -pub type BoxUpdateFn = Box; - -type ArcSwapFn = arc_swap::ArcSwapOption; - -type SerdeDatetime = SerdeBincode>; - -pub type MainWriter<'a, 'b> = heed::RwTxn<'a, 'b, MainT>; -pub type MainReader<'a, 'b> = heed::RoTxn<'a, MainT>; - -pub type UpdateWriter<'a, 'b> = heed::RwTxn<'a, 'b, UpdateT>; -pub type UpdateReader<'a> = heed::RoTxn<'a, UpdateT>; - -const LAST_UPDATE_KEY: &str = "last-update"; - -pub struct MainT; -pub struct UpdateT; - -pub struct Database { - env: heed::Env, - update_env: heed::Env, - common_store: heed::PolyDatabase, - indexes_store: heed::Database, - indexes: RwLock>)>>, - update_fn: Arc, - database_version: (u32, u32, u32), -} - -pub struct DatabaseOptions { - pub main_map_size: usize, - pub update_map_size: usize, -} - -impl Default for DatabaseOptions { - fn default() -> DatabaseOptions { - DatabaseOptions { - main_map_size: 100 * 1024 * 1024 * 1024, //100Gb - update_map_size: 100 * 1024 * 1024 * 1024, //100Gb - } - } -} - -macro_rules! r#break_try { - ($expr:expr, $msg:tt) => { - match $expr { - core::result::Result::Ok(val) => val, - core::result::Result::Err(err) => { - log::error!(concat!($msg, ": {}"), err); - break; - } - } - }; -} - -pub enum UpdateEvent { - NewUpdate, - MustClear, -} - -pub type UpdateEvents = Receiver; -pub type UpdateEventsEmitter = Sender; - -fn update_awaiter( - receiver: UpdateEvents, - env: heed::Env, - update_env: heed::Env, - index_uid: &str, - update_fn: Arc, - index: Index, -) -> MResult<()> { - for event in receiver { - - // if we receive a *MustClear* event, clear the index and break the loop - if let UpdateEvent::MustClear = event { - let mut writer = env.typed_write_txn::()?; - let mut update_writer = update_env.typed_write_txn::()?; - - store::clear(&mut writer, &mut update_writer, &index)?; - - writer.commit()?; - update_writer.commit()?; - - debug!("store {} cleared", index_uid); - - break - } - - loop { - // We instantiate a *write* transaction to *block* the thread - // until the *other*, notifiying, thread commits - let result = update_env.typed_write_txn::(); - let update_reader = break_try!(result, "LMDB read transaction (update) begin failed"); - - // retrieve the update that needs to be processed - let result = index.updates.first_update(&update_reader); - let (update_id, update) = match break_try!(result, "pop front update failed") { - Some(value) => value, - None => { - debug!("no more updates"); - break; - } - }; - - // do not keep the reader for too long - break_try!(update_reader.abort(), "aborting update transaction failed"); - - // instantiate a transaction to touch to the main env - let result = env.typed_write_txn::(); - let mut main_writer = break_try!(result, "LMDB nested write transaction failed"); - - // try to apply the update to the database using the main transaction - let result = update::update_task(&mut main_writer, &index, update_id, update); - let status = break_try!(result, "update task failed"); - - // commit the main transaction if the update was successful, abort it otherwise - if status.error.is_none() { - break_try!(main_writer.commit(), "commit nested transaction failed"); - } else { - break_try!(main_writer.abort(), "abborting nested transaction failed"); - } - - // now that the update has been processed we can instantiate - // a transaction to move the result to the updates-results store - let result = update_env.typed_write_txn::(); - let mut update_writer = break_try!(result, "LMDB write transaction begin failed"); - - // definitely remove the update from the updates store - index.updates.del_update(&mut update_writer, update_id)?; - - // write the result of the updates-results store - let updates_results = index.updates_results; - let result = updates_results.put_update_result(&mut update_writer, update_id, &status); - - // always commit the main transaction, even if the update was unsuccessful - break_try!(result, "update result store commit failed"); - break_try!(update_writer.commit(), "update transaction commit failed"); - - // call the user callback when the update and the result are written consistently - if let Some(ref callback) = *update_fn.load() { - (callback)(index_uid, status); - } - } - } - - debug!("update loop system stopped"); - - Ok(()) -} - -/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch. -/// If create is set to true, a VERSION file is created with the current version. -fn version_guard(path: &Path, create: bool) -> MResult<(u32, u32, u32)> { - let current_version_major = env!("CARGO_PKG_VERSION_MAJOR"); - let current_version_minor = env!("CARGO_PKG_VERSION_MINOR"); - let current_version_patch = env!("CARGO_PKG_VERSION_PATCH"); - let version_path = path.join("VERSION"); - - match File::open(&version_path) { - Ok(mut file) => { - let mut version = String::new(); - file.read_to_string(&mut version)?; - // Matches strings like XX.XX.XX - let re = Regex::new(r"(\d+).(\d+).(\d+)").unwrap(); - - // Make sure there is a result - let version = re - .captures_iter(&version) - .next() - .ok_or_else(|| Error::VersionMismatch("bad VERSION file".to_string()))?; - // the first is always the complete match, safe to unwrap because we have a match - let version_major = version.get(1).unwrap().as_str(); - let version_minor = version.get(2).unwrap().as_str(); - let version_patch = version.get(3).unwrap().as_str(); - - if version_major != current_version_major || version_minor != current_version_minor { - Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor))) - } else { - Ok(( - version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?, - version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?, - version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))? - )) - } - } - Err(error) => { - match error.kind() { - ErrorKind::NotFound => { - if create { - // when no version file is found, and we've been told to create one, - // create a new file with the current version in it. - let mut version_file = File::create(&version_path)?; - version_file.write_all(format!("{}.{}.{}", - current_version_major, - current_version_minor, - current_version_patch).as_bytes())?; - - Ok(( - current_version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?, - current_version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?, - current_version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))? - )) - } else { - // when no version file is found and we were not told to create one, this - // means that the version is inferior to the one this feature was added in. - Err(Error::VersionMismatch("<0.12.0".to_string())) - } - } - _ => Err(error.into()) - } - } - } -} - -impl Database { - pub fn open_or_create(path: impl AsRef, options: DatabaseOptions) -> MResult { - let main_path = path.as_ref().join("main"); - let update_path = path.as_ref().join("update"); - - //create db directory - fs::create_dir_all(&path)?; - - // create file only if main db wasn't created before (first run) - let database_version = version_guard(path.as_ref(), !main_path.exists() && !update_path.exists())?; - - fs::create_dir_all(&main_path)?; - let env = heed::EnvOpenOptions::new() - .map_size(options.main_map_size) - .max_dbs(3000) - .open(main_path)?; - - fs::create_dir_all(&update_path)?; - let update_env = heed::EnvOpenOptions::new() - .map_size(options.update_map_size) - .max_dbs(3000) - .open(update_path)?; - - let common_store = env.create_poly_database(Some("common"))?; - let indexes_store = env.create_database::(Some("indexes"))?; - let update_fn = Arc::new(ArcSwapFn::empty()); - - // list all indexes that needs to be opened - let mut must_open = Vec::new(); - let reader = env.read_txn()?; - for result in indexes_store.iter(&reader)? { - let (index_uid, _) = result?; - must_open.push(index_uid.to_owned()); - } - - reader.abort()?; - - // open the previously aggregated indexes - let mut indexes = HashMap::new(); - for index_uid in must_open { - let (sender, receiver) = crossbeam_channel::unbounded(); - let index = match store::open(&env, &update_env, &index_uid, sender.clone())? { - Some(index) => index, - None => { - log::warn!( - "the index {} doesn't exist or has not all the databases", - index_uid - ); - continue; - } - }; - - let env_clone = env.clone(); - let update_env_clone = update_env.clone(); - let index_clone = index.clone(); - let name_clone = index_uid.clone(); - let update_fn_clone = update_fn.clone(); - - let handle = thread::spawn(move || { - update_awaiter( - receiver, - env_clone, - update_env_clone, - &name_clone, - update_fn_clone, - index_clone, - ) - }); - - // send an update notification to make sure that - // possible pre-boot updates are consumed - sender.send(UpdateEvent::NewUpdate).unwrap(); - - let result = indexes.insert(index_uid, (index, handle)); - assert!( - result.is_none(), - "The index should not have been already open" - ); - } - - Ok(Database { - env, - update_env, - common_store, - indexes_store, - indexes: RwLock::new(indexes), - update_fn, - database_version, - }) - } - - pub fn open_index(&self, name: impl AsRef) -> Option { - let indexes_lock = self.indexes.read().unwrap(); - match indexes_lock.get(name.as_ref()) { - Some((index, ..)) => Some(index.clone()), - None => None, - } - } - - pub fn is_indexing(&self, reader: &UpdateReader, index: &str) -> MResult> { - match self.open_index(&index) { - Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())), - None => Ok(None), - } - } - - pub fn create_index(&self, name: impl AsRef) -> MResult { - let name = name.as_ref(); - let mut indexes_lock = self.indexes.write().unwrap(); - - match indexes_lock.entry(name.to_owned()) { - Entry::Occupied(_) => Err(crate::Error::IndexAlreadyExists), - Entry::Vacant(entry) => { - let (sender, receiver) = crossbeam_channel::unbounded(); - let index = store::create(&self.env, &self.update_env, name, sender)?; - - let mut writer = self.env.typed_write_txn::()?; - self.indexes_store.put(&mut writer, name, &())?; - - index.main.put_name(&mut writer, name)?; - index.main.put_created_at(&mut writer)?; - index.main.put_updated_at(&mut writer)?; - index.main.put_schema(&mut writer, &Schema::default())?; - - let env_clone = self.env.clone(); - let update_env_clone = self.update_env.clone(); - let index_clone = index.clone(); - let name_clone = name.to_owned(); - let update_fn_clone = self.update_fn.clone(); - - let handle = thread::spawn(move || { - update_awaiter( - receiver, - env_clone, - update_env_clone, - &name_clone, - update_fn_clone, - index_clone, - ) - }); - - writer.commit()?; - entry.insert((index.clone(), handle)); - - Ok(index) - } - } - } - - pub fn delete_index(&self, name: impl AsRef) -> MResult { - let name = name.as_ref(); - let mut indexes_lock = self.indexes.write().unwrap(); - - match indexes_lock.remove_entry(name) { - Some((name, (index, handle))) => { - // remove the index name from the list of indexes - // and clear all the LMDB dbi - let mut writer = self.env.write_txn()?; - self.indexes_store.delete(&mut writer, &name)?; - writer.commit()?; - - // send a stop event to the update loop of the index - index.updates_notifier.send(UpdateEvent::MustClear).unwrap(); - - drop(indexes_lock); - - // join the update loop thread to ensure it is stopped - handle.join().unwrap()?; - - Ok(true) - } - None => Ok(false), - } - } - - pub fn set_update_callback(&self, update_fn: BoxUpdateFn) { - let update_fn = Some(Arc::new(update_fn)); - self.update_fn.swap(update_fn); - } - - pub fn unset_update_callback(&self) { - self.update_fn.swap(None); - } - - pub fn main_read_txn(&self) -> MResult { - Ok(self.env.typed_read_txn::()?) - } - - pub(crate) fn main_write_txn(&self) -> MResult { - Ok(self.env.typed_write_txn::()?) - } - - /// Calls f providing it with a writer to the main database. After f is called, makes sure the - /// transaction is commited. Returns whatever result f returns. - pub fn main_write(&self, f: F) -> Result - where - F: FnOnce(&mut MainWriter) -> Result, - E: From, - { - let mut writer = self.main_write_txn()?; - let result = f(&mut writer)?; - writer.commit().map_err(Error::Heed)?; - Ok(result) - } - - /// provides a context with a reader to the main database. experimental. - pub fn main_read(&self, f: F) -> Result - where - F: FnOnce(&MainReader) -> Result, - E: From, - { - let reader = self.main_read_txn()?; - let result = f(&reader)?; - reader.abort().map_err(Error::Heed)?; - Ok(result) - } - - pub fn update_read_txn(&self) -> MResult { - Ok(self.update_env.typed_read_txn::()?) - } - - pub(crate) fn update_write_txn(&self) -> MResult> { - Ok(self.update_env.typed_write_txn::()?) - } - - /// Calls f providing it with a writer to the main database. After f is called, makes sure the - /// transaction is commited. Returns whatever result f returns. - pub fn update_write(&self, f: F) -> Result - where - F: FnOnce(&mut UpdateWriter) -> Result, - E: From, - { - let mut writer = self.update_write_txn()?; - let result = f(&mut writer)?; - writer.commit().map_err(Error::Heed)?; - Ok(result) - } - - /// provides a context with a reader to the update database. experimental. - pub fn update_read(&self, f: F) -> Result - where - F: FnOnce(&UpdateReader) -> Result, - E: From, - { - let reader = self.update_read_txn()?; - let result = f(&reader)?; - reader.abort().map_err(Error::Heed)?; - Ok(result) - } - - pub fn copy_and_compact_to_path>(&self, path: P) -> MResult<(File, File)> { - let path = path.as_ref(); - - let env_path = path.join("main"); - let env_update_path = path.join("update"); - let env_version_path = path.join("VERSION"); - - fs::create_dir(&env_path)?; - fs::create_dir(&env_update_path)?; - - // write Database Version - let (current_version_major, current_version_minor, current_version_patch) = self.database_version; - let mut version_file = File::create(&env_version_path)?; - version_file.write_all(format!("{}.{}.{}", - current_version_major, - current_version_minor, - current_version_patch).as_bytes())?; - - let env_path = env_path.join("data.mdb"); - let env_file = self.env.copy_to_path(&env_path, CompactionOption::Enabled)?; - - let env_update_path = env_update_path.join("data.mdb"); - match self.update_env.copy_to_path(env_update_path, CompactionOption::Enabled) { - Ok(update_env_file) => Ok((env_file, update_env_file)), - Err(e) => { - fs::remove_file(env_path)?; - Err(e.into()) - }, - } - } - - pub fn indexes_uids(&self) -> Vec { - let indexes = self.indexes.read().unwrap(); - indexes.keys().cloned().collect() - } - - pub(crate) fn common_store(&self) -> heed::PolyDatabase { - self.common_store - } - - pub fn last_update(&self, reader: &heed::RoTxn) -> MResult>> { - match self.common_store() - .get::<_, Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)? { - Some(datetime) => Ok(Some(datetime)), - None => Ok(None), - } - } - - pub fn set_last_update(&self, writer: &mut heed::RwTxn, time: &DateTime) -> MResult<()> { - self.common_store() - .put::<_, Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, time)?; - Ok(()) - } - - pub fn compute_stats(&self, writer: &mut MainWriter, index_uid: &str) -> MResult<()> { - let index = match self.open_index(&index_uid) { - Some(index) => index, - None => { - error!("Impossible to retrieve index {}", index_uid); - return Ok(()); - } - }; - - let schema = match index.main.schema(&writer)? { - Some(schema) => schema, - None => return Ok(()), - }; - - let all_documents_fields = index - .documents_fields_counts - .all_documents_fields_counts(&writer)?; - - // count fields frequencies - let mut fields_frequency = HashMap::<_, usize>::new(); - for result in all_documents_fields { - let (_, attr, _) = result?; - if let Some(field_id) = schema.indexed_pos_to_field_id(attr) { - *fields_frequency.entry(field_id).or_default() += 1; - } - } - - // convert attributes to their names - let frequency: BTreeMap<_, _> = fields_frequency - .into_iter() - .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c))) - .collect(); - - index - .main - .put_fields_distribution(writer, &frequency) - } - - pub fn version(&self) -> (u32, u32, u32) { self.database_version } -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::bucket_sort::SortResult; - use crate::criterion::{self, CriteriaBuilder}; - use crate::update::{ProcessedUpdateResult, UpdateStatus}; - use crate::settings::Settings; - use crate::{Document, DocumentId}; - use serde::de::IgnoredAny; - use std::sync::mpsc; - - #[test] - fn valid_updates() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut update_writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut update_writer, settings).unwrap(); - update_writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut update_writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut update_writer).unwrap(); - update_writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - } - - #[test] - fn invalid_updates() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut update_writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut update_writer, settings).unwrap(); - update_writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - let doc2 = serde_json::json!({ - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut update_writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut update_writer).unwrap(); - update_writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Failed { content }) if content.error.is_some()); - } - - #[test] - fn ignored_words_too_long() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name"], - "displayedAttributes": ["name"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut update_writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut update_writer, settings).unwrap(); - update_writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "s̷̡̢̡̧̺̜̞͕͉͉͕̜͔̟̼̥̝͍̟̖͔͔̪͉̲̹̝̣̖͎̞̤̥͓͎̭̩͕̙̩̿̀̋̅̈́̌́̏̍̄̽͂̆̾̀̿̕̚̚͜͠͠ͅͅļ̵̨̨̨̰̦̻̳̖̳͚̬̫͚̦͖͈̲̫̣̩̥̻̙̦̱̼̠̖̻̼̘̖͉̪̜̠̙͖̙̩͔̖̯̩̲̿̽͋̔̿̍̓͂̍̿͊͆̃͗̔̎͐͌̾̆͗́̆̒̔̾̅̚̚͜͜ͅͅī̵̛̦̅̔̓͂͌̾́͂͛̎̋͐͆̽̂̋̋́̾̀̉̓̏̽́̑̀͒̇͋͛̈́̃̉̏͊̌̄̽̿̏̇͘̕̚̕p̶̧̛̛̖̯̗͕̝̗̭̱͙̖̗̟̟̐͆̊̂͐̋̓̂̈́̓͊̆͌̾̾͐͋͗͌̆̿̅͆̈́̈́̉͋̍͊͗̌̓̅̈̎̇̃̎̈́̉̐̋͑̃͘̕͘d̴̢̨̛͕̘̯͖̭̮̝̝̐̊̈̅̐̀͒̀́̈́̀͌̽͛͆͑̀̽̿͛̃̋̇̎̀́̂́͘͠͝ǫ̵̨̛̮̩̘͚̬̯̖̱͍̼͑͑̓̐́̑̿̈́̔͌̂̄͐͝ģ̶̧̜͇̣̭̺̪̺̖̻͖̮̭̣̙̻͒͊͗̓̓͒̀̀ͅ", - }); - - additions.update_document(doc1); - - let mut update_writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut update_writer).unwrap(); - update_writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - } - - #[test] - fn add_schema_attributes_at_end() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut update_writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut update_writer, settings).unwrap(); - update_writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut update_writer = db.update_write_txn().unwrap(); - let _update_id = additions.finalize(&mut update_writer).unwrap(); - update_writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description", "age", "sex"], - "displayedAttributes": ["name", "description", "age", "sex"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.iter().find(|id| *id == update_id); - - // check if it has been accepted - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - update_reader.abort().unwrap(); - - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - "age": 21, - "sex": "Male", - }); - - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - "age": 23, - "sex": "Male", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.iter().find(|id| *id == update_id); - - // check if it has been accepted - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - update_reader.abort().unwrap(); - - // even try to search for a document - let reader = db.main_read_txn().unwrap(); - let SortResult {documents, .. } = index.query_builder().query(&reader, Some("21 "), 0..20).unwrap(); - assert_matches!(documents.len(), 1); - - reader.abort().unwrap(); - - // try to introduce attributes in the middle of the schema - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description", "city", "age", "sex"], - "displayedAttributes": ["name", "description", "city", "age", "sex"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.iter().find(|id| *id == update_id); - // check if it has been accepted - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - } - - #[test] - fn deserialize_documents() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - // DocumentId(7900334843754999545) - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - // DocumentId(8367468610878465872) - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - update_reader.abort().unwrap(); - - let reader = db.main_read_txn().unwrap(); - let document: Option = index.document(&reader, None, DocumentId(25)).unwrap(); - assert!(document.is_none()); - - let document: Option = index - .document(&reader, None, DocumentId(0)) - .unwrap(); - assert!(document.is_some()); - - let document: Option = index - .document(&reader, None, DocumentId(1)) - .unwrap(); - assert!(document.is_some()); - } - - #[test] - fn partial_document_update() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description", "id"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - // DocumentId(7900334843754999545) - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - // DocumentId(8367468610878465872) - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - update_reader.abort().unwrap(); - - let reader = db.main_read_txn().unwrap(); - let document: Option = index.document(&reader, None, DocumentId(25)).unwrap(); - assert!(document.is_none()); - - let document: Option = index - .document(&reader, None, DocumentId(0)) - .unwrap(); - assert!(document.is_some()); - - let document: Option = index - .document(&reader, None, DocumentId(1)) - .unwrap(); - assert!(document.is_some()); - - reader.abort().unwrap(); - - let mut partial_additions = index.documents_partial_addition(); - - // DocumentId(7900334843754999545) - let partial_doc1 = serde_json::json!({ - "id": 123, - "description": "I am the new Marvin", - }); - - // DocumentId(8367468610878465872) - let partial_doc2 = serde_json::json!({ - "id": 234, - "description": "I am the new Kevin", - }); - - partial_additions.update_document(partial_doc1); - partial_additions.update_document(partial_doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = partial_additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.iter().find(|id| *id == update_id); - - let update_reader = db.update_read_txn().unwrap(); - let result = index.update_status(&update_reader, update_id).unwrap(); - assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none()); - update_reader.abort().unwrap(); - - let reader = db.main_read_txn().unwrap(); - let document: Option = index - .document(&reader, None, DocumentId(0)) - .unwrap(); - - let new_doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "I am the new Marvin", - }); - assert_eq!(document, Some(new_doc1)); - - let document: Option = index - .document(&reader, None, DocumentId(1)) - .unwrap(); - - let new_doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "I am the new Kevin", - }); - assert_eq!(document, Some(new_doc2)); - } - - #[test] - fn delete_index() { - let dir = tempfile::tempdir().unwrap(); - - let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap()); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let db_cloned = database.clone(); - let update_fn = move |name: &str, update: ProcessedUpdateResult| { - // try to open index to trigger a lock - let _ = db_cloned.open_index(name); - sender.send(update.update_id).unwrap() - }; - - // create the index - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "searchableAttributes": ["name", "description"], - "displayedAttributes": ["name", "description"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - // add documents to the index - let mut additions = index.documents_addition(); - - let doc1 = serde_json::json!({ - "id": 123, - "name": "Marvin", - "description": "My name is Marvin", - }); - - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin", - "description": "My name is Kevin", - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // delete the index - let deleted = database.delete_index("test").unwrap(); - assert!(deleted); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let result = database.open_index("test"); - assert!(result.is_none()); - } - - #[test] - fn check_number_ordering() { - let dir = tempfile::tempdir().unwrap(); - - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); - let db = &database; - - let (sender, receiver) = mpsc::sync_channel(100); - let update_fn = move |_name: &str, update: ProcessedUpdateResult| { - sender.send(update.update_id).unwrap() - }; - let index = database.create_index("test").unwrap(); - - database.set_update_callback(Box::new(update_fn)); - - let mut writer = db.main_write_txn().unwrap(); - index.main.put_schema(&mut writer, &Schema::with_primary_key("id")).unwrap(); - writer.commit().unwrap(); - - let settings = { - let data = r#" - { - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(release_date)" - ], - "searchableAttributes": ["name", "release_date"], - "displayedAttributes": ["name", "release_date"] - } - "#; - let settings: Settings = serde_json::from_str(data).unwrap(); - settings.to_update().unwrap() - }; - - let mut writer = db.update_write_txn().unwrap(); - let _update_id = index.settings_update(&mut writer, settings).unwrap(); - writer.commit().unwrap(); - - let mut additions = index.documents_addition(); - - // DocumentId(7900334843754999545) - let doc1 = serde_json::json!({ - "id": 123, - "name": "Kevin the first", - "release_date": -10000, - }); - - // DocumentId(8367468610878465872) - let doc2 = serde_json::json!({ - "id": 234, - "name": "Kevin the second", - "release_date": 10000, - }); - - additions.update_document(doc1); - additions.update_document(doc2); - - let mut writer = db.update_write_txn().unwrap(); - let update_id = additions.finalize(&mut writer).unwrap(); - writer.commit().unwrap(); - - // block until the transaction is processed - let _ = receiver.into_iter().find(|id| *id == update_id); - - let reader = db.main_read_txn().unwrap(); - let schema = index.main.schema(&reader).unwrap().unwrap(); - let ranked_map = index.main.ranked_map(&reader).unwrap().unwrap(); - - let criteria = CriteriaBuilder::new() - .add( - criterion::SortByAttr::lower_is_better(&ranked_map, &schema, "release_date") - .unwrap(), - ) - .add(criterion::DocumentId) - .build(); - - let builder = index.query_builder_with_criteria(criteria); - - let SortResult {documents, .. } = builder.query(&reader, Some("Kevin"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!( - iter.next(), - Some(Document { - id: DocumentId(0), - .. - }) - ); - assert_matches!( - iter.next(), - Some(Document { - id: DocumentId(1), - .. - }) - ); - assert_matches!(iter.next(), None); - } -} diff --git a/meilisearch-core/src/distinct_map.rs b/meilisearch-core/src/distinct_map.rs deleted file mode 100644 index e53592afe..000000000 --- a/meilisearch-core/src/distinct_map.rs +++ /dev/null @@ -1,103 +0,0 @@ -use hashbrown::HashMap; -use std::hash::Hash; - -pub struct DistinctMap { - inner: HashMap, - limit: usize, - len: usize, -} - -impl DistinctMap { - pub fn new(limit: usize) -> Self { - DistinctMap { - inner: HashMap::new(), - limit, - len: 0, - } - } - - pub fn len(&self) -> usize { - self.len - } -} - -pub struct BufferedDistinctMap<'a, K> { - internal: &'a mut DistinctMap, - inner: HashMap, - len: usize, -} - -impl<'a, K: Hash + Eq> BufferedDistinctMap<'a, K> { - pub fn new(internal: &'a mut DistinctMap) -> BufferedDistinctMap<'a, K> { - BufferedDistinctMap { - internal, - inner: HashMap::new(), - len: 0, - } - } - - pub fn register(&mut self, key: K) -> bool { - let internal_seen = self.internal.inner.get(&key).unwrap_or(&0); - let inner_seen = self.inner.entry(key).or_insert(0); - let seen = *internal_seen + *inner_seen; - - if seen < self.internal.limit { - *inner_seen += 1; - self.len += 1; - true - } else { - false - } - } - - pub fn register_without_key(&mut self) -> bool { - self.len += 1; - true - } - - pub fn transfert_to_internal(&mut self) { - for (k, v) in self.inner.drain() { - let value = self.internal.inner.entry(k).or_insert(0); - *value += v; - } - - self.internal.len += self.len; - self.len = 0; - } - - pub fn len(&self) -> usize { - self.internal.len() + self.len - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn easy_distinct_map() { - let mut map = DistinctMap::new(2); - let mut buffered = BufferedDistinctMap::new(&mut map); - - for x in &[1, 1, 1, 2, 3, 4, 5, 6, 6, 6, 6, 6] { - buffered.register(x); - } - buffered.transfert_to_internal(); - assert_eq!(map.len(), 8); - - let mut map = DistinctMap::new(2); - let mut buffered = BufferedDistinctMap::new(&mut map); - assert_eq!(buffered.register(1), true); - assert_eq!(buffered.register(1), true); - assert_eq!(buffered.register(1), false); - assert_eq!(buffered.register(1), false); - - assert_eq!(buffered.register(2), true); - assert_eq!(buffered.register(3), true); - assert_eq!(buffered.register(2), true); - assert_eq!(buffered.register(2), false); - - buffered.transfert_to_internal(); - assert_eq!(map.len(), 5); - } -} diff --git a/meilisearch-core/src/error.rs b/meilisearch-core/src/error.rs deleted file mode 100644 index 1df2419f1..000000000 --- a/meilisearch-core/src/error.rs +++ /dev/null @@ -1,224 +0,0 @@ -use crate::serde::{DeserializerError, SerializerError}; -use serde_json::Error as SerdeJsonError; -use pest::error::Error as PestError; -use crate::filters::Rule; -use std::{error, fmt, io}; - -pub use bincode::Error as BincodeError; -pub use fst::Error as FstError; -pub use heed::Error as HeedError; -pub use pest::error as pest_error; - -use meilisearch_error::{ErrorCode, Code}; - -pub type MResult = Result; - -#[derive(Debug)] -pub enum Error { - Bincode(bincode::Error), - Deserializer(DeserializerError), - FacetError(FacetError), - FilterParseError(PestError), - Fst(fst::Error), - Heed(heed::Error), - IndexAlreadyExists, - Io(io::Error), - MaxFieldsLimitExceeded, - MissingDocumentId, - MissingPrimaryKey, - Schema(meilisearch_schema::Error), - SchemaMissing, - SerdeJson(SerdeJsonError), - Serializer(SerializerError), - VersionMismatch(String), - WordIndexMissing, -} - -impl ErrorCode for Error { - fn error_code(&self) -> Code { - use Error::*; - - match self { - FacetError(_) => Code::Facet, - FilterParseError(_) => Code::Filter, - IndexAlreadyExists => Code::IndexAlreadyExists, - MissingPrimaryKey => Code::MissingPrimaryKey, - MissingDocumentId => Code::MissingDocumentId, - MaxFieldsLimitExceeded => Code::MaxFieldsLimitExceeded, - Schema(s) => s.error_code(), - WordIndexMissing - | SchemaMissing => Code::InvalidState, - Heed(_) - | Fst(_) - | SerdeJson(_) - | Bincode(_) - | Serializer(_) - | Deserializer(_) - | VersionMismatch(_) - | Io(_) => Code::Internal, - } - } -} - -impl From for Error { - fn from(error: io::Error) -> Error { - Error::Io(error) - } -} - -impl From> for Error { - fn from(error: PestError) -> Error { - Error::FilterParseError(error.renamed_rules(|r| { - let s = match r { - Rule::or => "OR", - Rule::and => "AND", - Rule::not => "NOT", - Rule::string => "string", - Rule::word => "word", - Rule::greater => "field > value", - Rule::less => "field < value", - Rule::eq => "field = value", - Rule::leq => "field <= value", - Rule::geq => "field >= value", - Rule::key => "key", - _ => "other", - }; - s.to_string() - })) - } -} - -impl From for Error { - fn from(error: FacetError) -> Error { - Error::FacetError(error) - } -} - -impl From for Error { - fn from(error: meilisearch_schema::Error) -> Error { - Error::Schema(error) - } -} - -impl From for Error { - fn from(error: HeedError) -> Error { - Error::Heed(error) - } -} - -impl From for Error { - fn from(error: FstError) -> Error { - Error::Fst(error) - } -} - -impl From for Error { - fn from(error: SerdeJsonError) -> Error { - Error::SerdeJson(error) - } -} - -impl From for Error { - fn from(error: BincodeError) -> Error { - Error::Bincode(error) - } -} - -impl From for Error { - fn from(error: SerializerError) -> Error { - match error { - SerializerError::DocumentIdNotFound => Error::MissingDocumentId, - e => Error::Serializer(e), - } - } -} - -impl From for Error { - fn from(error: DeserializerError) -> Error { - Error::Deserializer(error) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::Error::*; - match self { - Bincode(e) => write!(f, "bincode error; {}", e), - Deserializer(e) => write!(f, "deserializer error; {}", e), - FacetError(e) => write!(f, "error processing facet filter: {}", e), - FilterParseError(e) => write!(f, "error parsing filter; {}", e), - Fst(e) => write!(f, "fst error; {}", e), - Heed(e) => write!(f, "heed error; {}", e), - IndexAlreadyExists => write!(f, "index already exists"), - Io(e) => write!(f, "{}", e), - MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"), - MissingDocumentId => write!(f, "document id is missing"), - MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"), - Schema(e) => write!(f, "schema error; {}", e), - SchemaMissing => write!(f, "this index does not have a schema"), - SerdeJson(e) => write!(f, "serde json error; {}", e), - Serializer(e) => write!(f, "serializer error; {}", e), - VersionMismatch(version) => write!(f, "Cannot open database, expected MeiliSearch engine version: {}, current engine version: {}.{}.{}", - version, - env!("CARGO_PKG_VERSION_MAJOR"), - env!("CARGO_PKG_VERSION_MINOR"), - env!("CARGO_PKG_VERSION_PATCH")), - WordIndexMissing => write!(f, "this index does not have a word index"), - } - } -} - -impl error::Error for Error {} - -struct FilterParseError(PestError); - -impl fmt::Display for FilterParseError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use crate::pest_error::LineColLocation::*; - - let (line, column) = match self.0.line_col { - Span((line, _), (column, _)) => (line, column), - Pos((line, column)) => (line, column), - }; - write!(f, "parsing error on line {} at column {}: {}", line, column, self.0.variant.message()) - } -} - -#[derive(Debug)] -pub enum FacetError { - EmptyArray, - ParsingError(String), - UnexpectedToken { expected: &'static [&'static str], found: String }, - InvalidFormat(String), - AttributeNotFound(String), - AttributeNotSet { expected: Vec, found: String }, - InvalidDocumentAttribute(String), - NoAttributesForFaceting, -} - -impl FacetError { - pub fn unexpected_token(expected: &'static [&'static str], found: impl ToString) -> FacetError { - FacetError::UnexpectedToken{ expected, found: found.to_string() } - } - - pub fn attribute_not_set(expected: Vec, found: impl ToString) -> FacetError { - FacetError::AttributeNotSet{ expected, found: found.to_string() } - } -} - -impl fmt::Display for FacetError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use FacetError::*; - - match self { - EmptyArray => write!(f, "empty array in facet filter is unspecified behavior"), - ParsingError(msg) => write!(f, "parsing error: {}", msg), - UnexpectedToken { expected, found } => write!(f, "unexpected token {}, expected {}", found, expected.join("or")), - InvalidFormat(found) => write!(f, "invalid facet: {}, facets should be \"facetName:facetValue\"", found), - AttributeNotFound(attr) => write!(f, "unknown {:?} attribute", attr), - AttributeNotSet { found, expected } => write!(f, "`{}` is not set as a faceted attribute. available facet attributes: {}", found, expected.join(", ")), - InvalidDocumentAttribute(attr) => write!(f, "invalid document attribute {}, accepted types: String and [String]", attr), - NoAttributesForFaceting => write!(f, "impossible to perform faceted search, no attributes for faceting are set"), - } - } -} diff --git a/meilisearch-core/src/facets.rs b/meilisearch-core/src/facets.rs deleted file mode 100644 index c4689ee87..000000000 --- a/meilisearch-core/src/facets.rs +++ /dev/null @@ -1,357 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::hash::Hash; -use std::ops::Deref; - -use cow_utils::CowUtils; -use either::Either; -use heed::types::{Str, OwnedType}; -use indexmap::IndexMap; -use serde_json::Value; - -use meilisearch_schema::{FieldId, Schema}; -use meilisearch_types::DocumentId; - -use crate::database::MainT; -use crate::error::{FacetError, MResult}; -use crate::store::BEU16; - -/// Data structure used to represent a boolean expression in the form of nested arrays. -/// Values in the outer array are and-ed together, values in the inner arrays are or-ed together. -#[derive(Debug, PartialEq)] -pub struct FacetFilter(Vec, FacetKey>>); - -impl Deref for FacetFilter { - type Target = Vec, FacetKey>>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl FacetFilter { - pub fn from_str( - s: &str, - schema: &Schema, - attributes_for_faceting: &[FieldId], - ) -> MResult { - if attributes_for_faceting.is_empty() { - return Err(FacetError::NoAttributesForFaceting.into()); - } - let parsed = serde_json::from_str::(s).map_err(|e| FacetError::ParsingError(e.to_string()))?; - let mut filter = Vec::new(); - match parsed { - Value::Array(and_exprs) => { - if and_exprs.is_empty() { - return Err(FacetError::EmptyArray.into()); - } - for expr in and_exprs { - match expr { - Value::String(s) => { - let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?; - filter.push(Either::Right(key)); - } - Value::Array(or_exprs) => { - if or_exprs.is_empty() { - return Err(FacetError::EmptyArray.into()); - } - let mut inner = Vec::new(); - for expr in or_exprs { - match expr { - Value::String(s) => { - let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?; - inner.push(key); - } - bad_value => return Err(FacetError::unexpected_token(&["String"], bad_value).into()), - } - } - filter.push(Either::Left(inner)); - } - bad_value => return Err(FacetError::unexpected_token(&["Array", "String"], bad_value).into()), - } - } - Ok(Self(filter)) - } - bad_value => Err(FacetError::unexpected_token(&["Array"], bad_value).into()), - } - } -} - -#[derive(Debug, Eq, PartialEq, Hash)] -#[repr(C)] -pub struct FacetKey(FieldId, String); - -impl FacetKey { - pub fn new(field_id: FieldId, value: String) -> Self { - let value = match value.cow_to_lowercase() { - Cow::Borrowed(_) => value, - Cow::Owned(s) => s, - }; - Self(field_id, value) - } - - pub fn key(&self) -> FieldId { - self.0 - } - - pub fn value(&self) -> &str { - &self.1 - } - - // TODO improve parser - fn from_str( - s: &str, - schema: &Schema, - attributes_for_faceting: &[FieldId], - ) -> Result { - let mut split = s.splitn(2, ':'); - let key = split - .next() - .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))? - .trim(); - let field_id = schema - .id(key) - .ok_or_else(|| FacetError::AttributeNotFound(key.to_string()))?; - - if !attributes_for_faceting.contains(&field_id) { - return Err(FacetError::attribute_not_set( - attributes_for_faceting - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect::>(), - key)) - } - let value = split - .next() - .ok_or_else(|| FacetError::InvalidFormat(s.to_string()))? - .trim(); - // unquoting the string if need be: - let mut indices = value.char_indices(); - let value = match (indices.next(), indices.last()) { - (Some((s, '\'')), Some((e, '\''))) | - (Some((s, '\"')), Some((e, '\"'))) => value[s + 1..e].to_string(), - _ => value.to_string(), - }; - Ok(Self::new(field_id, value)) - } -} - -impl<'a> heed::BytesEncode<'a> for FacetKey { - type EItem = FacetKey; - - fn bytes_encode(item: &'a Self::EItem) -> Option> { - let mut buffer = Vec::with_capacity(2 + item.1.len()); - let id = BEU16::new(item.key().into()); - let id_bytes = OwnedType::bytes_encode(&id)?; - let value_bytes = Str::bytes_encode(item.value())?; - buffer.extend_from_slice(id_bytes.as_ref()); - buffer.extend_from_slice(value_bytes.as_ref()); - Some(Cow::Owned(buffer)) - } -} - -impl<'a> heed::BytesDecode<'a> for FacetKey { - type DItem = FacetKey; - - fn bytes_decode(bytes: &'a [u8]) -> Option { - let (id_bytes, value_bytes) = bytes.split_at(2); - let id = OwnedType::::bytes_decode(id_bytes)?; - let id = id.get().into(); - let string = Str::bytes_decode(&value_bytes)?; - Some(FacetKey(id, string.to_string())) - } -} - -pub fn add_to_facet_map( - facet_map: &mut HashMap)>, - field_id: FieldId, - value: Value, - document_id: DocumentId, -) -> Result<(), FacetError> { - let value = match value { - Value::String(s) => s, - // ignore null - Value::Null => return Ok(()), - value => return Err(FacetError::InvalidDocumentAttribute(value.to_string())), - }; - let key = FacetKey::new(field_id, value.clone()); - facet_map.entry(key).or_insert_with(|| (value, Vec::new())).1.push(document_id); - Ok(()) -} - -pub fn facet_map_from_docids( - rtxn: &heed::RoTxn, - index: &crate::Index, - document_ids: &[DocumentId], - attributes_for_facetting: &[FieldId], -) -> MResult)>> { - // A hashmap that ascociate a facet key to a pair containing the original facet attribute - // string with it's case preserved, and a list of document ids for that facet attribute. - let mut facet_map: HashMap)> = HashMap::new(); - for document_id in document_ids { - for result in index - .documents_fields - .document_fields(rtxn, *document_id)? - { - let (field_id, bytes) = result?; - if attributes_for_facetting.contains(&field_id) { - match serde_json::from_slice(bytes)? { - Value::Array(values) => { - for v in values { - add_to_facet_map(&mut facet_map, field_id, v, *document_id)?; - } - } - v => add_to_facet_map(&mut facet_map, field_id, v, *document_id)?, - }; - } - } - } - Ok(facet_map) -} - -pub fn facet_map_from_docs( - schema: &Schema, - documents: &HashMap>, - attributes_for_facetting: &[FieldId], -) -> MResult)>> { - let mut facet_map = HashMap::new(); - let attributes_for_facetting = attributes_for_facetting - .iter() - .filter_map(|&id| schema.name(id).map(|name| (id, name))) - .collect::>(); - - for (id, document) in documents { - for (field_id, name) in &attributes_for_facetting { - if let Some(value) = document.get(*name) { - match value { - Value::Array(values) => { - for v in values { - add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?; - } - } - v => add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?, - } - } - } - } - Ok(facet_map) -} - -#[cfg(test)] -mod test { - use super::*; - use meilisearch_schema::Schema; - - #[test] - fn test_facet_key() { - let mut schema = Schema::default(); - let id = schema.insert_with_position("hello").unwrap().0; - let facet_list = [schema.id("hello").unwrap()]; - assert_eq!( - FacetKey::from_str("hello:12", &schema, &facet_list).unwrap(), - FacetKey::new(id, "12".to_string()) - ); - assert_eq!( - FacetKey::from_str("hello:\"foo bar\"", &schema, &facet_list).unwrap(), - FacetKey::new(id, "foo bar".to_string()) - ); - assert_eq!( - FacetKey::from_str("hello:'foo bar'", &schema, &facet_list).unwrap(), - FacetKey::new(id, "foo bar".to_string()) - ); - // weird case - assert_eq!( - FacetKey::from_str("hello:blabla:machin", &schema, &facet_list).unwrap(), - FacetKey::new(id, "blabla:machin".to_string()) - ); - - assert_eq!( - FacetKey::from_str("hello:\"\"", &schema, &facet_list).unwrap(), - FacetKey::new(id, "".to_string()) - ); - - assert_eq!( - FacetKey::from_str("hello:'", &schema, &facet_list).unwrap(), - FacetKey::new(id, "'".to_string()) - ); - assert_eq!( - FacetKey::from_str("hello:''", &schema, &facet_list).unwrap(), - FacetKey::new(id, "".to_string()) - ); - assert!(FacetKey::from_str("hello", &schema, &facet_list).is_err()); - assert!(FacetKey::from_str("toto:12", &schema, &facet_list).is_err()); - } - - #[test] - fn test_parse_facet_array() { - use either::Either::{Left, Right}; - let mut schema = Schema::default(); - let _id = schema.insert_with_position("hello").unwrap(); - let facet_list = [schema.id("hello").unwrap()]; - assert_eq!( - FacetFilter::from_str("[[\"hello:12\"]]", &schema, &facet_list).unwrap(), - FacetFilter(vec![Left(vec![FacetKey(FieldId(0), "12".to_string())])]) - ); - assert_eq!( - FacetFilter::from_str("[\"hello:12\"]", &schema, &facet_list).unwrap(), - FacetFilter(vec![Right(FacetKey(FieldId(0), "12".to_string()))]) - ); - assert_eq!( - FacetFilter::from_str("[\"hello:12\", \"hello:13\"]", &schema, &facet_list).unwrap(), - FacetFilter(vec![ - Right(FacetKey(FieldId(0), "12".to_string())), - Right(FacetKey(FieldId(0), "13".to_string())) - ]) - ); - assert_eq!( - FacetFilter::from_str("[[\"hello:12\", \"hello:13\"]]", &schema, &facet_list).unwrap(), - FacetFilter(vec![Left(vec![ - FacetKey(FieldId(0), "12".to_string()), - FacetKey(FieldId(0), "13".to_string()) - ])]) - ); - assert_eq!( - FacetFilter::from_str( - "[[\"hello:12\", \"hello:13\"], \"hello:14\"]", - &schema, - &facet_list - ) - .unwrap(), - FacetFilter(vec![ - Left(vec![ - FacetKey(FieldId(0), "12".to_string()), - FacetKey(FieldId(0), "13".to_string()) - ]), - Right(FacetKey(FieldId(0), "14".to_string())) - ]) - ); - - // invalid array depths - assert!(FacetFilter::from_str( - "[[[\"hello:12\", \"hello:13\"], \"hello:14\"]]", - &schema, - &facet_list - ) - .is_err()); - assert!(FacetFilter::from_str( - "[[[\"hello:12\", \"hello:13\"]], \"hello:14\"]]", - &schema, - &facet_list - ) - .is_err()); - assert!(FacetFilter::from_str("\"hello:14\"", &schema, &facet_list).is_err()); - - // unexisting key - assert!(FacetFilter::from_str("[\"foo:12\"]", &schema, &facet_list).is_err()); - - // invalid facet key - assert!(FacetFilter::from_str("[\"foo=12\"]", &schema, &facet_list).is_err()); - assert!(FacetFilter::from_str("[\"foo12\"]", &schema, &facet_list).is_err()); - assert!(FacetFilter::from_str("[\"\"]", &schema, &facet_list).is_err()); - - // empty array error - assert!(FacetFilter::from_str("[]", &schema, &facet_list).is_err()); - assert!(FacetFilter::from_str("[\"hello:12\", []]", &schema, &facet_list).is_err()); - } -} diff --git a/meilisearch-core/src/filters/condition.rs b/meilisearch-core/src/filters/condition.rs deleted file mode 100644 index d22f9d905..000000000 --- a/meilisearch-core/src/filters/condition.rs +++ /dev/null @@ -1,276 +0,0 @@ -use std::str::FromStr; -use std::cmp::Ordering; - -use crate::error::Error; -use crate::{store::Index, DocumentId, MainT}; -use heed::RoTxn; -use meilisearch_schema::{FieldId, Schema}; -use pest::error::{Error as PestError, ErrorVariant}; -use pest::iterators::Pair; -use serde_json::{Value, Number}; -use super::parser::Rule; - -#[derive(Debug, PartialEq)] -enum ConditionType { - Greater, - Less, - Equal, - LessEqual, - GreaterEqual, - NotEqual, -} - -/// We need to infer type when the filter is constructed -/// and match every possible types it can be parsed into. -#[derive(Debug)] -struct ConditionValue<'a> { - string: &'a str, - boolean: Option, - number: Option -} - -impl<'a> ConditionValue<'a> { - pub fn new(value: &Pair<'a, Rule>) -> Self { - match value.as_rule() { - Rule::string | Rule::word => { - let string = value.as_str(); - let boolean = match value.as_str() { - "true" => Some(true), - "false" => Some(false), - _ => None, - }; - let number = Number::from_str(value.as_str()).ok(); - ConditionValue { string, boolean, number } - }, - _ => unreachable!(), - } - } - - pub fn as_str(&self) -> &str { - self.string - } - - pub fn as_number(&self) -> Option<&Number> { - self.number.as_ref() - } - - pub fn as_bool(&self) -> Option { - self.boolean - } -} - -#[derive(Debug)] -pub struct Condition<'a> { - field: FieldId, - condition: ConditionType, - value: ConditionValue<'a> -} - -fn get_field_value<'a>(schema: &Schema, pair: Pair<'a, Rule>) -> Result<(FieldId, ConditionValue<'a>), Error> { - let mut items = pair.into_inner(); - // lexing ensures that we at least have a key - let key = items.next().unwrap(); - let field = schema - .id(key.as_str()) - .ok_or_else(|| PestError::new_from_span( - ErrorVariant::CustomError { - message: format!( - "attribute `{}` not found, available attributes are: {}", - key.as_str(), - schema.names().collect::>().join(", ") - ), - }, - key.as_span()))?; - let value = ConditionValue::new(&items.next().unwrap()); - Ok((field, value)) -} - -// undefined behavior with big numbers -fn compare_numbers(lhs: &Number, rhs: &Number) -> Option { - match (lhs.as_i64(), lhs.as_u64(), lhs.as_f64(), - rhs.as_i64(), rhs.as_u64(), rhs.as_f64()) { - // i64 u64 f64 i64 u64 f64 - (Some(lhs), _, _, Some(rhs), _, _) => lhs.partial_cmp(&rhs), - (_, Some(lhs), _, _, Some(rhs), _) => lhs.partial_cmp(&rhs), - (_, _, Some(lhs), _, _, Some(rhs)) => lhs.partial_cmp(&rhs), - (_, _, _, _, _, _) => None, - } -} - -impl<'a> Condition<'a> { - pub fn less( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::Less; - Ok(Self { field, condition, value }) - } - - pub fn greater( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::Greater; - Ok(Self { field, condition, value }) - } - - pub fn neq( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::NotEqual; - Ok(Self { field, condition, value }) - } - - pub fn geq( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::GreaterEqual; - Ok(Self { field, condition, value }) - } - - pub fn leq( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::LessEqual; - Ok(Self { field, condition, value }) - } - - pub fn eq( - item: Pair<'a, Rule>, - schema: &'a Schema, - ) -> Result { - let (field, value) = get_field_value(schema, item)?; - let condition = ConditionType::Equal; - Ok(Self { field, condition, value }) - } - - pub fn test( - &self, - reader: &RoTxn, - index: &Index, - document_id: DocumentId, - ) -> Result { - match index.document_attribute::(reader, document_id, self.field)? { - Some(Value::Array(values)) => Ok(values.iter().any(|v| self.match_value(Some(v)))), - other => Ok(self.match_value(other.as_ref())), - } - } - - fn match_value(&self, value: Option<&Value>) -> bool { - match value { - Some(Value::String(s)) => { - let value = self.value.as_str(); - match self.condition { - ConditionType::Equal => unicase::eq(value, &s), - ConditionType::NotEqual => !unicase::eq(value, &s), - _ => false - } - }, - Some(Value::Number(n)) => { - if let Some(value) = self.value.as_number() { - if let Some(ord) = compare_numbers(&n, value) { - let res = match self.condition { - ConditionType::Equal => ord == Ordering::Equal, - ConditionType::NotEqual => ord != Ordering::Equal, - ConditionType::GreaterEqual => ord != Ordering::Less, - ConditionType::LessEqual => ord != Ordering::Greater, - ConditionType::Greater => ord == Ordering::Greater, - ConditionType::Less => ord == Ordering::Less, - }; - return res - } - } - false - }, - Some(Value::Bool(b)) => { - if let Some(value) = self.value.as_bool() { - let res = match self.condition { - ConditionType::Equal => *b == value, - ConditionType::NotEqual => *b != value, - _ => false - }; - return res - } - false - }, - // if field is not supported (or not found), all values are different from it, - // so != should always return true in this case. - _ => self.condition == ConditionType::NotEqual, - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use serde_json::Number; - use std::cmp::Ordering; - - #[test] - fn test_number_comp() { - // test both u64 - let n1 = Number::from(1u64); - let n2 = Number::from(2u64); - assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1)); - let n1 = Number::from(1u64); - let n2 = Number::from(1u64); - assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2)); - - // test both i64 - let n1 = Number::from(1i64); - let n2 = Number::from(2i64); - assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1)); - let n1 = Number::from(1i64); - let n2 = Number::from(1i64); - assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2)); - - // test both f64 - let n1 = Number::from_f64(1f64).unwrap(); - let n2 = Number::from_f64(2f64).unwrap(); - assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1)); - let n1 = Number::from_f64(1f64).unwrap(); - let n2 = Number::from_f64(1f64).unwrap(); - assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2)); - - // test one u64 and one f64 - let n1 = Number::from_f64(1f64).unwrap(); - let n2 = Number::from(2u64); - assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1)); - - // equality - let n1 = Number::from_f64(1f64).unwrap(); - let n2 = Number::from(1u64); - assert_eq!(Some(Ordering::Equal), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Equal), compare_numbers(&n2, &n1)); - - // float is neg - let n1 = Number::from_f64(-1f64).unwrap(); - let n2 = Number::from(1u64); - assert_eq!(Some(Ordering::Less), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n2, &n1)); - - // float is too big - let n1 = Number::from_f64(std::f64::MAX).unwrap(); - let n2 = Number::from(1u64); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2)); - assert_eq!(Some(Ordering::Less), compare_numbers(&n2, &n1)); - - // misc - let n1 = Number::from_f64(std::f64::MAX).unwrap(); - let n2 = Number::from(std::u64::MAX); - assert_eq!(Some(Ordering::Greater), compare_numbers(&n1, &n2)); - assert_eq!(Some( Ordering::Less ), compare_numbers(&n2, &n1)); - } -} diff --git a/meilisearch-core/src/filters/mod.rs b/meilisearch-core/src/filters/mod.rs deleted file mode 100644 index ea2090a09..000000000 --- a/meilisearch-core/src/filters/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -mod parser; -mod condition; - -pub(crate) use parser::Rule; - -use std::ops::Not; - -use condition::Condition; -use crate::error::Error; -use crate::{DocumentId, MainT, store::Index}; -use heed::RoTxn; -use meilisearch_schema::Schema; -use parser::{PREC_CLIMBER, FilterParser}; -use pest::iterators::{Pair, Pairs}; -use pest::Parser; - -type FilterResult<'a> = Result, Error>; - -#[derive(Debug)] -pub enum Filter<'a> { - Condition(Condition<'a>), - Or(Box, Box), - And(Box, Box), - Not(Box), -} - -impl<'a> Filter<'a> { - pub fn parse(expr: &'a str, schema: &'a Schema) -> FilterResult<'a> { - let mut lexed = FilterParser::parse(Rule::prgm, expr)?; - Self::build(lexed.next().unwrap().into_inner(), schema) - } - - pub fn test( - &self, - reader: &RoTxn, - index: &Index, - document_id: DocumentId, - ) -> Result { - use Filter::*; - match self { - Condition(c) => c.test(reader, index, document_id), - Or(lhs, rhs) => Ok( - lhs.test(reader, index, document_id)? || rhs.test(reader, index, document_id)? - ), - And(lhs, rhs) => Ok( - lhs.test(reader, index, document_id)? && rhs.test(reader, index, document_id)? - ), - Not(op) => op.test(reader, index, document_id).map(bool::not), - } - } - - fn build(expression: Pairs<'a, Rule>, schema: &'a Schema) -> FilterResult<'a> { - PREC_CLIMBER.climb( - expression, - |pair: Pair| match pair.as_rule() { - Rule::eq => Ok(Filter::Condition(Condition::eq(pair, schema)?)), - Rule::greater => Ok(Filter::Condition(Condition::greater(pair, schema)?)), - Rule::less => Ok(Filter::Condition(Condition::less(pair, schema)?)), - Rule::neq => Ok(Filter::Condition(Condition::neq(pair, schema)?)), - Rule::geq => Ok(Filter::Condition(Condition::geq(pair, schema)?)), - Rule::leq => Ok(Filter::Condition(Condition::leq(pair, schema)?)), - Rule::prgm => Self::build(pair.into_inner(), schema), - Rule::term => Self::build(pair.into_inner(), schema), - Rule::not => Ok(Filter::Not(Box::new(Self::build( - pair.into_inner(), - schema, - )?))), - _ => unreachable!(), - }, - |lhs: FilterResult, op: Pair, rhs: FilterResult| match op.as_rule() { - Rule::or => Ok(Filter::Or(Box::new(lhs?), Box::new(rhs?))), - Rule::and => Ok(Filter::And(Box::new(lhs?), Box::new(rhs?))), - _ => unreachable!(), - }, - ) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn invalid_syntax() { - assert!(FilterParser::parse(Rule::prgm, "field : id").is_err()); - assert!(FilterParser::parse(Rule::prgm, "field=hello hello").is_err()); - assert!(FilterParser::parse(Rule::prgm, "field=hello OR OR").is_err()); - assert!(FilterParser::parse(Rule::prgm, "OR field:hello").is_err()); - assert!(FilterParser::parse(Rule::prgm, r#"field="hello world"#).is_err()); - assert!(FilterParser::parse(Rule::prgm, r#"field='hello world"#).is_err()); - assert!(FilterParser::parse(Rule::prgm, "NOT field=").is_err()); - assert!(FilterParser::parse(Rule::prgm, "N").is_err()); - assert!(FilterParser::parse(Rule::prgm, "(field=1").is_err()); - assert!(FilterParser::parse(Rule::prgm, "(field=1))").is_err()); - assert!(FilterParser::parse(Rule::prgm, "field=1ORfield=2").is_err()); - assert!(FilterParser::parse(Rule::prgm, "field=1 ( OR field=2)").is_err()); - assert!(FilterParser::parse(Rule::prgm, "hello world=1").is_err()); - assert!(FilterParser::parse(Rule::prgm, "").is_err()); - assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world)))))"#).is_err()); - } - - #[test] - fn valid_syntax() { - assert!(FilterParser::parse(Rule::prgm, "field = id").is_ok()); - assert!(FilterParser::parse(Rule::prgm, "field=id").is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field >= 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field <= 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field="hello world""#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field='hello world'"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field > 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field < 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field=5"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field < 10 AND NOT field > 7.5"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field=true OR NOT field=5"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"NOT field=true OR NOT field=5"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field='hello world' OR ( NOT field=true OR NOT field=5 )"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field='hello \'worl\'d' OR ( NOT field=true OR NOT field=5 )"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"field="hello \"worl\"d" OR ( NOT field=true OR NOT field=5 )"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"((((((hello=world))))))"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#""foo bar" > 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#""foo bar" = 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' = 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' <= 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"'foo bar' != 10"#).is_ok()); - assert!(FilterParser::parse(Rule::prgm, r#"bar != 10"#).is_ok()); - } -} diff --git a/meilisearch-core/src/filters/parser/grammar.pest b/meilisearch-core/src/filters/parser/grammar.pest deleted file mode 100644 index e7095bb63..000000000 --- a/meilisearch-core/src/filters/parser/grammar.pest +++ /dev/null @@ -1,28 +0,0 @@ -key = _{quoted | word} -value = _{quoted | word} -quoted = _{ (PUSH("'") | PUSH("\"")) ~ string ~ POP } -string = {char*} -word = ${(LETTER | NUMBER | "_" | "-" | ".")+} - -char = _{ !(PEEK | "\\") ~ ANY - | "\\" ~ (PEEK | "\\" | "/" | "b" | "f" | "n" | "r" | "t") - | "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4})} - -condition = _{eq | greater | less | geq | leq | neq} -geq = {key ~ ">=" ~ value} -leq = {key ~ "<=" ~ value} -neq = {key ~ "!=" ~ value} -eq = {key ~ "=" ~ value} -greater = {key ~ ">" ~ value} -less = {key ~ "<" ~ value} - -prgm = {SOI ~ expr ~ EOI} -expr = _{ ( term ~ (operation ~ term)* ) } -term = { ("(" ~ expr ~ ")") | condition | not } -operation = _{ and | or } - and = {"AND"} - or = {"OR"} - -not = {"NOT" ~ term} - -WHITESPACE = _{ " " } diff --git a/meilisearch-core/src/filters/parser/mod.rs b/meilisearch-core/src/filters/parser/mod.rs deleted file mode 100644 index e8f69d0dd..000000000 --- a/meilisearch-core/src/filters/parser/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -use once_cell::sync::Lazy; -use pest::prec_climber::{Operator, Assoc, PrecClimber}; - -pub static PREC_CLIMBER: Lazy> = Lazy::new(|| { - use Assoc::*; - use Rule::*; - pest::prec_climber::PrecClimber::new(vec![Operator::new(or, Left), Operator::new(and, Left)]) -}); - -#[derive(Parser)] -#[grammar = "filters/parser/grammar.pest"] -pub struct FilterParser; diff --git a/meilisearch-core/src/levenshtein.rs b/meilisearch-core/src/levenshtein.rs deleted file mode 100644 index 6e781b550..000000000 --- a/meilisearch-core/src/levenshtein.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::cmp::min; -use std::collections::BTreeMap; -use std::ops::{Index, IndexMut}; - -// A simple wrapper around vec so we can get contiguous but index it like it's 2D array. -struct N2Array { - y_size: usize, - buf: Vec, -} - -impl N2Array { - fn new(x: usize, y: usize, value: T) -> N2Array { - N2Array { - y_size: y, - buf: vec![value; x * y], - } - } -} - -impl Index<(usize, usize)> for N2Array { - type Output = T; - - #[inline] - fn index(&self, (x, y): (usize, usize)) -> &T { - &self.buf[(x * self.y_size) + y] - } -} - -impl IndexMut<(usize, usize)> for N2Array { - #[inline] - fn index_mut(&mut self, (x, y): (usize, usize)) -> &mut T { - &mut self.buf[(x * self.y_size) + y] - } -} - -pub fn prefix_damerau_levenshtein(source: &[u8], target: &[u8]) -> (u32, usize) { - let (n, m) = (source.len(), target.len()); - - assert!( - n <= m, - "the source string must be shorter than the target one" - ); - - if n == 0 { - return (m as u32, 0); - } - if m == 0 { - return (n as u32, 0); - } - - if n == m && source == target { - return (0, m); - } - - let inf = n + m; - let mut matrix = N2Array::new(n + 2, m + 2, 0); - - matrix[(0, 0)] = inf; - for i in 0..n + 1 { - matrix[(i + 1, 0)] = inf; - matrix[(i + 1, 1)] = i; - } - for j in 0..m + 1 { - matrix[(0, j + 1)] = inf; - matrix[(1, j + 1)] = j; - } - - let mut last_row = BTreeMap::new(); - - for (row, char_s) in source.iter().enumerate() { - let mut last_match_col = 0; - let row = row + 1; - - for (col, char_t) in target.iter().enumerate() { - let col = col + 1; - let last_match_row = *last_row.get(&char_t).unwrap_or(&0); - let cost = if char_s == char_t { 0 } else { 1 }; - - let dist_add = matrix[(row, col + 1)] + 1; - let dist_del = matrix[(row + 1, col)] + 1; - let dist_sub = matrix[(row, col)] + cost; - let dist_trans = matrix[(last_match_row, last_match_col)] - + (row - last_match_row - 1) - + 1 - + (col - last_match_col - 1); - - let dist = min(min(dist_add, dist_del), min(dist_sub, dist_trans)); - - matrix[(row + 1, col + 1)] = dist; - - if cost == 0 { - last_match_col = col; - } - } - - last_row.insert(char_s, row); - } - - let mut minimum = (u32::max_value(), 0); - - for x in n..=m { - let dist = matrix[(n + 1, x + 1)] as u32; - if dist < minimum.0 { - minimum = (dist, x) - } - } - - minimum -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn matched_length() { - let query = "Levenste"; - let text = "Levenshtein"; - - let (dist, length) = prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes()); - assert_eq!(dist, 1); - assert_eq!(&text[..length], "Levenshte"); - } - - #[test] - #[should_panic] - fn matched_length_panic() { - let query = "Levenshtein"; - let text = "Levenste"; - - // this function will panic if source if longer than target - prefix_damerau_levenshtein(query.as_bytes(), text.as_bytes()); - } -} diff --git a/meilisearch-core/src/lib.rs b/meilisearch-core/src/lib.rs deleted file mode 100644 index 947ad5fb7..000000000 --- a/meilisearch-core/src/lib.rs +++ /dev/null @@ -1,203 +0,0 @@ -#![allow(clippy::type_complexity)] - -#[cfg(test)] -#[macro_use] -extern crate assert_matches; -#[macro_use] -extern crate pest_derive; - -mod automaton; -mod bucket_sort; -mod database; -mod distinct_map; -mod error; -mod filters; -mod levenshtein; -mod number; -mod query_builder; -mod query_tree; -mod query_words_mapper; -mod ranked_map; -mod raw_document; -mod reordered_attrs; -pub mod criterion; -pub mod facets; -pub mod raw_indexer; -pub mod serde; -pub mod settings; -pub mod store; -pub mod update; - -pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT, MainWriter, MainReader, UpdateWriter, UpdateReader}; -pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError}; -pub use self::filters::Filter; -pub use self::number::{Number, ParseNumberError}; -pub use self::ranked_map::RankedMap; -pub use self::raw_document::RawDocument; -pub use self::store::Index; -pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType}; -pub use meilisearch_types::{DocIndex, DocumentId, Highlight}; -pub use meilisearch_schema::Schema; -pub use query_words_mapper::QueryWordsMapper; -pub use query_tree::MAX_QUERY_LEN; - -use compact_arena::SmallArena; -use log::{error, trace}; -use std::borrow::Cow; -use std::collections::HashMap; -use std::convert::TryFrom; - -use crate::bucket_sort::PostingsListView; -use crate::levenshtein::prefix_damerau_levenshtein; -use crate::query_tree::{QueryId, QueryKind}; -use crate::reordered_attrs::ReorderedAttrs; - -type FstSetCow<'a> = fst::Set>; -type FstMapCow<'a> = fst::Map>; - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Document { - pub id: DocumentId, - pub highlights: Vec, - - #[cfg(test)] - pub matches: Vec, -} - -fn highlights_from_raw_document<'a, 'tag, 'txn>( - raw_document: &RawDocument<'a, 'tag>, - queries_kinds: &HashMap, - arena: &SmallArena<'tag, PostingsListView<'txn>>, - searchable_attrs: Option<&ReorderedAttrs>, - schema: &Schema, -) -> Vec -{ - let mut highlights = Vec::new(); - - for bm in raw_document.bare_matches.iter() { - let postings_list = &arena[bm.postings_list]; - let input = postings_list.input(); - let kind = &queries_kinds.get(&bm.query_index); - - for di in postings_list.iter() { - let covered_area = match kind { - Some(QueryKind::NonTolerant(query)) | Some(QueryKind::Tolerant(query)) => { - let len = if query.len() > input.len() { - input.len() - } else { - prefix_damerau_levenshtein(query.as_bytes(), input).1 - }; - u16::try_from(len).unwrap_or(u16::max_value()) - }, - _ => di.char_length, - }; - - let attribute = searchable_attrs - .and_then(|sa| sa.reverse(di.attribute)) - .unwrap_or(di.attribute); - - let attribute = match schema.indexed_pos_to_field_id(attribute) { - Some(field_id) => field_id.0, - None => { - error!("Cannot convert indexed_pos {} to field_id", attribute); - trace!("Schema is compromized; {:?}", schema); - continue - } - }; - - let highlight = Highlight { - attribute, - char_index: di.char_index, - char_length: covered_area, - }; - - highlights.push(highlight); - } - } - - highlights -} - -impl Document { - #[cfg(not(test))] - pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document { - Document { id, highlights: highlights.to_owned() } - } - - #[cfg(test)] - pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document { - Document { id, highlights: highlights.to_owned(), matches: Vec::new() } - } - - #[cfg(not(test))] - pub fn from_raw<'a, 'tag, 'txn>( - raw_document: RawDocument<'a, 'tag>, - queries_kinds: &HashMap, - arena: &SmallArena<'tag, PostingsListView<'txn>>, - searchable_attrs: Option<&ReorderedAttrs>, - schema: &Schema, - ) -> Document - { - let highlights = highlights_from_raw_document( - &raw_document, - queries_kinds, - arena, - searchable_attrs, - schema, - ); - - Document { id: raw_document.id, highlights } - } - - #[cfg(test)] - pub fn from_raw<'a, 'tag, 'txn>( - raw_document: RawDocument<'a, 'tag>, - queries_kinds: &HashMap, - arena: &SmallArena<'tag, PostingsListView<'txn>>, - searchable_attrs: Option<&ReorderedAttrs>, - schema: &Schema, - ) -> Document - { - use crate::bucket_sort::SimpleMatch; - - let highlights = highlights_from_raw_document( - &raw_document, - queries_kinds, - arena, - searchable_attrs, - schema, - ); - - let mut matches = Vec::new(); - for sm in raw_document.processed_matches { - let attribute = searchable_attrs - .and_then(|sa| sa.reverse(sm.attribute)) - .unwrap_or(sm.attribute); - - let attribute = match schema.indexed_pos_to_field_id(attribute) { - Some(field_id) => field_id.0, - None => { - error!("Cannot convert indexed_pos {} to field_id", attribute); - trace!("Schema is compromized; {:?}", schema); - continue - } - }; - - matches.push(SimpleMatch { attribute, ..sm }); - } - matches.sort_unstable(); - - Document { id: raw_document.id, highlights, matches } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::mem; - - #[test] - fn docindex_mem_size() { - assert_eq!(mem::size_of::(), 12); - } -} diff --git a/meilisearch-core/src/number.rs b/meilisearch-core/src/number.rs deleted file mode 100644 index 38f7ca975..000000000 --- a/meilisearch-core/src/number.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::cmp::Ordering; -use std::fmt; -use std::num::{ParseFloatError, ParseIntError}; -use std::str::FromStr; - -use ordered_float::OrderedFloat; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Debug, Copy, Clone)] -pub enum Number { - Unsigned(u64), - Signed(i64), - Float(OrderedFloat), - Null, -} - -impl Default for Number { - fn default() -> Self { - Self::Null - } -} - -impl FromStr for Number { - type Err = ParseNumberError; - - fn from_str(s: &str) -> Result { - let uint_error = match u64::from_str(s) { - Ok(unsigned) => return Ok(Number::Unsigned(unsigned)), - Err(error) => error, - }; - - let int_error = match i64::from_str(s) { - Ok(signed) => return Ok(Number::Signed(signed)), - Err(error) => error, - }; - - let float_error = match f64::from_str(s) { - Ok(float) => return Ok(Number::Float(OrderedFloat(float))), - Err(error) => error, - }; - - Err(ParseNumberError { - uint_error, - int_error, - float_error, - }) - } -} - -impl PartialEq for Number { - fn eq(&self, other: &Number) -> bool { - self.cmp(other) == Ordering::Equal - } -} - -impl Eq for Number {} - -impl PartialOrd for Number { - fn partial_cmp(&self, other: &Number) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Number { - fn cmp(&self, other: &Self) -> Ordering { - use Number::{Float, Signed, Unsigned, Null}; - - match (*self, *other) { - (Unsigned(a), Unsigned(b)) => a.cmp(&b), - (Unsigned(a), Signed(b)) => { - if b < 0 { - Ordering::Greater - } else { - a.cmp(&(b as u64)) - } - } - (Unsigned(a), Float(b)) => (OrderedFloat(a as f64)).cmp(&b), - (Signed(a), Unsigned(b)) => { - if a < 0 { - Ordering::Less - } else { - (a as u64).cmp(&b) - } - } - (Signed(a), Signed(b)) => a.cmp(&b), - (Signed(a), Float(b)) => OrderedFloat(a as f64).cmp(&b), - (Float(a), Unsigned(b)) => a.cmp(&OrderedFloat(b as f64)), - (Float(a), Signed(b)) => a.cmp(&OrderedFloat(b as f64)), - (Float(a), Float(b)) => a.cmp(&b), - (Null, Null) => Ordering::Equal, - (_, Null) => Ordering::Less, - (Null, _) => Ordering::Greater, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParseNumberError { - uint_error: ParseIntError, - int_error: ParseIntError, - float_error: ParseFloatError, -} - -impl fmt::Display for ParseNumberError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.uint_error == self.int_error { - write!( - f, - "can not parse number: {}, {}", - self.uint_error, self.float_error - ) - } else { - write!( - f, - "can not parse number: {}, {}, {}", - self.uint_error, self.int_error, self.float_error - ) - } - } -} diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs deleted file mode 100644 index 41acaeb7a..000000000 --- a/meilisearch-core/src/query_builder.rs +++ /dev/null @@ -1,1443 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::ops::{Deref, Range}; -use std::time::Duration; - -use either::Either; -use sdset::{SetOperation, SetBuf, Set}; - -use meilisearch_schema::FieldId; - -use crate::bucket_sort::{bucket_sort, bucket_sort_with_distinct, SortResult, placeholder_document_sort, facet_count}; -use crate::database::MainT; -use crate::facets::FacetFilter; -use crate::distinct_map::{DistinctMap, BufferedDistinctMap}; -use crate::Document; -use crate::{criterion::Criteria, DocumentId}; -use crate::{reordered_attrs::ReorderedAttrs, store, MResult, MainReader}; - -pub struct QueryBuilder<'c, 'f, 'd, 'i> { - criteria: Criteria<'c>, - searchable_attrs: Option, - filter: Option bool + 'f>>, - distinct: Option<(Box Option + 'd>, usize)>, - timeout: Option, - index: &'i store::Index, - facet_filter: Option, - facets: Option>, -} - -impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> { - pub fn new(index: &'i store::Index) -> Self { - QueryBuilder::with_criteria(index, Criteria::default()) - } - - /// sets facet attributes to filter on - pub fn set_facet_filter(&mut self, facets: Option) { - self.facet_filter = facets; - } - - /// sets facet attributes for which to return the count - pub fn set_facets(&mut self, facets: Option>) { - self.facets = facets; - } - - pub fn with_criteria(index: &'i store::Index, criteria: Criteria<'c>) -> Self { - QueryBuilder { - criteria, - searchable_attrs: None, - filter: None, - distinct: None, - timeout: None, - index, - facet_filter: None, - facets: None, - } - } - - pub fn with_filter(&mut self, function: F) - where - F: Fn(DocumentId) -> bool + 'f, - { - self.filter = Some(Box::new(function)) - } - - pub fn with_fetch_timeout(&mut self, timeout: Duration) { - self.timeout = Some(timeout) - } - - pub fn with_distinct(&mut self, size: usize, function: F) - where - F: Fn(DocumentId) -> Option + 'd, - { - self.distinct = Some((Box::new(function), size)) - } - - pub fn add_searchable_attribute(&mut self, attribute: u16) { - let reorders = self.searchable_attrs.get_or_insert_with(ReorderedAttrs::new); - reorders.insert_attribute(attribute); - } - - /// returns the documents ids associated with a facet filter by computing the union and - /// intersection of the document sets - fn facets_docids(&self, reader: &MainReader) -> MResult>> { - let facet_docids = match self.facet_filter { - Some(ref facets) => { - let mut ands = Vec::with_capacity(facets.len()); - let mut ors = Vec::new(); - for f in facets.deref() { - match f { - Either::Left(keys) => { - ors.reserve(keys.len()); - for key in keys { - let docids = self - .index - .facets - .facet_document_ids(reader, &key)? - .unwrap_or_default(); - ors.push(docids); - } - let sets: Vec<_> = ors.iter().map(|(_, i)| i).map(Cow::deref).collect(); - let or_result = sdset::multi::OpBuilder::from_vec(sets).union().into_set_buf(); - ands.push(Cow::Owned(or_result)); - ors.clear(); - } - Either::Right(key) => { - match self.index.facets.facet_document_ids(reader, &key)? { - Some((_name, docids)) => ands.push(docids), - // no candidates for search, early return. - None => return Ok(Some(SetBuf::default())), - } - } - }; - } - let ands: Vec<_> = ands.iter().map(Cow::deref).collect(); - Some( - sdset::multi::OpBuilder::from_vec(ands) - .intersection() - .into_set_buf(), - ) - } - None => None, - }; - Ok(facet_docids) - } - - fn standard_query(self, reader: &MainReader, query: &str, range: Range) -> MResult { - let facets_docids = match self.facets_docids(reader)? { - Some(ids) if ids.is_empty() => return Ok(SortResult::default()), - other => other - }; - // for each field to retrieve the count for, create an HashMap associating the attribute - // value to a set of matching documents. The HashMaps are them collected in another - // HashMap, associating each HashMap to it's field. - let facet_count_docids = self.facet_count_docids(reader)?; - - match self.distinct { - Some((distinct, distinct_size)) => bucket_sort_with_distinct( - reader, - query, - range, - facets_docids, - facet_count_docids, - self.filter, - distinct, - distinct_size, - self.criteria, - self.searchable_attrs, - self.index, - ), - None => bucket_sort( - reader, - query, - range, - facets_docids, - facet_count_docids, - self.filter, - self.criteria, - self.searchable_attrs, - self.index, - ), - } - } - - fn placeholder_query(self, reader: &heed::RoTxn, range: Range) -> MResult { - match self.facets_docids(reader)? { - Some(docids) => { - // We sort the docids from facets according to the criteria set by the user - let mut sorted_docids = docids.clone().into_vec(); - let mut sort_result = match self.index.main.ranked_map(reader)? { - Some(ranked_map) => { - placeholder_document_sort(&mut sorted_docids, self.index, reader, &ranked_map)?; - self.sort_result_from_docids(&sorted_docids, range) - }, - // if we can't perform a sort, we return documents unordered - None => self.sort_result_from_docids(&docids, range), - }; - - if let Some(f) = self.facet_count_docids(reader)? { - sort_result.exhaustive_facets_count = Some(true); - sort_result.facets = Some(facet_count(f, &docids)); - } - - Ok(sort_result) - }, - None => { - match self.index.main.sorted_document_ids_cache(reader)? { - // build result from cached document ids - Some(docids) => { - let mut sort_result = self.sort_result_from_docids(&docids, range); - - if let Some(f) = self.facet_count_docids(reader)? { - sort_result.exhaustive_facets_count = Some(true); - // document ids are not sorted in natural order, we need to construct a new set - let document_set = SetBuf::from_dirty(Vec::from(docids)); - sort_result.facets = Some(facet_count(f, &document_set)); - } - - Ok(sort_result) - }, - // no document id cached, return empty result - None => Ok(SortResult::default()), - } - } - } - } - - fn facet_count_docids<'a>(&self, reader: &'a MainReader) -> MResult>)>>>> { - match self.facets { - Some(ref field_ids) => { - let mut facet_count_map = HashMap::new(); - for (field_id, field_name) in field_ids { - let mut key_map = HashMap::new(); - for pair in self.index.facets.field_document_ids(reader, *field_id)? { - let (facet_key, document_ids) = pair?; - let value = facet_key.value(); - key_map.insert(value.to_string(), document_ids); - } - facet_count_map.insert(field_name.clone(), key_map); - } - Ok(Some(facet_count_map)) - } - None => Ok(None), - } - } - - fn sort_result_from_docids(&self, docids: &[DocumentId], range: Range) -> SortResult { - let mut sort_result = SortResult::default(); - let mut filtered_count = 0; - let mut result = match self.filter { - Some(ref filter) => docids - .iter() - .filter(|item| { - let accepted = (filter)(**item); - if !accepted { - filtered_count += 1; - } - accepted - }) - .skip(range.start) - .take(range.end - range.start) - .map(|&id| Document::from_highlights(id, &[])) - .collect::>(), - None => docids - .iter() - .skip(range.start) - .take(range.end - range.start) - .map(|&id| Document::from_highlights(id, &[])) - .collect::>(), - }; - - // distinct is set, remove duplicates with disctinct function - if let Some((distinct, distinct_size)) = &self.distinct { - let mut distinct_map = DistinctMap::new(*distinct_size); - let mut distinct_map = BufferedDistinctMap::new(&mut distinct_map); - result.retain(|doc| { - let id = doc.id; - let key = (distinct)(id); - let distinct_accepted = match key { - Some(key) => distinct_map.register(key), - None => distinct_map.register_without_key(), - }; - if !distinct_accepted { - filtered_count += 1; - } - distinct_accepted - }); - } - - sort_result.documents = result; - sort_result.nb_hits = docids.len() - filtered_count; - sort_result - } - - pub fn query( - self, - reader: &heed::RoTxn, - query: Option<&str>, - range: Range, - ) -> MResult { - match query { - Some(query) => self.standard_query(reader, query, range), - None => self.placeholder_query(reader, range), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use std::collections::{BTreeSet, HashMap}; - use std::iter::FromIterator; - - use fst::IntoStreamer; - use meilisearch_schema::IndexedPos; - use sdset::SetBuf; - use tempfile::TempDir; - - use crate::bucket_sort::SimpleMatch; - use crate::database::{Database, DatabaseOptions}; - use crate::store::Index; - use crate::DocIndex; - use crate::Document; - use meilisearch_schema::Schema; - - fn is_cjk(c: char) -> bool { - ('\u{1100}'..'\u{11ff}').contains(&c) // Hangul Jamo - || ('\u{2e80}'..'\u{2eff}').contains(&c) // CJK Radicals Supplement - || ('\u{2f00}'..'\u{2fdf}').contains(&c) // Kangxi radical - || ('\u{3000}'..'\u{303f}').contains(&c) // Japanese-style punctuation - || ('\u{3040}'..'\u{309f}').contains(&c) // Japanese Hiragana - || ('\u{30a0}'..'\u{30ff}').contains(&c) // Japanese Katakana - || ('\u{3100}'..'\u{312f}').contains(&c) - || ('\u{3130}'..'\u{318F}').contains(&c) // Hangul Compatibility Jamo - || ('\u{3200}'..'\u{32ff}').contains(&c) // Enclosed CJK Letters and Months - || ('\u{3400}'..'\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A - || ('\u{4e00}'..'\u{9fff}').contains(&c) // CJK Unified Ideographs - || ('\u{a960}'..'\u{a97f}').contains(&c) // Hangul Jamo Extended-A - || ('\u{ac00}'..'\u{d7a3}').contains(&c) // Hangul Syllables - || ('\u{d7b0}'..'\u{d7ff}').contains(&c) // Hangul Jamo Extended-B - || ('\u{f900}'..'\u{faff}').contains(&c) // CJK Compatibility Ideographs - || ('\u{ff00}'..'\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana - } - - fn normalize_str(string: &str) -> String { - let mut string = string.to_lowercase(); - - if !string.contains(is_cjk) { - string = deunicode::deunicode_with_tofu(&string, ""); - } - - string - } - - fn set_from_stream<'f, I, S>(stream: I) -> fst::Set> - where - I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>, - S: 'f + for<'a> fst::Streamer<'a, Item = &'a [u8]>, - { - let mut builder = fst::SetBuilder::memory(); - builder.extend_stream(stream).unwrap(); - builder.into_set() - } - - fn insert_key>(set: &fst::Set, key: &[u8]) -> fst::Set> { - let unique_key = { - let mut builder = fst::SetBuilder::memory(); - builder.insert(key).unwrap(); - builder.into_set() - }; - - let union_ = set.op().add(unique_key.into_stream()).r#union(); - - set_from_stream(union_) - } - - fn sdset_into_fstset(set: &sdset::Set<&str>) -> fst::Set> { - let mut builder = fst::SetBuilder::memory(); - let set = SetBuf::from_dirty(set.into_iter().map(|s| normalize_str(s)).collect()); - builder.extend_iter(set.into_iter()).unwrap(); - builder.into_set() - } - - const fn doc_index(document_id: u32, word_index: u16) -> DocIndex { - DocIndex { - document_id: DocumentId(document_id), - attribute: 0, - word_index, - char_index: 0, - char_length: 0, - } - } - - const fn doc_char_index(document_id: u32, word_index: u16, char_index: u16) -> DocIndex { - DocIndex { - document_id: DocumentId(document_id), - attribute: 0, - word_index, - char_index, - char_length: 0, - } - } - - pub struct TempDatabase { - database: Database, - index: Index, - _tempdir: TempDir, - } - - impl TempDatabase { - pub fn query_builder(&self) -> QueryBuilder { - self.index.query_builder() - } - - pub fn add_synonym(&mut self, word: &str, new: SetBuf<&str>) { - let db = &self.database; - let mut writer = db.main_write_txn().unwrap(); - - let word = normalize_str(word); - - let alternatives = self - .index - .synonyms - .synonyms_fst(&writer, word.as_bytes()) - .unwrap(); - - let new = sdset_into_fstset(&new); - let new_alternatives = - set_from_stream(alternatives.op().add(new.into_stream()).r#union()); - self.index - .synonyms - .put_synonyms(&mut writer, word.as_bytes(), &new_alternatives) - .unwrap(); - - let synonyms = self.index.main.synonyms_fst(&writer).unwrap(); - - let synonyms_fst = insert_key(&synonyms, word.as_bytes()); - self.index - .main - .put_synonyms_fst(&mut writer, &synonyms_fst) - .unwrap(); - - writer.commit().unwrap(); - } - } - - impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase { - fn from_iter>(iter: I) -> Self { - let tempdir = TempDir::new().unwrap(); - let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap(); - let index = database.create_index("default").unwrap(); - - let db = &database; - let mut writer = db.main_write_txn().unwrap(); - - let mut words_fst = BTreeSet::new(); - let mut postings_lists = HashMap::new(); - let mut fields_counts = HashMap::<_, u16>::new(); - - let mut schema = Schema::with_primary_key("id"); - - for (word, indexes) in iter { - let mut final_indexes = Vec::new(); - for index in indexes { - let name = index.attribute.to_string(); - schema.insert(&name).unwrap(); - let indexed_pos = schema.insert_with_position(&name).unwrap().1; - let index = DocIndex { - attribute: indexed_pos.0, - ..*index - }; - final_indexes.push(index); - } - - let word = word.to_lowercase().into_bytes(); - words_fst.insert(word.clone()); - postings_lists - .entry(word) - .or_insert_with(Vec::new) - .extend_from_slice(&final_indexes); - for idx in final_indexes { - fields_counts.insert((idx.document_id, idx.attribute, idx.word_index), 1); - } - } - - index.main.put_schema(&mut writer, &schema).unwrap(); - - let words_fst = fst::Set::from_iter(words_fst).unwrap(); - - index.main.put_words_fst(&mut writer, &words_fst).unwrap(); - - for (word, postings_list) in postings_lists { - let postings_list = SetBuf::from_dirty(postings_list); - index - .postings_lists - .put_postings_list(&mut writer, &word, &postings_list) - .unwrap(); - } - - for ((docid, attr, _), count) in fields_counts { - let prev = index - .documents_fields_counts - .document_field_count(&writer, docid, IndexedPos(attr)) - .unwrap(); - - let prev = prev.unwrap_or(0); - - index - .documents_fields_counts - .put_document_field_count(&mut writer, docid, IndexedPos(attr), prev + count) - .unwrap(); - } - - writer.commit().unwrap(); - - TempDatabase { database, index, _tempdir: tempdir } - } - } - - #[test] - fn simple() { - let store = TempDatabase::from_iter(vec![ - ("iphone", &[doc_char_index(0, 0, 0)][..]), - ("from", &[doc_char_index(0, 1, 1)][..]), - ("apple", &[doc_char_index(0, 2, 2)][..]), - ]); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("iphone from apple"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, .. })); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn simple_synonyms() { - let mut store = TempDatabase::from_iter(vec![("hello", &[doc_index(0, 0)][..])]); - - store.add_synonym("bonjour", SetBuf::from_dirty(vec!["hello"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("hello"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("bonjour"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - // #[test] - // fn prefix_synonyms() { - // let mut store = TempDatabase::from_iter(vec![("hello", &[doc_index(0, 0)][..])]); - - // store.add_synonym("bonjour", SetBuf::from_dirty(vec!["hello"])); - // store.add_synonym("salut", SetBuf::from_dirty(vec!["hello"])); - - // let db = &store.database; - // let reader = db.main_read_txn().unwrap(); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "sal", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - // let mut matches = matches.into_iter(); - // assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - // assert_matches!(matches.next(), None); - // }); - // assert_matches!(iter.next(), None); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "bonj", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - // let mut matches = matches.into_iter(); - // assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - // assert_matches!(matches.next(), None); - // }); - // assert_matches!(iter.next(), None); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "sal blabla", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), None); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "bonj blabla", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), None); - // } - - // #[test] - // fn levenshtein_synonyms() { - // let mut store = TempDatabase::from_iter(vec![("hello", &[doc_index(0, 0)][..])]); - - // store.add_synonym("salutation", SetBuf::from_dirty(vec!["hello"])); - - // let db = &store.database; - // let reader = db.main_read_txn().unwrap(); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "salutution", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - // let mut matches = matches.into_iter(); - // assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - // assert_matches!(matches.next(), None); - // }); - // assert_matches!(iter.next(), None); - - // let builder = store.query_builder(); - // let results = builder.query(&reader, "saluttion", 0..20).unwrap(); - // let mut iter = documents.into_iter(); - - // assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - // let mut matches = matches.into_iter(); - // assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - // assert_matches!(matches.next(), None); - // }); - // assert_matches!(iter.next(), None); - // } - - #[test] - fn harder_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("hello", &[doc_index(0, 0)][..]), - ("bonjour", &[doc_index(1, 3)]), - ("salut", &[doc_index(2, 5)]), - ]); - - store.add_synonym("hello", SetBuf::from_dirty(vec!["bonjour", "salut"])); - store.add_synonym("bonjour", SetBuf::from_dirty(vec!["hello", "salut"])); - store.add_synonym("salut", SetBuf::from_dirty(vec!["hello", "bonjour"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("hello"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 3, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 5, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("bonjour"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 3, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 5, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("salut"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 3, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 5, .. })); - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - /// Unique word has multi-word synonyms - fn unique_to_multiword_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("new", &[doc_char_index(0, 0, 0)][..]), - ("york", &[doc_char_index(0, 1, 1)][..]), - ("city", &[doc_char_index(0, 2, 2)][..]), - ("subway", &[doc_char_index(0, 3, 3)][..]), - ("NY", &[doc_char_index(1, 0, 0)][..]), - ("subway", &[doc_char_index(1, 1, 1)][..]), - ]); - - store.add_synonym( - "NY", - SetBuf::from_dirty(vec!["NYC", "new york", "new york city"]), - ); - store.add_synonym( - "NYC", - SetBuf::from_dirty(vec!["NY", "new york", "new york city"]), - ); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // NY ± new - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // NY ± york - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // NY ± city - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // NYC ± new - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // NYC ± york - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // NYC ± city - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn unique_to_multiword_synonyms_words_proximity() { - let mut store = TempDatabase::from_iter(vec![ - ("new", &[doc_char_index(0, 0, 0)][..]), - ("york", &[doc_char_index(0, 1, 1)][..]), - ("city", &[doc_char_index(0, 2, 2)][..]), - ("subway", &[doc_char_index(0, 3, 3)][..]), - ("york", &[doc_char_index(1, 0, 0)][..]), - ("new", &[doc_char_index(1, 1, 1)][..]), - ("subway", &[doc_char_index(1, 2, 2)][..]), - ("NY", &[doc_char_index(2, 0, 0)][..]), - ("subway", &[doc_char_index(2, 1, 1)][..]), - ]); - - store.add_synonym("NY", SetBuf::from_dirty(vec!["york new"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NY"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); // NY ± york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, .. })); // NY ± new - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); // york = NY - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, .. })); // new = NY - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 1, .. })); // york = NY - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 0, .. })); // new = NY - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("new york"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, .. })); // york - assert_matches!(matches.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 1, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 0, .. })); // new - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn unique_to_multiword_synonyms_cumulative_word_index() { - let mut store = TempDatabase::from_iter(vec![ - ("NY", &[doc_char_index(0, 0, 0)][..]), - ("subway", &[doc_char_index(0, 1, 1)][..]), - ("new", &[doc_char_index(1, 0, 0)][..]), - ("york", &[doc_char_index(1, 1, 1)][..]), - ("subway", &[doc_char_index(1, 2, 2)][..]), - ]); - - store.add_synonym("new york", SetBuf::from_dirty(vec!["NY"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // NY - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - // assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - // let mut matches = matches.into_iter(); - // assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 2, is_exact: true, .. })); // subway - // assert_matches!(matches.next(), None); - // }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = - builder.query(&reader, Some("new york subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NY - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NY - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - /// Unique word has multi-word synonyms - fn harder_unique_to_multiword_synonyms_one() { - let mut store = TempDatabase::from_iter(vec![ - ("new", &[doc_char_index(0, 0, 0)][..]), - ("york", &[doc_char_index(0, 1, 1)][..]), - ("city", &[doc_char_index(0, 2, 2)][..]), - ("yellow", &[doc_char_index(0, 3, 3)][..]), - ("subway", &[doc_char_index(0, 4, 4)][..]), - ("broken", &[doc_char_index(0, 5, 5)][..]), - ("NY", &[doc_char_index(1, 0, 0)][..]), - ("blue", &[doc_char_index(1, 1, 1)][..]), - ("subway", &[doc_char_index(1, 2, 2)][..]), - ]); - - store.add_synonym( - "NY", - SetBuf::from_dirty(vec!["NYC", "new york", "new york city"]), - ); - store.add_synonym( - "NYC", - SetBuf::from_dirty(vec!["NY", "new york", "new york city"]), - ); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NY subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // NYC - // because one-word to one-word ^^^^ - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), None); - } - - #[test] - /// Unique word has multi-word synonyms - fn even_harder_unique_to_multiword_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("new", &[doc_char_index(0, 0, 0)][..]), - ("york", &[doc_char_index(0, 1, 1)][..]), - ("city", &[doc_char_index(0, 2, 2)][..]), - ("yellow", &[doc_char_index(0, 3, 3)][..]), - ("underground", &[doc_char_index(0, 4, 4)][..]), - ("train", &[doc_char_index(0, 5, 5)][..]), - ("broken", &[doc_char_index(0, 6, 6)][..]), - ("NY", &[doc_char_index(1, 0, 0)][..]), - ("blue", &[doc_char_index(1, 1, 1)][..]), - ("subway", &[doc_char_index(1, 2, 2)][..]), - ]); - - store.add_synonym( - "NY", - SetBuf::from_dirty(vec!["NYC", "new york", "new york city"]), - ); - store.add_synonym( - "NYC", - SetBuf::from_dirty(vec!["NY", "new york", "new york city"]), - ); - store.add_synonym("subway", SetBuf::from_dirty(vec!["underground train"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult {documents, .. } = builder.query(&reader, Some("NY subway broken"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city = NY - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: false, .. })); // underground = subway - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 4, word_index: 5, is_exact: false, .. })); // train = subway - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 5, word_index: 6, is_exact: true, .. })); // broken - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NYC subway"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // underground = subway - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 4, word_index: 5, is_exact: true, .. })); // train = subway - assert_matches!(iter.next(), None); // position rewritten ^ - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york = NYC - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city = NYC - // because one-word to one-word ^^^^ - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: false, .. })); // subway = underground - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 4, word_index: 5, is_exact: false, .. })); // subway = train - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - /// Multi-word has multi-word synonyms - fn multiword_to_multiword_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("NY", &[doc_char_index(0, 0, 0)][..]), - ("subway", &[doc_char_index(0, 1, 1)][..]), - ("NYC", &[doc_char_index(1, 0, 0)][..]), - ("blue", &[doc_char_index(1, 1, 1)][..]), - ("subway", &[doc_char_index(1, 2, 2)][..]), - ("broken", &[doc_char_index(1, 3, 3)][..]), - ("new", &[doc_char_index(2, 0, 0)][..]), - ("york", &[doc_char_index(2, 1, 1)][..]), - ("underground", &[doc_char_index(2, 2, 2)][..]), - ("train", &[doc_char_index(2, 3, 3)][..]), - ("broken", &[doc_char_index(2, 4, 4)][..]), - ]); - - store.add_synonym( - "new york", - SetBuf::from_dirty(vec!["NYC", "NY", "new york city"]), - ); - store.add_synonym( - "new york city", - SetBuf::from_dirty(vec!["NYC", "NY", "new york"]), - ); - store.add_synonym("underground train", SetBuf::from_dirty(vec!["subway"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder - .query(&reader, Some("new york underground train broken"), 0..20) - .unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // underground - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 4, is_exact: true, .. })); // train - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 5, word_index: 5, is_exact: true, .. })); // broken - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // NYC = new - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // NYC = york - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // NYC = city - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 4, is_exact: true, .. })); // subway = underground - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 4, word_index: 5, is_exact: true, .. })); // subway = train - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 5, word_index: 6, is_exact: true, .. })); // broken - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder - .query(&reader, Some("new york city underground train broken"), 0..20) - .unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 3, word_index: 2, is_exact: true, .. })); // underground - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 3, is_exact: true, .. })); // train - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 5, word_index: 4, is_exact: true, .. })); // broken - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // NYC = new - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // NYC = york - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // subway = underground - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 4, word_index: 4, is_exact: true, .. })); // subway = train - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 5, word_index: 5, is_exact: true, .. })); // broken - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn intercrossed_multiword_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("new", &[doc_index(0, 0)][..]), - ("york", &[doc_index(0, 1)][..]), - ("big", &[doc_index(0, 2)][..]), - ("city", &[doc_index(0, 3)][..]), - ]); - - store.add_synonym("new york", SetBuf::from_dirty(vec!["new york city"])); - store.add_synonym("new york city", SetBuf::from_dirty(vec!["new york"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("new york big "), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: false, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 4, is_exact: false, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // big - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - - let mut store = TempDatabase::from_iter(vec![ - ("NY", &[doc_index(0, 0)][..]), - ("city", &[doc_index(0, 1)][..]), - ("subway", &[doc_index(0, 2)][..]), - ("NY", &[doc_index(1, 0)][..]), - ("subway", &[doc_index(1, 1)][..]), - ("NY", &[doc_index(2, 0)][..]), - ("york", &[doc_index(2, 1)][..]), - ("city", &[doc_index(2, 2)][..]), - ("subway", &[doc_index(2, 3)][..]), - ]); - - store.add_synonym("NY", SetBuf::from_dirty(vec!["new york city story"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("NY subway "), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: false, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: false, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 3, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // story - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 4, is_exact: true, .. })); // subway - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn cumulative_word_indices() { - let mut store = TempDatabase::from_iter(vec![ - ("NYC", &[doc_index(0, 0)][..]), - ("long", &[doc_index(0, 1)][..]), - ("subway", &[doc_index(0, 2)][..]), - ("cool", &[doc_index(0, 3)][..]), - ]); - - store.add_synonym("new york city", SetBuf::from_dirty(vec!["NYC"])); - store.add_synonym("subway", SetBuf::from_dirty(vec!["underground train"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder - .query(&reader, Some("new york city long subway cool "), 0..20) - .unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut matches = matches.into_iter(); - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 0, word_index: 0, is_exact: true, .. })); // new = NYC - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 1, word_index: 1, is_exact: true, .. })); // york = NYC - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 2, word_index: 2, is_exact: true, .. })); // city = NYC - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 3, word_index: 3, is_exact: true, .. })); // long - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 4, word_index: 4, is_exact: true, .. })); // subway = underground - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 5, word_index: 5, is_exact: true, .. })); // subway = train - assert_matches!(matches.next(), Some(SimpleMatch { query_index: 6, word_index: 6, is_exact: true, .. })); // cool - assert_matches!(matches.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn deunicoded_synonyms() { - let mut store = TempDatabase::from_iter(vec![ - ("telephone", &[doc_index(0, 0)][..]), // meilisearch indexes the unidecoded - ("téléphone", &[doc_index(0, 0)][..]), // and the original words on the same DocIndex - ("iphone", &[doc_index(1, 0)][..]), - ]); - - store.add_synonym("téléphone", SetBuf::from_dirty(vec!["iphone"])); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("telephone"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("téléphone"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("télephone"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. })); - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn simple_concatenation() { - let store = TempDatabase::from_iter(vec![ - ("iphone", &[doc_index(0, 0)][..]), - ("case", &[doc_index(0, 1)][..]), - ]); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("i phone case"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 0, .. })); // iphone - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 1, distance: 0, .. })); // iphone - // assert_matches!(iter.next(), Some(SimpleMatch { query_index: 1, word_index: 0, distance: 1, .. })); "phone" - // but no typo on first letter ^^^^^^^ - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 2, word_index: 2, distance: 0, .. })); // case - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn exact_field_count_one_word() { - let store = TempDatabase::from_iter(vec![ - ("searchengine", &[doc_index(0, 0)][..]), - ("searchengine", &[doc_index(1, 0)][..]), - ("blue", &[doc_index(1, 1)][..]), - ("searchangine", &[doc_index(2, 0)][..]), - ("searchengine", &[doc_index(3, 0)][..]), - ]); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 0, .. })); // searchengine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(3), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 0, .. })); // searchengine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 0, .. })); // searchengine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 1, .. })); // searchengine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn simple_phrase_query_splitting() { - let store = TempDatabase::from_iter(vec![ - ("search", &[doc_index(0, 0)][..]), - ("engine", &[doc_index(0, 1)][..]), - ("search", &[doc_index(1, 0)][..]), - ("slow", &[doc_index(1, 1)][..]), - ("engine", &[doc_index(1, 2)][..]), - ]); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 0, distance: 0, .. })); // search - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 1, distance: 0, .. })); // engine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } - - #[test] - fn harder_phrase_query_splitting() { - let store = TempDatabase::from_iter(vec![ - ("search", &[doc_index(0, 0)][..]), - ("search", &[doc_index(0, 1)][..]), - ("engine", &[doc_index(0, 2)][..]), - ("search", &[doc_index(1, 0)][..]), - ("slow", &[doc_index(1, 1)][..]), - ("search", &[doc_index(1, 2)][..]), - ("engine", &[doc_index(1, 3)][..]), - ("search", &[doc_index(1, 0)][..]), - ("search", &[doc_index(1, 1)][..]), - ("slow", &[doc_index(1, 2)][..]), - ("engine", &[doc_index(1, 3)][..]), - ]); - - let db = &store.database; - let reader = db.main_read_txn().unwrap(); - - let builder = store.query_builder(); - let SortResult { documents, .. } = builder.query(&reader, Some("searchengine"), 0..20).unwrap(); - let mut iter = documents.into_iter(); - - assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 1, distance: 0, .. })); // search - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 2, distance: 0, .. })); // engine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { - let mut iter = matches.into_iter(); - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 2, distance: 0, .. })); // search - assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, word_index: 3, distance: 0, .. })); // engine - assert_matches!(iter.next(), None); - }); - assert_matches!(iter.next(), None); - } -} diff --git a/meilisearch-core/src/query_tree.rs b/meilisearch-core/src/query_tree.rs deleted file mode 100644 index 7255ef3db..000000000 --- a/meilisearch-core/src/query_tree.rs +++ /dev/null @@ -1,573 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::ops::Range; -use std::time::Instant; -use std::{cmp, fmt, iter::once}; - -use fst::{IntoStreamer, Streamer}; -use itertools::{EitherOrBoth, merge_join_by}; -use log::debug; -use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig}; -use sdset::{Set, SetBuf, SetOperation}; - -use crate::database::MainT; -use crate::{store, DocumentId, DocIndex, MResult, FstSetCow}; -use crate::automaton::{build_dfa, build_prefix_dfa, build_exact_dfa}; -use crate::QueryWordsMapper; - -pub const MAX_QUERY_LEN: usize = 10; - -#[derive(Clone, PartialEq, Eq, Hash)] -pub enum Operation { - And(Vec), - Or(Vec), - Query(Query), -} - -impl fmt::Debug for Operation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fn pprint_tree(f: &mut fmt::Formatter<'_>, op: &Operation, depth: usize) -> fmt::Result { - match op { - Operation::And(children) => { - writeln!(f, "{:1$}AND", "", depth * 2)?; - children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1)) - }, - Operation::Or(children) => { - writeln!(f, "{:1$}OR", "", depth * 2)?; - children.iter().try_for_each(|c| pprint_tree(f, c, depth + 1)) - }, - Operation::Query(query) => writeln!(f, "{:2$}{:?}", "", query, depth * 2), - } - } - - pprint_tree(f, self, 0) - } -} - -impl Operation { - fn tolerant(id: QueryId, prefix: bool, s: &str) -> Operation { - Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::Tolerant(s.to_string()) }) - } - - fn non_tolerant(id: QueryId, prefix: bool, s: &str) -> Operation { - Operation::Query(Query { id, prefix, exact: true, kind: QueryKind::NonTolerant(s.to_string()) }) - } - - fn phrase2(id: QueryId, prefix: bool, (left, right): (&str, &str)) -> Operation { - let kind = QueryKind::Phrase(vec![left.to_owned(), right.to_owned()]); - Operation::Query(Query { id, prefix, exact: true, kind }) - } -} - -pub type QueryId = usize; - -#[derive(Clone, Eq)] -pub struct Query { - pub id: QueryId, - pub prefix: bool, - pub exact: bool, - pub kind: QueryKind, -} - -impl PartialEq for Query { - fn eq(&self, other: &Self) -> bool { - self.prefix == other.prefix && self.kind == other.kind - } -} - -impl Hash for Query { - fn hash(&self, state: &mut H) { - self.prefix.hash(state); - self.kind.hash(state); - } -} - -#[derive(Clone, PartialEq, Eq, Hash)] -pub enum QueryKind { - Tolerant(String), - NonTolerant(String), - Phrase(Vec), -} - -impl fmt::Debug for Query { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Query { id, prefix, kind, .. } = self; - let prefix = if *prefix { String::from("Prefix") } else { String::default() }; - match kind { - QueryKind::NonTolerant(word) => { - f.debug_struct(&(prefix + "NonTolerant")).field("id", &id).field("word", &word).finish() - }, - QueryKind::Tolerant(word) => { - f.debug_struct(&(prefix + "Tolerant")).field("id", &id).field("word", &word).finish() - }, - QueryKind::Phrase(words) => { - f.debug_struct(&(prefix + "Phrase")).field("id", &id).field("words", &words).finish() - }, - } - } -} - -#[derive(Debug, Default)] -pub struct PostingsList { - docids: SetBuf, - matches: SetBuf, -} - -pub struct Context<'a> { - pub words_set: FstSetCow<'a>, - pub stop_words: FstSetCow<'a>, - pub synonyms: store::Synonyms, - pub postings_lists: store::PostingsLists, - pub prefix_postings_lists: store::PrefixPostingsListsCache, -} - -fn split_best_frequency<'a>(reader: &heed::RoTxn, ctx: &Context, word: &'a str) -> MResult> { - let chars = word.char_indices().skip(1); - let mut best = None; - - for (i, _) in chars { - let (left, right) = word.split_at(i); - - let left_freq = ctx.postings_lists - .postings_list(reader, left.as_bytes())? - .map(|p| p.docids.len()) - .unwrap_or(0); - let right_freq = ctx.postings_lists - .postings_list(reader, right.as_bytes())? - .map(|p| p.docids.len()) - .unwrap_or(0); - - let min_freq = cmp::min(left_freq, right_freq); - if min_freq != 0 && best.map_or(true, |(old, _, _)| min_freq > old) { - best = Some((min_freq, left, right)); - } - } - - Ok(best.map(|(_, l, r)| (l, r))) -} - -fn fetch_synonyms(reader: &heed::RoTxn, ctx: &Context, words: &[&str]) -> MResult>> { - let words = &words.join(" "); - let set = ctx.synonyms.synonyms_fst(reader, words.as_bytes())?; - - let mut strings = Vec::new(); - let mut stream = set.stream(); - while let Some(input) = stream.next() { - if let Ok(input) = std::str::from_utf8(input) { - let alts = input.split_ascii_whitespace().map(ToOwned::to_owned).collect(); - strings.push(alts); - } - } - - Ok(strings) -} - -fn create_operation(iter: I, f: F) -> Operation -where I: IntoIterator, - F: Fn(Vec) -> Operation, -{ - let mut iter = iter.into_iter(); - match (iter.next(), iter.next()) { - (Some(first), None) => first, - (first, second) => f(first.into_iter().chain(second).chain(iter).collect()), - } -} - -const MAX_NGRAM: usize = 3; - -fn split_query_string>(s: &str, stop_words: &fst::Set) -> Vec<(usize, String)> { - // TODO: Use global instance instead - Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)) - .analyze(s) - .tokens() - .filter(|t| t.is_word()) - .map(|t| t.word.to_string()) - .take(MAX_QUERY_LEN) - .enumerate() - .collect() -} - -pub fn create_query_tree( - reader: &heed::RoTxn, - ctx: &Context, - query: &str, -) -> MResult<(Operation, HashMap>)> -{ - // TODO: use a shared analyzer instance - let words = split_query_string(query, &ctx.stop_words); - - let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w)); - - fn create_inner( - reader: &heed::RoTxn, - ctx: &Context, - mapper: &mut QueryWordsMapper, - words: &[(usize, String)], - ) -> MResult> - { - let mut alts = Vec::new(); - - for ngram in 1..=MAX_NGRAM { - if let Some(group) = words.get(..ngram) { - let mut group_ops = Vec::new(); - - let tail = &words[ngram..]; - let is_last = tail.is_empty(); - - let mut group_alts = Vec::new(); - match group { - [(id, word)] => { - let mut idgen = ((id + 1) * 100)..; - let range = (*id)..id+1; - - let phrase = split_best_frequency(reader, ctx, word)? - .map(|ws| { - let id = idgen.next().unwrap(); - idgen.next().unwrap(); - mapper.declare(range.clone(), id, &[ws.0, ws.1]); - Operation::phrase2(id, is_last, ws) - }); - - let synonyms = fetch_synonyms(reader, ctx, &[word])? - .into_iter() - .map(|alts| { - let exact = alts.len() == 1; - let id = idgen.next().unwrap(); - mapper.declare(range.clone(), id, &alts); - - let mut idgen = once(id).chain(&mut idgen); - let iter = alts.into_iter().map(|w| { - let id = idgen.next().unwrap(); - let kind = QueryKind::NonTolerant(w); - Operation::Query(Query { id, prefix: false, exact, kind }) - }); - - create_operation(iter, Operation::And) - }); - - let original = Operation::tolerant(*id, is_last, word); - - group_alts.push(original); - group_alts.extend(synonyms.chain(phrase)); - }, - words => { - let id = words[0].0; - let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..; - let range = id..id+ngram; - - let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect(); - - for synonym in fetch_synonyms(reader, ctx, &words)? { - let exact = synonym.len() == 1; - let id = idgen.next().unwrap(); - mapper.declare(range.clone(), id, &synonym); - - let mut idgen = once(id).chain(&mut idgen); - let synonym = synonym.into_iter().map(|s| { - let id = idgen.next().unwrap(); - let kind = QueryKind::NonTolerant(s); - Operation::Query(Query { id, prefix: false, exact, kind }) - }); - group_alts.push(create_operation(synonym, Operation::And)); - } - - let id = idgen.next().unwrap(); - let concat = words.concat(); - mapper.declare(range.clone(), id, &[&concat]); - group_alts.push(Operation::non_tolerant(id, is_last, &concat)); - } - } - - group_ops.push(create_operation(group_alts, Operation::Or)); - - if !tail.is_empty() { - let tail_ops = create_inner(reader, ctx, mapper, tail)?; - group_ops.push(create_operation(tail_ops, Operation::Or)); - } - - alts.push(create_operation(group_ops, Operation::And)); - } - } - - Ok(alts) - } - - let alternatives = create_inner(reader, ctx, &mut mapper, &words)?; - let operation = Operation::Or(alternatives); - let mapping = mapper.mapping(); - - Ok((operation, mapping)) -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct PostingsKey<'o> { - pub query: &'o Query, - pub input: Vec, - pub distance: u8, - pub is_exact: bool, -} - -pub type Postings<'o, 'txn> = HashMap, Cow<'txn, Set>>; -pub type Cache<'o, 'txn> = HashMap<&'o Operation, Cow<'txn, Set>>; - -pub struct QueryResult<'o, 'txn> { - pub docids: Cow<'txn, Set>, - pub queries: Postings<'o, 'txn>, -} - -pub fn traverse_query_tree<'o, 'txn>( - reader: &'txn heed::RoTxn, - ctx: &Context, - tree: &'o Operation, -) -> MResult> -{ - fn execute_and<'o, 'txn>( - reader: &'txn heed::RoTxn, - ctx: &Context, - cache: &mut Cache<'o, 'txn>, - postings: &mut Postings<'o, 'txn>, - depth: usize, - operations: &'o [Operation], - ) -> MResult>> - { - debug!("{:1$}AND", "", depth * 2); - - let before = Instant::now(); - let mut results = Vec::new(); - - for op in operations { - if cache.get(op).is_none() { - let docids = match op { - Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?, - Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?, - Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?, - }; - cache.insert(op, docids); - } - } - - for op in operations { - if let Some(docids) = cache.get(op) { - results.push(docids.as_ref()); - } - } - - let op = sdset::multi::Intersection::new(results); - let docids = op.into_set_buf(); - - debug!("{:3$}--- AND fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2); - - Ok(Cow::Owned(docids)) - } - - fn execute_or<'o, 'txn>( - reader: &'txn heed::RoTxn, - ctx: &Context, - cache: &mut Cache<'o, 'txn>, - postings: &mut Postings<'o, 'txn>, - depth: usize, - operations: &'o [Operation], - ) -> MResult>> - { - debug!("{:1$}OR", "", depth * 2); - - let before = Instant::now(); - let mut results = Vec::new(); - - for op in operations { - if cache.get(op).is_none() { - let docids = match op { - Operation::And(ops) => execute_and(reader, ctx, cache, postings, depth + 1, &ops)?, - Operation::Or(ops) => execute_or(reader, ctx, cache, postings, depth + 1, &ops)?, - Operation::Query(query) => execute_query(reader, ctx, postings, depth + 1, &query)?, - }; - cache.insert(op, docids); - } - } - - for op in operations { - if let Some(docids) = cache.get(op) { - results.push(docids.as_ref()); - } - } - - let op = sdset::multi::Union::new(results); - let docids = op.into_set_buf(); - - debug!("{:3$}--- OR fetched {} documents in {:.02?}", "", docids.len(), before.elapsed(), depth * 2); - - Ok(Cow::Owned(docids)) - } - - fn execute_query<'o, 'txn>( - reader: &'txn heed::RoTxn, - ctx: &Context, - postings: &mut Postings<'o, 'txn>, - depth: usize, - query: &'o Query, - ) -> MResult>> - { - let before = Instant::now(); - - let Query { prefix, kind, exact, .. } = query; - let docids: Cow> = match kind { - QueryKind::Tolerant(word) => { - if *prefix && word.len() <= 2 { - let prefix = { - let mut array = [0; 4]; - let bytes = word.as_bytes(); - array[..bytes.len()].copy_from_slice(bytes); - array - }; - - // We retrieve the cached postings lists for all - // the words that starts with this short prefix. - let result = ctx.prefix_postings_lists.prefix_postings_list(reader, prefix)?.unwrap_or_default(); - let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: false }; - postings.insert(key, result.matches); - let prefix_docids = &result.docids; - - // We retrieve the exact postings list for the prefix, - // because we must consider these matches as exact. - let result = ctx.postings_lists.postings_list(reader, word.as_bytes())?.unwrap_or_default(); - let key = PostingsKey { query, input: word.clone().into_bytes(), distance: 0, is_exact: true }; - postings.insert(key, result.matches); - let exact_docids = &result.docids; - - let before = Instant::now(); - let docids = sdset::duo::Union::new(prefix_docids, exact_docids).into_set_buf(); - debug!("{:4$}prefix docids ({} and {}) construction took {:.02?}", - "", prefix_docids.len(), exact_docids.len(), before.elapsed(), depth * 2); - - Cow::Owned(docids) - - } else { - let dfa = if *prefix { build_prefix_dfa(word) } else { build_dfa(word) }; - - let byte = word.as_bytes()[0]; - let mut stream = if byte == u8::max_value() { - ctx.words_set.search(&dfa).ge(&[byte]).into_stream() - } else { - ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream() - }; - - let before = Instant::now(); - let mut results = Vec::new(); - while let Some(input) = stream.next() { - if let Some(result) = ctx.postings_lists.postings_list(reader, input)? { - let distance = dfa.eval(input).to_u8(); - let is_exact = *exact && distance == 0 && input.len() == word.len(); - results.push(result.docids); - let key = PostingsKey { query, input: input.to_owned(), distance, is_exact }; - postings.insert(key, result.matches); - } - } - debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2); - - let before = Instant::now(); - let docids = if results.len() > 10 { - let cap = results.iter().map(|dis| dis.len()).sum(); - let mut docids = Vec::with_capacity(cap); - for dis in results { - docids.extend_from_slice(&dis); - } - SetBuf::from_dirty(docids) - } else { - let sets = results.iter().map(AsRef::as_ref).collect(); - sdset::multi::Union::new(sets).into_set_buf() - }; - debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2); - - Cow::Owned(docids) - } - }, - QueryKind::NonTolerant(word) => { - // TODO support prefix and non-prefix exact DFA - let dfa = build_exact_dfa(word); - - let byte = word.as_bytes()[0]; - let mut stream = if byte == u8::max_value() { - ctx.words_set.search(&dfa).ge(&[byte]).into_stream() - } else { - ctx.words_set.search(&dfa).ge(&[byte]).lt(&[byte + 1]).into_stream() - }; - - let before = Instant::now(); - let mut results = Vec::new(); - while let Some(input) = stream.next() { - if let Some(result) = ctx.postings_lists.postings_list(reader, input)? { - let distance = dfa.eval(input).to_u8(); - results.push(result.docids); - let key = PostingsKey { query, input: input.to_owned(), distance, is_exact: *exact }; - postings.insert(key, result.matches); - } - } - debug!("{:3$}docids retrieval ({:?}) took {:.02?}", "", results.len(), before.elapsed(), depth * 2); - - let before = Instant::now(); - let docids = if results.len() > 10 { - let cap = results.iter().map(|dis| dis.len()).sum(); - let mut docids = Vec::with_capacity(cap); - for dis in results { - docids.extend_from_slice(&dis); - } - SetBuf::from_dirty(docids) - } else { - let sets = results.iter().map(AsRef::as_ref).collect(); - sdset::multi::Union::new(sets).into_set_buf() - }; - debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2); - - Cow::Owned(docids) - }, - QueryKind::Phrase(words) => { - // TODO support prefix and non-prefix exact DFA - if let [first, second] = words.as_slice() { - let first = ctx.postings_lists.postings_list(reader, first.as_bytes())?.unwrap_or_default(); - let second = ctx.postings_lists.postings_list(reader, second.as_bytes())?.unwrap_or_default(); - - let iter = merge_join_by(first.matches.as_slice(), second.matches.as_slice(), |a, b| { - let x = (a.document_id, a.attribute, (a.word_index as u32) + 1); - let y = (b.document_id, b.attribute, b.word_index as u32); - x.cmp(&y) - }); - - let matches: Vec<_> = iter - .filter_map(EitherOrBoth::both) - .flat_map(|(a, b)| once(*a).chain(Some(*b))) - .collect(); - - let before = Instant::now(); - let mut docids: Vec<_> = matches.iter().map(|m| m.document_id).collect(); - docids.dedup(); - let docids = SetBuf::new(docids).unwrap(); - debug!("{:2$}docids construction took {:.02?}", "", before.elapsed(), depth * 2); - - let matches = Cow::Owned(SetBuf::from_dirty(matches)); - let key = PostingsKey { query, input: vec![], distance: 0, is_exact: true }; - postings.insert(key, matches); - - Cow::Owned(docids) - } else { - debug!("{:2$}{:?} skipped", "", words, depth * 2); - Cow::default() - } - }, - }; - - debug!("{:4$}{:?} fetched {:?} documents in {:.02?}", "", query, docids.len(), before.elapsed(), depth * 2); - Ok(docids) - } - - let mut cache = Cache::new(); - let mut postings = Postings::new(); - - let docids = match tree { - Operation::And(ops) => execute_and(reader, ctx, &mut cache, &mut postings, 0, &ops)?, - Operation::Or(ops) => execute_or(reader, ctx, &mut cache, &mut postings, 0, &ops)?, - Operation::Query(query) => execute_query(reader, ctx, &mut postings, 0, &query)?, - }; - - Ok(QueryResult { docids, queries: postings }) -} diff --git a/meilisearch-core/src/query_words_mapper.rs b/meilisearch-core/src/query_words_mapper.rs deleted file mode 100644 index 7ff07b459..000000000 --- a/meilisearch-core/src/query_words_mapper.rs +++ /dev/null @@ -1,416 +0,0 @@ -use std::collections::HashMap; -use std::iter::FromIterator; -use std::ops::Range; -use intervaltree::{Element, IntervalTree}; - -pub type QueryId = usize; - -pub struct QueryWordsMapper { - originals: Vec, - mappings: HashMap, Vec)>, -} - -impl QueryWordsMapper { - pub fn new(originals: I) -> QueryWordsMapper - where I: IntoIterator, - A: ToString, - { - let originals = originals.into_iter().map(|s| s.to_string()).collect(); - QueryWordsMapper { originals, mappings: HashMap::new() } - } - - #[allow(clippy::len_zero)] - pub fn declare(&mut self, range: Range, id: QueryId, replacement: I) - where I: IntoIterator, - A: ToString, - { - assert!(range.len() != 0); - assert!(self.originals.get(range.clone()).is_some()); - assert!(id >= self.originals.len()); - - let replacement: Vec<_> = replacement.into_iter().map(|s| s.to_string()).collect(); - - assert!(!replacement.is_empty()); - - // We detect words at the end and at the front of the - // replacement that are common with the originals: - // - // x a b c d e f g - // ^^^/ \^^^ - // a b x c d k j e f - // ^^^ ^^^ - // - - let left = &self.originals[..range.start]; - let right = &self.originals[range.end..]; - - let common_left = longest_common_prefix(left, &replacement); - let common_right = longest_common_prefix(&replacement, right); - - for i in 0..common_left { - let range = range.start - common_left + i..range.start - common_left + i + 1; - let replacement = vec![replacement[i].clone()]; - self.mappings.insert(id + i, (range, replacement)); - } - - { - let replacement = replacement[common_left..replacement.len() - common_right].to_vec(); - self.mappings.insert(id + common_left, (range.clone(), replacement)); - } - - for i in 0..common_right { - let id = id + replacement.len() - common_right + i; - let range = range.end + i..range.end + i + 1; - let replacement = vec![replacement[replacement.len() - common_right + i].clone()]; - self.mappings.insert(id, (range, replacement)); - } - } - - pub fn mapping(self) -> HashMap> { - let mappings = self.mappings.into_iter().map(|(i, (r, v))| (r, (i, v))); - let intervals = IntervalTree::from_iter(mappings); - - let mut output = HashMap::new(); - let mut offset = 0; - - // We map each original word to the biggest number of - // associated words. - for i in 0..self.originals.len() { - let max = intervals.query_point(i) - .filter_map(|e| { - if e.range.end - 1 == i { - let len = e.value.1.iter().skip(i - e.range.start).count(); - if len != 0 { Some(len) } else { None } - } else { None } - }) - .max() - .unwrap_or(1); - - let range = i + offset..i + offset + max; - output.insert(i, range); - offset += max - 1; - } - - // We retrieve the range that each original word - // is mapped to and apply it to each of the words. - for i in 0..self.originals.len() { - - let iter = intervals.query_point(i).filter(|e| e.range.end - 1 == i); - for Element { range, value: (id, words) } in iter { - - // We ask for the complete range mapped to the area we map. - let start = output.get(&range.start).map(|r| r.start).unwrap_or(range.start); - let end = output.get(&(range.end - 1)).map(|r| r.end).unwrap_or(range.end); - let range = start..end; - - // We map each query id to one word until the last, - // we map it to the remainings words. - let add = range.len() - words.len(); - for (j, x) in range.take(words.len()).enumerate() { - let add = if j == words.len() - 1 { add } else { 0 }; // is last? - let range = x..x + 1 + add; - output.insert(id + j, range); - } - } - } - - output - } -} - -fn longest_common_prefix(a: &[T], b: &[T]) -> usize { - let mut best = None; - for i in (0..a.len()).rev() { - let count = a[i..].iter().zip(b).take_while(|(a, b)| a == b).count(); - best = match best { - Some(old) if count > old => Some(count), - Some(_) => break, - None => Some(count), - }; - } - best.unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn original_unmodified() { - let query = ["new", "york", "city", "subway"]; - // 0 1 2 3 - let mut builder = QueryWordsMapper::new(&query); - - // new york = new york city - builder.declare(0..2, 4, &["new", "york", "city"]); - // ^ 4 5 6 - - // new = new york city - builder.declare(0..1, 7, &["new", "york", "city"]); - // ^ 7 8 9 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // new - assert_eq!(mapping[&1], 1..2); // york - assert_eq!(mapping[&2], 2..3); // city - assert_eq!(mapping[&3], 3..4); // subway - - assert_eq!(mapping[&4], 0..1); // new - assert_eq!(mapping[&5], 1..2); // york - assert_eq!(mapping[&6], 2..3); // city - - assert_eq!(mapping[&7], 0..1); // new - assert_eq!(mapping[&8], 1..2); // york - assert_eq!(mapping[&9], 2..3); // city - } - - #[test] - fn original_unmodified2() { - let query = ["new", "york", "city", "subway"]; - // 0 1 2 3 - let mut builder = QueryWordsMapper::new(&query); - - // city subway = new york city underground train - builder.declare(2..4, 4, &["new", "york", "city", "underground", "train"]); - // ^ 4 5 6 7 8 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // new - assert_eq!(mapping[&1], 1..2); // york - assert_eq!(mapping[&2], 2..3); // city - assert_eq!(mapping[&3], 3..5); // subway - - assert_eq!(mapping[&4], 0..1); // new - assert_eq!(mapping[&5], 1..2); // york - assert_eq!(mapping[&6], 2..3); // city - assert_eq!(mapping[&7], 3..4); // underground - assert_eq!(mapping[&8], 4..5); // train - } - - #[test] - fn original_unmodified3() { - let query = ["a", "b", "x", "x", "a", "b", "c", "d", "e", "f", "g"]; - // 0 1 2 3 4 5 6 7 8 9 10 - let mut builder = QueryWordsMapper::new(&query); - - // c d = a b x c d k j e f - builder.declare(6..8, 11, &["a", "b", "x", "c", "d", "k", "j", "e", "f"]); - // ^^ 11 12 13 14 15 16 17 18 19 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // a - assert_eq!(mapping[&1], 1..2); // b - assert_eq!(mapping[&2], 2..3); // x - assert_eq!(mapping[&3], 3..4); // x - assert_eq!(mapping[&4], 4..5); // a - assert_eq!(mapping[&5], 5..6); // b - assert_eq!(mapping[&6], 6..7); // c - assert_eq!(mapping[&7], 7..11); // d - assert_eq!(mapping[&8], 11..12); // e - assert_eq!(mapping[&9], 12..13); // f - assert_eq!(mapping[&10], 13..14); // g - - assert_eq!(mapping[&11], 4..5); // a - assert_eq!(mapping[&12], 5..6); // b - assert_eq!(mapping[&13], 6..7); // x - assert_eq!(mapping[&14], 7..8); // c - assert_eq!(mapping[&15], 8..9); // d - assert_eq!(mapping[&16], 9..10); // k - assert_eq!(mapping[&17], 10..11); // j - assert_eq!(mapping[&18], 11..12); // e - assert_eq!(mapping[&19], 12..13); // f - } - - #[test] - fn simple_growing() { - let query = ["new", "york", "subway"]; - // 0 1 2 - let mut builder = QueryWordsMapper::new(&query); - - // new york = new york city - builder.declare(0..2, 3, &["new", "york", "city"]); - // ^ 3 4 5 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // new - assert_eq!(mapping[&1], 1..3); // york - assert_eq!(mapping[&2], 3..4); // subway - assert_eq!(mapping[&3], 0..1); // new - assert_eq!(mapping[&4], 1..2); // york - assert_eq!(mapping[&5], 2..3); // city - } - - #[test] - fn same_place_growings() { - let query = ["NY", "subway"]; - // 0 1 - let mut builder = QueryWordsMapper::new(&query); - - // NY = new york - builder.declare(0..1, 2, &["new", "york"]); - // ^ 2 3 - - // NY = new york city - builder.declare(0..1, 4, &["new", "york", "city"]); - // ^ 4 5 6 - - // NY = NYC - builder.declare(0..1, 7, &["NYC"]); - // ^ 7 - - // NY = new york city - builder.declare(0..1, 8, &["new", "york", "city"]); - // ^ 8 9 10 - - // subway = underground train - builder.declare(1..2, 11, &["underground", "train"]); - // ^ 11 12 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..3); // NY - assert_eq!(mapping[&1], 3..5); // subway - assert_eq!(mapping[&2], 0..1); // new - assert_eq!(mapping[&3], 1..3); // york - assert_eq!(mapping[&4], 0..1); // new - assert_eq!(mapping[&5], 1..2); // york - assert_eq!(mapping[&6], 2..3); // city - assert_eq!(mapping[&7], 0..3); // NYC - assert_eq!(mapping[&8], 0..1); // new - assert_eq!(mapping[&9], 1..2); // york - assert_eq!(mapping[&10], 2..3); // city - assert_eq!(mapping[&11], 3..4); // underground - assert_eq!(mapping[&12], 4..5); // train - } - - #[test] - fn bigger_growing() { - let query = ["NYC", "subway"]; - // 0 1 - let mut builder = QueryWordsMapper::new(&query); - - // NYC = new york city - builder.declare(0..1, 2, &["new", "york", "city"]); - // ^ 2 3 4 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..3); // NYC - assert_eq!(mapping[&1], 3..4); // subway - assert_eq!(mapping[&2], 0..1); // new - assert_eq!(mapping[&3], 1..2); // york - assert_eq!(mapping[&4], 2..3); // city - } - - #[test] - fn middle_query_growing() { - let query = ["great", "awesome", "NYC", "subway"]; - // 0 1 2 3 - let mut builder = QueryWordsMapper::new(&query); - - // NYC = new york city - builder.declare(2..3, 4, &["new", "york", "city"]); - // ^ 4 5 6 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // great - assert_eq!(mapping[&1], 1..2); // awesome - assert_eq!(mapping[&2], 2..5); // NYC - assert_eq!(mapping[&3], 5..6); // subway - assert_eq!(mapping[&4], 2..3); // new - assert_eq!(mapping[&5], 3..4); // york - assert_eq!(mapping[&6], 4..5); // city - } - - #[test] - fn end_query_growing() { - let query = ["NYC", "subway"]; - // 0 1 - let mut builder = QueryWordsMapper::new(&query); - - // NYC = new york city - builder.declare(1..2, 2, &["underground", "train"]); - // ^ 2 3 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // NYC - assert_eq!(mapping[&1], 1..3); // subway - assert_eq!(mapping[&2], 1..2); // underground - assert_eq!(mapping[&3], 2..3); // train - } - - #[test] - fn multiple_growings() { - let query = ["great", "awesome", "NYC", "subway"]; - // 0 1 2 3 - let mut builder = QueryWordsMapper::new(&query); - - // NYC = new york city - builder.declare(2..3, 4, &["new", "york", "city"]); - // ^ 4 5 6 - - // subway = underground train - builder.declare(3..4, 7, &["underground", "train"]); - // ^ 7 8 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // great - assert_eq!(mapping[&1], 1..2); // awesome - assert_eq!(mapping[&2], 2..5); // NYC - assert_eq!(mapping[&3], 5..7); // subway - assert_eq!(mapping[&4], 2..3); // new - assert_eq!(mapping[&5], 3..4); // york - assert_eq!(mapping[&6], 4..5); // city - assert_eq!(mapping[&7], 5..6); // underground - assert_eq!(mapping[&8], 6..7); // train - } - - #[test] - fn multiple_probable_growings() { - let query = ["great", "awesome", "NYC", "subway"]; - // 0 1 2 3 - let mut builder = QueryWordsMapper::new(&query); - - // NYC = new york city - builder.declare(2..3, 4, &["new", "york", "city"]); - // ^ 4 5 6 - - // subway = underground train - builder.declare(3..4, 7, &["underground", "train"]); - // ^ 7 8 - - // great awesome = good - builder.declare(0..2, 9, &["good"]); - // ^ 9 - - // awesome NYC = NY - builder.declare(1..3, 10, &["NY"]); - // ^^ 10 - - // NYC subway = metro - builder.declare(2..4, 11, &["metro"]); - // ^^ 11 - - let mapping = builder.mapping(); - - assert_eq!(mapping[&0], 0..1); // great - assert_eq!(mapping[&1], 1..2); // awesome - assert_eq!(mapping[&2], 2..5); // NYC - assert_eq!(mapping[&3], 5..7); // subway - assert_eq!(mapping[&4], 2..3); // new - assert_eq!(mapping[&5], 3..4); // york - assert_eq!(mapping[&6], 4..5); // city - assert_eq!(mapping[&7], 5..6); // underground - assert_eq!(mapping[&8], 6..7); // train - assert_eq!(mapping[&9], 0..2); // good - assert_eq!(mapping[&10], 1..5); // NY - assert_eq!(mapping[&11], 2..7); // metro - } -} diff --git a/meilisearch-core/src/ranked_map.rs b/meilisearch-core/src/ranked_map.rs deleted file mode 100644 index 48858c78c..000000000 --- a/meilisearch-core/src/ranked_map.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::io::{Read, Write}; - -use hashbrown::HashMap; -use meilisearch_schema::FieldId; -use serde::{Deserialize, Serialize}; - -use crate::{DocumentId, Number}; - -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(transparent)] -pub struct RankedMap(HashMap<(DocumentId, FieldId), Number>); - -impl RankedMap { - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn insert(&mut self, document: DocumentId, field: FieldId, number: Number) { - self.0.insert((document, field), number); - } - - pub fn remove(&mut self, document: DocumentId, field: FieldId) { - self.0.remove(&(document, field)); - } - - pub fn get(&self, document: DocumentId, field: FieldId) -> Option { - self.0.get(&(document, field)).cloned() - } - - pub fn read_from_bin(reader: R) -> bincode::Result { - bincode::deserialize_from(reader).map(RankedMap) - } - - pub fn write_to_bin(&self, writer: W) -> bincode::Result<()> { - bincode::serialize_into(writer, &self.0) - } -} diff --git a/meilisearch-core/src/raw_document.rs b/meilisearch-core/src/raw_document.rs deleted file mode 100644 index 17955824e..000000000 --- a/meilisearch-core/src/raw_document.rs +++ /dev/null @@ -1,51 +0,0 @@ -use compact_arena::SmallArena; -use sdset::SetBuf; -use crate::DocIndex; -use crate::bucket_sort::{SimpleMatch, BareMatch, PostingsListView}; -use crate::reordered_attrs::ReorderedAttrs; - -pub struct RawDocument<'a, 'tag> { - pub id: crate::DocumentId, - pub bare_matches: &'a mut [BareMatch<'tag>], - pub processed_matches: Vec, - /// The list of minimum `distance` found - pub processed_distances: Vec>, - /// Does this document contains a field - /// with one word that is exactly matching - pub contains_one_word_field: bool, -} - -impl<'a, 'tag> RawDocument<'a, 'tag> { - pub fn new<'txn>( - bare_matches: &'a mut [BareMatch<'tag>], - postings_lists: &mut SmallArena<'tag, PostingsListView<'txn>>, - searchable_attrs: Option<&ReorderedAttrs>, - ) -> RawDocument<'a, 'tag> - { - if let Some(reordered_attrs) = searchable_attrs { - for bm in bare_matches.iter() { - let postings_list = &postings_lists[bm.postings_list]; - - let mut rewritten = Vec::new(); - for di in postings_list.iter() { - if let Some(attribute) = reordered_attrs.get(di.attribute) { - rewritten.push(DocIndex { attribute, ..*di }); - } - } - - let new_postings = SetBuf::from_dirty(rewritten); - postings_lists[bm.postings_list].rewrite_with(new_postings); - } - } - - bare_matches.sort_unstable_by_key(|m| m.query_index); - - RawDocument { - id: bare_matches[0].document_id, - bare_matches, - processed_matches: Vec::new(), - processed_distances: Vec::new(), - contains_one_word_field: false, - } - } -} diff --git a/meilisearch-core/src/raw_indexer.rs b/meilisearch-core/src/raw_indexer.rs deleted file mode 100644 index 3aded1ca5..000000000 --- a/meilisearch-core/src/raw_indexer.rs +++ /dev/null @@ -1,344 +0,0 @@ -use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap}; -use std::convert::TryFrom; - -use meilisearch_schema::IndexedPos; -use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig}; -use meilisearch_tokenizer::{Token, token::SeparatorKind, TokenKind}; -use sdset::SetBuf; - -use crate::{DocIndex, DocumentId}; -use crate::FstSetCow; - -const WORD_LENGTH_LIMIT: usize = 80; - -type Word = Vec; // TODO make it be a SmallVec - -pub struct RawIndexer<'a, A> { - word_limit: usize, // the maximum number of indexed words - words_doc_indexes: BTreeMap>, - docs_words: HashMap>, - analyzer: Analyzer<'a, A>, -} - -pub struct Indexed<'a> { - pub words_doc_indexes: BTreeMap>, - pub docs_words: HashMap>, -} - -impl<'a, A> RawIndexer<'a, A> -where - A: AsRef<[u8]> -{ - pub fn new(stop_words: &'a fst::Set) -> RawIndexer<'a, A> { - RawIndexer::with_word_limit(stop_words, 1000) - } - - pub fn with_word_limit(stop_words: &'a fst::Set, limit: usize) -> RawIndexer { - RawIndexer { - word_limit: limit, - words_doc_indexes: BTreeMap::new(), - docs_words: HashMap::new(), - analyzer: Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)), - } - } - - pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize { - let mut number_of_words = 0; - - let analyzed_text = self.analyzer.analyze(text); - for (token_pos, (word_pos, token)) in process_tokens(analyzed_text.tokens()).enumerate() { - let must_continue = index_token( - token, - word_pos, - token_pos, - id, - indexed_pos, - self.word_limit, - &mut self.words_doc_indexes, - &mut self.docs_words, - ); - - number_of_words += 1; - - if !must_continue { - break; - } - } - - number_of_words - } - - pub fn index_text_seq<'s, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, text_iter: I) - where - I: IntoIterator, - { - let mut word_offset = 0; - - for text in text_iter.into_iter() { - let current_word_offset = word_offset; - - let analyzed_text = self.analyzer.analyze(text); - let tokens = process_tokens(analyzed_text.tokens()) - .map(|(i, t)| (i + current_word_offset, t)) - .enumerate(); - - for (token_pos, (word_pos, token)) in tokens { - word_offset = word_pos + 1; - - let must_continue = index_token( - token, - word_pos, - token_pos, - id, - indexed_pos, - self.word_limit, - &mut self.words_doc_indexes, - &mut self.docs_words, - ); - - if !must_continue { - break; - } - } - } - } - - pub fn build(self) -> Indexed<'static> { - let words_doc_indexes = self - .words_doc_indexes - .into_iter() - .map(|(word, indexes)| (word, SetBuf::from_dirty(indexes))) - .collect(); - - let docs_words = self - .docs_words - .into_iter() - .map(|(id, mut words)| { - words.sort_unstable(); - words.dedup(); - let fst = fst::Set::from_iter(words).unwrap().map_data(Cow::Owned).unwrap(); - (id, fst) - }) - .collect(); - - Indexed { - words_doc_indexes, - docs_words, - } - } -} - -fn process_tokens<'a>(tokens: impl Iterator>) -> impl Iterator)> { - tokens - .skip_while(|token| !token.is_word()) - .scan((0, None), |(offset, prev_kind), token| { - match token.kind { - TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => { - *offset += match *prev_kind { - Some(TokenKind::Separator(SeparatorKind::Hard)) => 8, - Some(_) => 1, - None => 0, - }; - *prev_kind = Some(token.kind) - } - TokenKind::Separator(SeparatorKind::Hard) => { - *prev_kind = Some(token.kind); - } - TokenKind::Separator(SeparatorKind::Soft) - if *prev_kind != Some(TokenKind::Separator(SeparatorKind::Hard)) => { - *prev_kind = Some(token.kind); - } - _ => (), - } - Some((*offset, token)) - }) - .filter(|(_, t)| t.is_word()) -} - -#[allow(clippy::too_many_arguments)] -fn index_token( - token: Token, - word_pos: usize, - token_pos: usize, - id: DocumentId, - indexed_pos: IndexedPos, - word_limit: usize, - words_doc_indexes: &mut BTreeMap>, - docs_words: &mut HashMap>, -) -> bool -{ - if token_pos >= word_limit { - return false; - } - - if !token.is_stopword() { - match token_to_docindex(id, indexed_pos, &token, word_pos) { - Some(docindex) => { - let word = Vec::from(token.word.as_ref()); - - if word.len() <= WORD_LENGTH_LIMIT { - words_doc_indexes - .entry(word.clone()) - .or_insert_with(Vec::new) - .push(docindex); - docs_words.entry(id).or_insert_with(Vec::new).push(word); - } - } - None => return false, - } - } - - true -} - -fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: &Token, word_index: usize) -> Option { - let word_index = u16::try_from(word_index).ok()?; - let char_index = u16::try_from(token.byte_start).ok()?; - let char_length = u16::try_from(token.word.len()).ok()?; - - let docindex = DocIndex { - document_id: id, - attribute: indexed_pos.0, - word_index, - char_index, - char_length, - }; - - Some(docindex) -} - -#[cfg(test)] -mod tests { - use super::*; - use meilisearch_schema::IndexedPos; - use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; - use fst::Set; - - #[test] - fn test_process_token() { - let text = " 為一包含一千多萬目詞的帶標記平衡語料庫"; - let stopwords = Set::default(); - let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(&stopwords)); - let analyzer = analyzer.analyze(text); - let tokens: Vec<_> = process_tokens(analyzer.tokens()).map(|(_, t)| t.text().to_string()).collect(); - assert_eq!(tokens, ["为", "一", "包含", "一千多万", "目词", "的", "带", "标记", "平衡", "语料库"]); - } - - #[test] - fn strange_apostrophe() { - let stop_words = fst::Set::default(); - let mut indexer = RawIndexer::new(&stop_words); - - let docid = DocumentId(0); - let indexed_pos = IndexedPos(0); - let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !"; - indexer.index_text(docid, indexed_pos, text); - - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - - assert!(words_doc_indexes.get(&b"l"[..]).is_some()); - assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some()); - assert!(words_doc_indexes.get(&b"ai"[..]).is_some()); - assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some()); - } - - #[test] - fn strange_apostrophe_in_sequence() { - let stop_words = fst::Set::default(); - let mut indexer = RawIndexer::new(&stop_words); - - let docid = DocumentId(0); - let indexed_pos = IndexedPos(0); - let text = vec!["Zut, l’aspirateur, j’ai oublié de l’éteindre !"]; - indexer.index_text_seq(docid, indexed_pos, text); - - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - - assert!(words_doc_indexes.get(&b"l"[..]).is_some()); - assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some()); - assert!(words_doc_indexes.get(&b"ai"[..]).is_some()); - assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some()); - } - - #[test] - fn basic_stop_words() { - let stop_words = sdset::SetBuf::from_dirty(vec!["l", "j", "ai", "de"]); - let stop_words = fst::Set::from_iter(stop_words).unwrap(); - - let mut indexer = RawIndexer::new(&stop_words); - - let docid = DocumentId(0); - let indexed_pos = IndexedPos(0); - let text = "Zut, l’aspirateur, j’ai oublié de l’éteindre !"; - indexer.index_text(docid, indexed_pos, text); - - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - - assert!(words_doc_indexes.get(&b"l"[..]).is_none()); - assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some()); - assert!(words_doc_indexes.get(&b"j"[..]).is_none()); - assert!(words_doc_indexes.get(&b"ai"[..]).is_none()); - assert!(words_doc_indexes.get(&b"de"[..]).is_none()); - assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some()); - } - - #[test] - fn no_empty_unidecode() { - let stop_words = fst::Set::default(); - let mut indexer = RawIndexer::new(&stop_words); - - let docid = DocumentId(0); - let indexed_pos = IndexedPos(0); - let text = "🇯🇵"; - indexer.index_text(docid, indexed_pos, text); - - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - - assert!(words_doc_indexes - .get(&"🇯🇵".to_owned().into_bytes()) - .is_some()); - } - - #[test] - // test sample from 807 - fn very_long_text() { - let stop_words = fst::Set::default(); - let mut indexer = RawIndexer::new(&stop_words); - let indexed_pos = IndexedPos(0); - let docid = DocumentId(0); - let text = " The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n "; - indexer.index_text(docid, indexed_pos, text); - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - assert!(words_doc_indexes.get(&"request".to_owned().into_bytes()).is_some()); - } - - #[test] - fn words_over_index_1000_not_indexed() { - let stop_words = fst::Set::default(); - let mut indexer = RawIndexer::new(&stop_words); - let indexed_pos = IndexedPos(0); - let docid = DocumentId(0); - let mut text = String::with_capacity(5000); - for _ in 0..1000 { - text.push_str("less "); - } - text.push_str("more"); - indexer.index_text(docid, indexed_pos, &text); - let Indexed { - words_doc_indexes, .. - } = indexer.build(); - assert!(words_doc_indexes.get(&"less".to_owned().into_bytes()).is_some()); - assert!(words_doc_indexes.get(&"more".to_owned().into_bytes()).is_none()); - } -} diff --git a/meilisearch-core/src/reordered_attrs.rs b/meilisearch-core/src/reordered_attrs.rs deleted file mode 100644 index 590cac7b2..000000000 --- a/meilisearch-core/src/reordered_attrs.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::cmp; - -#[derive(Default, Clone)] -pub struct ReorderedAttrs { - reorders: Vec>, - reverse: Vec, -} - -impl ReorderedAttrs { - pub fn new() -> ReorderedAttrs { - ReorderedAttrs { reorders: Vec::new(), reverse: Vec::new() } - } - - pub fn insert_attribute(&mut self, attribute: u16) { - let new_len = cmp::max(attribute as usize + 1, self.reorders.len()); - self.reorders.resize(new_len, None); - self.reorders[attribute as usize] = Some(self.reverse.len() as u16); - self.reverse.push(attribute); - } - - pub fn get(&self, attribute: u16) -> Option { - match self.reorders.get(attribute as usize)? { - Some(attribute) => Some(*attribute), - None => None, - } - } - - pub fn reverse(&self, attribute: u16) -> Option { - self.reverse.get(attribute as usize).copied() - } -} diff --git a/meilisearch-core/src/serde/deserializer.rs b/meilisearch-core/src/serde/deserializer.rs deleted file mode 100644 index 0d091951e..000000000 --- a/meilisearch-core/src/serde/deserializer.rs +++ /dev/null @@ -1,161 +0,0 @@ -use std::collections::HashSet; -use std::io::Cursor; -use std::{error::Error, fmt}; - -use meilisearch_schema::{Schema, FieldId}; -use serde::{de, forward_to_deserialize_any}; -use serde_json::de::IoRead as SerdeJsonIoRead; -use serde_json::Deserializer as SerdeJsonDeserializer; -use serde_json::Error as SerdeJsonError; - -use crate::database::MainT; -use crate::store::DocumentsFields; -use crate::DocumentId; - -#[derive(Debug)] -pub enum DeserializerError { - SerdeJson(SerdeJsonError), - Zlmdb(heed::Error), - Custom(String), -} - -impl de::Error for DeserializerError { - fn custom(msg: T) -> Self { - DeserializerError::Custom(msg.to_string()) - } -} - -impl fmt::Display for DeserializerError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e), - DeserializerError::Zlmdb(e) => write!(f, "heed related error: {}", e), - DeserializerError::Custom(s) => f.write_str(s), - } - } -} - -impl Error for DeserializerError {} - -impl From for DeserializerError { - fn from(error: SerdeJsonError) -> DeserializerError { - DeserializerError::SerdeJson(error) - } -} - -impl From for DeserializerError { - fn from(error: heed::Error) -> DeserializerError { - DeserializerError::Zlmdb(error) - } -} - -pub struct Deserializer<'a> { - pub document_id: DocumentId, - pub reader: &'a heed::RoTxn<'a, MainT>, - pub documents_fields: DocumentsFields, - pub schema: &'a Schema, - pub fields: Option<&'a HashSet>, -} - -impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> { - type Error = DeserializerError; - - fn deserialize_any(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_option(visitor) - } - - fn deserialize_option(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_map(visitor) - } - - fn deserialize_map(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let mut error = None; - - let iter = self - .documents_fields - .document_fields(self.reader, self.document_id)? - .filter_map(|result| { - let (attr, value) = match result { - Ok(value) => value, - Err(e) => { - error = Some(e); - return None; - } - }; - - let is_displayed = self.schema.is_displayed(attr); - if is_displayed && self.fields.map_or(true, |f| f.contains(&attr)) { - if let Some(attribute_name) = self.schema.name(attr) { - let cursor = Cursor::new(value.to_owned()); - let ioread = SerdeJsonIoRead::new(cursor); - let value = Value(SerdeJsonDeserializer::new(ioread)); - - Some((attribute_name, value)) - } else { - None - } - } else { - None - } - }); - - let mut iter = iter.peekable(); - - let result = match iter.peek() { - Some(_) => { - let map_deserializer = de::value::MapDeserializer::new(iter); - visitor - .visit_some(map_deserializer) - .map_err(DeserializerError::from) - } - None => visitor.visit_none(), - }; - - match error.take() { - Some(error) => Err(error.into()), - None => result, - } - } - - forward_to_deserialize_any! { - bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string - bytes byte_buf unit unit_struct newtype_struct seq tuple - tuple_struct struct enum identifier ignored_any - } -} - -struct Value(SerdeJsonDeserializer>>>); - -impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value { - type Deserializer = Self; - - fn into_deserializer(self) -> Self::Deserializer { - self - } -} - -impl<'de> de::Deserializer<'de> for Value { - type Error = SerdeJsonError; - - fn deserialize_any(mut self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.0.deserialize_any(visitor) - } - - forward_to_deserialize_any! { - bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string - bytes byte_buf option unit unit_struct newtype_struct seq tuple - tuple_struct map struct enum identifier ignored_any - } -} diff --git a/meilisearch-core/src/serde/mod.rs b/meilisearch-core/src/serde/mod.rs deleted file mode 100644 index 24834b1b7..000000000 --- a/meilisearch-core/src/serde/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -mod deserializer; - -pub use self::deserializer::{Deserializer, DeserializerError}; - -use std::{error::Error, fmt}; - -use serde::ser; -use serde_json::Error as SerdeJsonError; -use meilisearch_schema::Error as SchemaError; - -use crate::ParseNumberError; - -#[derive(Debug)] -pub enum SerializerError { - DocumentIdNotFound, - InvalidDocumentIdFormat, - Zlmdb(heed::Error), - SerdeJson(SerdeJsonError), - ParseNumber(ParseNumberError), - Schema(SchemaError), - UnserializableType { type_name: &'static str }, - UnindexableType { type_name: &'static str }, - UnrankableType { type_name: &'static str }, - Custom(String), -} - -impl ser::Error for SerializerError { - fn custom(msg: T) -> Self { - SerializerError::Custom(msg.to_string()) - } -} - -impl fmt::Display for SerializerError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - SerializerError::DocumentIdNotFound => { - f.write_str("Primary key is missing.") - } - SerializerError::InvalidDocumentIdFormat => { - f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).") - } - SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e), - SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e), - SerializerError::ParseNumber(e) => { - write!(f, "error while trying to parse a number: {}", e) - } - SerializerError::Schema(e) => write!(f, "impossible to update schema: {}", e), - SerializerError::UnserializableType { type_name } => { - write!(f, "{} is not a serializable type", type_name) - } - SerializerError::UnindexableType { type_name } => { - write!(f, "{} is not an indexable type", type_name) - } - SerializerError::UnrankableType { type_name } => { - write!(f, "{} types can not be used for ranking", type_name) - } - SerializerError::Custom(s) => f.write_str(s), - } - } -} - -impl Error for SerializerError {} - -impl From for SerializerError { - fn from(value: String) -> SerializerError { - SerializerError::Custom(value) - } -} - -impl From for SerializerError { - fn from(error: SerdeJsonError) -> SerializerError { - SerializerError::SerdeJson(error) - } -} - -impl From for SerializerError { - fn from(error: heed::Error) -> SerializerError { - SerializerError::Zlmdb(error) - } -} - -impl From for SerializerError { - fn from(error: ParseNumberError) -> SerializerError { - SerializerError::ParseNumber(error) - } -} - -impl From for SerializerError { - fn from(error: SchemaError) -> SerializerError { - SerializerError::Schema(error) - } -} diff --git a/meilisearch-core/src/settings.rs b/meilisearch-core/src/settings.rs deleted file mode 100644 index f26865dd7..000000000 --- a/meilisearch-core/src/settings.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::str::FromStr; -use std::iter::IntoIterator; - -use serde::{Deserialize, Deserializer, Serialize}; -use once_cell::sync::Lazy; - -use self::RankingRule::*; - -pub const DEFAULT_RANKING_RULES: [RankingRule; 6] = [Typo, Words, Proximity, Attribute, WordsPosition, Exactness]; - -static RANKING_RULE_REGEX: Lazy = Lazy::new(|| { - regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap() -}); - -#[derive(Default, Clone, Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Settings { - #[serde(default, deserialize_with = "deserialize_some")] - pub ranking_rules: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub distinct_attribute: Option>, - #[serde(default, deserialize_with = "deserialize_some")] - pub searchable_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub displayed_attributes: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub stop_words: Option>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub synonyms: Option>>>, - #[serde(default, deserialize_with = "deserialize_some")] - pub attributes_for_faceting: Option>>, -} - -// Any value that is present is considered Some value, including null. -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> - where T: Deserialize<'de>, - D: Deserializer<'de> -{ - Deserialize::deserialize(deserializer).map(Some) -} - -impl Settings { - pub fn to_update(&self) -> Result { - let settings = self.clone(); - - let ranking_rules = match settings.ranking_rules { - Some(Some(rules)) => UpdateState::Update(RankingRule::try_from_iter(rules.iter())?), - Some(None) => UpdateState::Clear, - None => UpdateState::Nothing, - }; - - Ok(SettingsUpdate { - ranking_rules, - distinct_attribute: settings.distinct_attribute.into(), - primary_key: UpdateState::Nothing, - searchable_attributes: settings.searchable_attributes.into(), - displayed_attributes: settings.displayed_attributes.into(), - stop_words: settings.stop_words.into(), - synonyms: settings.synonyms.into(), - attributes_for_faceting: settings.attributes_for_faceting.into(), - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateState { - Update(T), - Clear, - Nothing, -} - -impl From>> for UpdateState { - fn from(opt: Option>) -> UpdateState { - match opt { - Some(Some(t)) => UpdateState::Update(t), - Some(None) => UpdateState::Clear, - None => UpdateState::Nothing, - } - } -} - -#[derive(Debug, Clone)] -pub struct RankingRuleConversionError; - -impl std::fmt::Display for RankingRuleConversionError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "impossible to convert into RankingRule") - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum RankingRule { - Typo, - Words, - Proximity, - Attribute, - WordsPosition, - Exactness, - Asc(String), - Desc(String), -} - -impl std::fmt::Display for RankingRule { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - RankingRule::Typo => f.write_str("typo"), - RankingRule::Words => f.write_str("words"), - RankingRule::Proximity => f.write_str("proximity"), - RankingRule::Attribute => f.write_str("attribute"), - RankingRule::WordsPosition => f.write_str("wordsPosition"), - RankingRule::Exactness => f.write_str("exactness"), - RankingRule::Asc(field) => write!(f, "asc({})", field), - RankingRule::Desc(field) => write!(f, "desc({})", field), - } - } -} - -impl FromStr for RankingRule { - type Err = RankingRuleConversionError; - - fn from_str(s: &str) -> Result { - let rule = match s { - "typo" => RankingRule::Typo, - "words" => RankingRule::Words, - "proximity" => RankingRule::Proximity, - "attribute" => RankingRule::Attribute, - "wordsPosition" => RankingRule::WordsPosition, - "exactness" => RankingRule::Exactness, - _ => { - let captures = RANKING_RULE_REGEX.captures(s).ok_or(RankingRuleConversionError)?; - match (captures.get(1).map(|m| m.as_str()), captures.get(2)) { - (Some("asc"), Some(field)) => RankingRule::Asc(field.as_str().to_string()), - (Some("desc"), Some(field)) => RankingRule::Desc(field.as_str().to_string()), - _ => return Err(RankingRuleConversionError) - } - } - }; - Ok(rule) - } -} - -impl RankingRule { - pub fn field(&self) -> Option<&str> { - match self { - RankingRule::Asc(field) | RankingRule::Desc(field) => Some(field), - _ => None, - } - } - - pub fn try_from_iter(rules: impl IntoIterator>) -> Result, RankingRuleConversionError> { - rules.into_iter() - .map(|s| RankingRule::from_str(s.as_ref())) - .collect() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SettingsUpdate { - pub ranking_rules: UpdateState>, - pub distinct_attribute: UpdateState, - pub primary_key: UpdateState, - pub searchable_attributes: UpdateState>, - pub displayed_attributes: UpdateState>, - pub stop_words: UpdateState>, - pub synonyms: UpdateState>>, - pub attributes_for_faceting: UpdateState>, -} - -impl Default for SettingsUpdate { - fn default() -> Self { - Self { - ranking_rules: UpdateState::Nothing, - distinct_attribute: UpdateState::Nothing, - primary_key: UpdateState::Nothing, - searchable_attributes: UpdateState::Nothing, - displayed_attributes: UpdateState::Nothing, - stop_words: UpdateState::Nothing, - synonyms: UpdateState::Nothing, - attributes_for_faceting: UpdateState::Nothing, - } - } -} diff --git a/meilisearch-core/src/store/cow_set.rs b/meilisearch-core/src/store/cow_set.rs deleted file mode 100644 index 063f73198..000000000 --- a/meilisearch-core/src/store/cow_set.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::borrow::Cow; - -use heed::{types::CowSlice, BytesEncode, BytesDecode}; -use sdset::{Set, SetBuf}; -use zerocopy::{AsBytes, FromBytes}; - -pub struct CowSet(std::marker::PhantomData); - -impl<'a, T: 'a> BytesEncode<'a> for CowSet -where - T: AsBytes, -{ - type EItem = Set; - - fn bytes_encode(item: &'a Self::EItem) -> Option> { - CowSlice::bytes_encode(item.as_slice()) - } -} - -impl<'a, T: 'a> BytesDecode<'a> for CowSet -where - T: FromBytes + Copy, -{ - type DItem = Cow<'a, Set>; - - fn bytes_decode(bytes: &'a [u8]) -> Option { - match CowSlice::::bytes_decode(bytes)? { - Cow::Owned(vec) => Some(Cow::Owned(SetBuf::new_unchecked(vec))), - Cow::Borrowed(slice) => Some(Cow::Borrowed(Set::new_unchecked(slice))), - } - } -} diff --git a/meilisearch-core/src/store/docs_words.rs b/meilisearch-core/src/store/docs_words.rs deleted file mode 100644 index 6b412d584..000000000 --- a/meilisearch-core/src/store/docs_words.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::borrow::Cow; - -use heed::Result as ZResult; -use heed::types::{ByteSlice, OwnedType}; - -use crate::database::MainT; -use crate::{DocumentId, FstSetCow}; -use super::BEU32; - -#[derive(Copy, Clone)] -pub struct DocsWords { - pub(crate) docs_words: heed::Database, ByteSlice>, -} - -impl DocsWords { - pub fn put_doc_words( - self, - writer: &mut heed::RwTxn, - document_id: DocumentId, - words: &FstSetCow, - ) -> ZResult<()> { - let document_id = BEU32::new(document_id.0); - let bytes = words.as_fst().as_bytes(); - self.docs_words.put(writer, &document_id, bytes) - } - - pub fn del_doc_words(self, writer: &mut heed::RwTxn, document_id: DocumentId) -> ZResult { - let document_id = BEU32::new(document_id.0); - self.docs_words.delete(writer, &document_id) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.docs_words.clear(writer) - } - - pub fn doc_words<'a>(self, reader: &'a heed::RoTxn<'a, MainT>, document_id: DocumentId) -> ZResult { - let document_id = BEU32::new(document_id.0); - match self.docs_words.get(reader, &document_id)? { - Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), - } - } -} diff --git a/meilisearch-core/src/store/documents_fields.rs b/meilisearch-core/src/store/documents_fields.rs deleted file mode 100644 index 1dcad8488..000000000 --- a/meilisearch-core/src/store/documents_fields.rs +++ /dev/null @@ -1,79 +0,0 @@ -use heed::types::{ByteSlice, OwnedType}; -use crate::database::MainT; -use heed::Result as ZResult; -use meilisearch_schema::FieldId; - -use super::DocumentFieldStoredKey; -use crate::DocumentId; - -#[derive(Copy, Clone)] -pub struct DocumentsFields { - pub(crate) documents_fields: heed::Database, ByteSlice>, -} - -impl DocumentsFields { - pub fn put_document_field( - self, - writer: &mut heed::RwTxn, - document_id: DocumentId, - field: FieldId, - value: &[u8], - ) -> ZResult<()> { - let key = DocumentFieldStoredKey::new(document_id, field); - self.documents_fields.put(writer, &key, value) - } - - pub fn del_all_document_fields( - self, - writer: &mut heed::RwTxn, - document_id: DocumentId, - ) -> ZResult { - let start = DocumentFieldStoredKey::new(document_id, FieldId::min()); - let end = DocumentFieldStoredKey::new(document_id, FieldId::max()); - self.documents_fields.delete_range(writer, &(start..=end)) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.documents_fields.clear(writer) - } - - pub fn document_attribute<'txn>( - self, - reader: &'txn heed::RoTxn, - document_id: DocumentId, - field: FieldId, - ) -> ZResult> { - let key = DocumentFieldStoredKey::new(document_id, field); - self.documents_fields.get(reader, &key) - } - - pub fn document_fields<'txn>( - self, - reader: &'txn heed::RoTxn, - document_id: DocumentId, - ) -> ZResult> { - let start = DocumentFieldStoredKey::new(document_id, FieldId::min()); - let end = DocumentFieldStoredKey::new(document_id, FieldId::max()); - let iter = self.documents_fields.range(reader, &(start..=end))?; - Ok(DocumentFieldsIter { iter }) - } -} - -pub struct DocumentFieldsIter<'txn> { - iter: heed::RoRange<'txn, OwnedType, ByteSlice>, -} - -impl<'txn> Iterator for DocumentFieldsIter<'txn> { - type Item = ZResult<(FieldId, &'txn [u8])>; - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((key, bytes))) => { - let field_id = FieldId(key.field_id.get()); - Some(Ok((field_id, bytes))) - } - Some(Err(e)) => Some(Err(e)), - None => None, - } - } -} diff --git a/meilisearch-core/src/store/documents_fields_counts.rs b/meilisearch-core/src/store/documents_fields_counts.rs deleted file mode 100644 index f0d23c99b..000000000 --- a/meilisearch-core/src/store/documents_fields_counts.rs +++ /dev/null @@ -1,143 +0,0 @@ -use super::DocumentFieldIndexedKey; -use crate::database::MainT; -use crate::DocumentId; -use heed::types::OwnedType; -use heed::Result as ZResult; -use meilisearch_schema::IndexedPos; -use crate::MResult; - -#[derive(Copy, Clone)] -pub struct DocumentsFieldsCounts { - pub(crate) documents_fields_counts: heed::Database, OwnedType>, -} - -impl DocumentsFieldsCounts { - pub fn put_document_field_count( - self, - writer: &mut heed::RwTxn, - document_id: DocumentId, - attribute: IndexedPos, - value: u16, - ) -> ZResult<()> { - let key = DocumentFieldIndexedKey::new(document_id, attribute); - self.documents_fields_counts.put(writer, &key, &value) - } - - pub fn del_all_document_fields_counts( - self, - writer: &mut heed::RwTxn, - document_id: DocumentId, - ) -> ZResult { - let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min()); - let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max()); - self.documents_fields_counts.delete_range(writer, &(start..=end)) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.documents_fields_counts.clear(writer) - } - - pub fn document_field_count( - self, - reader: &heed::RoTxn, - document_id: DocumentId, - attribute: IndexedPos, - ) -> ZResult> { - let key = DocumentFieldIndexedKey::new(document_id, attribute); - match self.documents_fields_counts.get(reader, &key)? { - Some(count) => Ok(Some(count)), - None => Ok(None), - } - } - - pub fn document_fields_counts<'txn>( - self, - reader: &'txn heed::RoTxn, - document_id: DocumentId, - ) -> ZResult> { - let start = DocumentFieldIndexedKey::new(document_id, IndexedPos::min()); - let end = DocumentFieldIndexedKey::new(document_id, IndexedPos::max()); - let iter = self.documents_fields_counts.range(reader, &(start..=end))?; - Ok(DocumentFieldsCountsIter { iter }) - } - - pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn) -> MResult> { - let iter = self.documents_fields_counts.iter(reader)?; - Ok(DocumentsIdsIter { - last_seen_id: None, - iter, - }) - } - - pub fn all_documents_fields_counts<'txn>( - self, - reader: &'txn heed::RoTxn, - ) -> ZResult> { - let iter = self.documents_fields_counts.iter(reader)?; - Ok(AllDocumentsFieldsCountsIter { iter }) - } -} - -pub struct DocumentFieldsCountsIter<'txn> { - iter: heed::RoRange<'txn, OwnedType, OwnedType>, -} - -impl Iterator for DocumentFieldsCountsIter<'_> { - type Item = ZResult<(IndexedPos, u16)>; - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((key, count))) => { - let indexed_pos = IndexedPos(key.indexed_pos.get()); - Some(Ok((indexed_pos, count))) - } - Some(Err(e)) => Some(Err(e)), - None => None, - } - } -} - -pub struct DocumentsIdsIter<'txn> { - last_seen_id: Option, - iter: heed::RoIter<'txn, OwnedType, OwnedType>, -} - -impl Iterator for DocumentsIdsIter<'_> { - type Item = MResult; - - fn next(&mut self) -> Option { - for result in &mut self.iter { - match result { - Ok((key, _)) => { - let document_id = DocumentId(key.docid.get()); - if Some(document_id) != self.last_seen_id { - self.last_seen_id = Some(document_id); - return Some(Ok(document_id)); - } - } - Err(e) => return Some(Err(e.into())), - } - } - None - } -} - -pub struct AllDocumentsFieldsCountsIter<'txn> { - iter: heed::RoIter<'txn, OwnedType, OwnedType>, -} - -impl Iterator for AllDocumentsFieldsCountsIter<'_> { - type Item = ZResult<(DocumentId, IndexedPos, u16)>; - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((key, count))) => { - let docid = DocumentId(key.docid.get()); - let indexed_pos = IndexedPos(key.indexed_pos.get()); - Some(Ok((docid, indexed_pos, count))) - } - Some(Err(e)) => Some(Err(e)), - None => None, - } - } -} diff --git a/meilisearch-core/src/store/documents_ids.rs b/meilisearch-core/src/store/documents_ids.rs deleted file mode 100644 index a0628e9e2..000000000 --- a/meilisearch-core/src/store/documents_ids.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::borrow::Cow; - -use heed::{BytesDecode, BytesEncode}; -use sdset::Set; - -use crate::DocumentId; -use super::cow_set::CowSet; - -pub struct DocumentsIds; - -impl BytesEncode<'_> for DocumentsIds { - type EItem = Set; - - fn bytes_encode(item: &Self::EItem) -> Option> { - CowSet::bytes_encode(item) - } -} - -impl<'a> BytesDecode<'a> for DocumentsIds { - type DItem = Cow<'a, Set>; - - fn bytes_decode(bytes: &'a [u8]) -> Option { - CowSet::bytes_decode(bytes) - } -} - -pub struct DiscoverIds<'a> { - ids_iter: std::slice::Iter<'a, DocumentId>, - left_id: Option, - right_id: Option, - available_range: std::ops::Range, -} - -impl DiscoverIds<'_> { - pub fn new(ids: &Set) -> DiscoverIds { - let mut ids_iter = ids.iter(); - let right_id = ids_iter.next().map(|id| id.0); - let available_range = 0..right_id.unwrap_or(u32::max_value()); - DiscoverIds { ids_iter, left_id: None, right_id, available_range } - } -} - -impl Iterator for DiscoverIds<'_> { - type Item = DocumentId; - - fn next(&mut self) -> Option { - loop { - match self.available_range.next() { - // The available range gives us a new id, we return it. - Some(id) => return Some(DocumentId(id)), - // The available range is exhausted, we need to find the next one. - None if self.available_range.end == u32::max_value() => return None, - None => loop { - self.left_id = self.right_id.take(); - self.right_id = self.ids_iter.next().map(|id| id.0); - match (self.left_id, self.right_id) { - // We found a gap in the used ids, we can yield all ids - // until the end of the gap - (Some(l), Some(r)) => if l.saturating_add(1) != r { - self.available_range = (l + 1)..r; - break; - }, - // The last used id has been reached, we can use all ids - // until u32 MAX - (Some(l), None) => { - self.available_range = l.saturating_add(1)..u32::max_value(); - break; - }, - _ => (), - } - }, - } - } - } -} diff --git a/meilisearch-core/src/store/facets.rs b/meilisearch-core/src/store/facets.rs deleted file mode 100644 index 6766a8a01..000000000 --- a/meilisearch-core/src/store/facets.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::borrow::Cow; -use std::collections::HashMap; -use std::mem; - -use heed::{RwTxn, RoTxn, RoPrefix, types::Str, BytesEncode, BytesDecode}; -use sdset::{SetBuf, Set, SetOperation}; - -use meilisearch_types::DocumentId; -use meilisearch_schema::FieldId; - -use crate::MResult; -use crate::database::MainT; -use crate::facets::FacetKey; -use super::cow_set::CowSet; - -/// contains facet info -#[derive(Clone, Copy)] -pub struct Facets { - pub(crate) facets: heed::Database, -} - -pub struct FacetData; - -impl<'a> BytesEncode<'a> for FacetData { - type EItem = (&'a str, &'a Set); - - fn bytes_encode(item: &'a Self::EItem) -> Option> { - // get size of the first item - let first_size = item.0.as_bytes().len(); - let size = mem::size_of::() - + first_size - + item.1.len() * mem::size_of::(); - let mut buffer = Vec::with_capacity(size); - // encode the length of the first item - buffer.extend_from_slice(&first_size.to_be_bytes()); - buffer.extend_from_slice(Str::bytes_encode(&item.0)?.as_ref()); - let second_slice = CowSet::bytes_encode(&item.1)?; - buffer.extend_from_slice(second_slice.as_ref()); - Some(Cow::Owned(buffer)) - } -} - -impl<'a> BytesDecode<'a> for FacetData { - type DItem = (&'a str, Cow<'a, Set>); - - fn bytes_decode(bytes: &'a [u8]) -> Option { - const LEN: usize = mem::size_of::(); - let mut size_buf = [0; LEN]; - size_buf.copy_from_slice(bytes.get(0..LEN)?); - // decode size of the first item from the bytes - let first_size = u64::from_be_bytes(size_buf); - // decode first and second items - let first_item = Str::bytes_decode(bytes.get(LEN..(LEN + first_size as usize))?)?; - let second_item = CowSet::bytes_decode(bytes.get((LEN + first_size as usize)..)?)?; - Some((first_item, second_item)) - } -} - -impl Facets { - // we use sdset::SetBuf to ensure the docids are sorted. - pub fn put_facet_document_ids(&self, writer: &mut RwTxn, facet_key: FacetKey, doc_ids: &Set, facet_value: &str) -> MResult<()> { - Ok(self.facets.put(writer, &facet_key, &(facet_value, doc_ids))?) - } - - pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn, field_id: FieldId) -> MResult> { - Ok(self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))?) - } - - pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn, facet_key: &FacetKey) -> MResult>)>> { - Ok(self.facets.get(reader, &facet_key)?) - } - - /// updates the facets store, revmoving the documents from the facets provided in the - /// `facet_map` argument - pub fn remove(&self, writer: &mut RwTxn, facet_map: HashMap)>) -> MResult<()> { - for (key, (name, document_ids)) in facet_map { - if let Some((_, old)) = self.facets.get(writer, &key)? { - let to_remove = SetBuf::from_dirty(document_ids); - let new = sdset::duo::OpBuilder::new(old.as_ref(), to_remove.as_set()).difference().into_set_buf(); - self.facets.put(writer, &key, &(&name, new.as_set()))?; - } - } - Ok(()) - } - - pub fn add(&self, writer: &mut RwTxn, facet_map: HashMap)>) -> MResult<()> { - for (key, (facet_name, document_ids)) in facet_map { - let set = SetBuf::from_dirty(document_ids); - self.put_facet_document_ids(writer, key, set.as_set(), &facet_name)?; - } - Ok(()) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> MResult<()> { - Ok(self.facets.clear(writer)?) - } -} diff --git a/meilisearch-core/src/store/main.rs b/meilisearch-core/src/store/main.rs deleted file mode 100644 index 2b60a5680..000000000 --- a/meilisearch-core/src/store/main.rs +++ /dev/null @@ -1,320 +0,0 @@ -use std::borrow::Cow; -use std::collections::BTreeMap; - -use chrono::{DateTime, Utc}; -use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str, CowSlice}; -use meilisearch_schema::{FieldId, Schema}; -use meilisearch_types::DocumentId; -use sdset::Set; - -use crate::database::MainT; -use crate::{RankedMap, MResult}; -use crate::settings::RankingRule; -use crate::{FstSetCow, FstMapCow}; -use super::{CowSet, DocumentsIds}; - -const ATTRIBUTES_FOR_FACETING_KEY: &str = "attributes-for-faceting"; -const CREATED_AT_KEY: &str = "created-at"; -const CUSTOMS_KEY: &str = "customs"; -const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute"; -const EXTERNAL_DOCIDS_KEY: &str = "external-docids"; -const FIELDS_DISTRIBUTION_KEY: &str = "fields-distribution"; -const INTERNAL_DOCIDS_KEY: &str = "internal-docids"; -const NAME_KEY: &str = "name"; -const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; -const RANKED_MAP_KEY: &str = "ranked-map"; -const RANKING_RULES_KEY: &str = "ranking-rules"; -const SCHEMA_KEY: &str = "schema"; -const SORTED_DOCUMENT_IDS_CACHE_KEY: &str = "sorted-document-ids-cache"; -const STOP_WORDS_KEY: &str = "stop-words"; -const SYNONYMS_KEY: &str = "synonyms"; -const UPDATED_AT_KEY: &str = "updated-at"; -const WORDS_KEY: &str = "words"; - -pub type FreqsMap = BTreeMap; -type SerdeFreqsMap = SerdeBincode; -type SerdeDatetime = SerdeBincode>; - -#[derive(Copy, Clone)] -pub struct Main { - pub(crate) main: heed::PolyDatabase, -} - -impl Main { - pub fn clear(self, writer: &mut heed::RwTxn) -> MResult<()> { - Ok(self.main.clear(writer)?) - } - - pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> MResult<()> { - Ok(self.main.put::<_, Str, Str>(writer, NAME_KEY, name)?) - } - - pub fn name(self, reader: &heed::RoTxn) -> MResult> { - Ok(self - .main - .get::<_, Str, Str>(reader, NAME_KEY)? - .map(|name| name.to_owned())) - } - - pub fn put_created_at(self, writer: &mut heed::RwTxn) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())?) - } - - pub fn created_at(self, reader: &heed::RoTxn) -> MResult>> { - Ok(self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)?) - } - - pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())?) - } - - pub fn updated_at(self, reader: &heed::RoTxn) -> MResult>> { - Ok(self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)?) - } - - pub fn put_internal_docids(self, writer: &mut heed::RwTxn, ids: &sdset::Set) -> MResult<()> { - Ok(self.main.put::<_, Str, DocumentsIds>(writer, INTERNAL_DOCIDS_KEY, ids)?) - } - - pub fn internal_docids<'txn>(self, reader: &'txn heed::RoTxn) -> MResult>> { - match self.main.get::<_, Str, DocumentsIds>(reader, INTERNAL_DOCIDS_KEY)? { - Some(ids) => Ok(ids), - None => Ok(Cow::default()), - } - } - - pub fn merge_internal_docids(self, writer: &mut heed::RwTxn, new_ids: &sdset::Set) -> MResult<()> { - use sdset::SetOperation; - - // We do an union of the old and new internal ids. - let internal_docids = self.internal_docids(writer)?; - let internal_docids = sdset::duo::Union::new(&internal_docids, new_ids).into_set_buf(); - Ok(self.put_internal_docids(writer, &internal_docids)?) - } - - pub fn remove_internal_docids(self, writer: &mut heed::RwTxn, ids: &sdset::Set) -> MResult<()> { - use sdset::SetOperation; - - // We do a difference of the old and new internal ids. - let internal_docids = self.internal_docids(writer)?; - let internal_docids = sdset::duo::Difference::new(&internal_docids, ids).into_set_buf(); - Ok(self.put_internal_docids(writer, &internal_docids)?) - } - - pub fn put_external_docids(self, writer: &mut heed::RwTxn, ids: &fst::Map) -> MResult<()> - where A: AsRef<[u8]>, - { - Ok(self.main.put::<_, Str, ByteSlice>(writer, EXTERNAL_DOCIDS_KEY, ids.as_fst().as_bytes())?) - } - - pub fn merge_external_docids(self, writer: &mut heed::RwTxn, new_docids: &fst::Map) -> MResult<()> - where A: AsRef<[u8]>, - { - use fst::{Streamer, IntoStreamer}; - - // Do an union of the old and the new set of external docids. - let external_docids = self.external_docids(writer)?; - let mut op = external_docids.op().add(new_docids.into_stream()).r#union(); - let mut build = fst::MapBuilder::memory(); - while let Some((docid, values)) = op.next() { - build.insert(docid, values[0].value).unwrap(); - } - drop(op); - - let external_docids = build.into_map(); - Ok(self.put_external_docids(writer, &external_docids)?) - } - - pub fn remove_external_docids(self, writer: &mut heed::RwTxn, ids: &fst::Map) -> MResult<()> - where A: AsRef<[u8]>, - { - use fst::{Streamer, IntoStreamer}; - - // Do an union of the old and the new set of external docids. - let external_docids = self.external_docids(writer)?; - let mut op = external_docids.op().add(ids.into_stream()).difference(); - let mut build = fst::MapBuilder::memory(); - while let Some((docid, values)) = op.next() { - build.insert(docid, values[0].value).unwrap(); - } - drop(op); - - let external_docids = build.into_map(); - self.put_external_docids(writer, &external_docids) - } - - pub fn external_docids<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { - match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? { - Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()), - } - } - - pub fn external_to_internal_docid(self, reader: &heed::RoTxn, external_docid: &str) -> MResult> { - let external_ids = self.external_docids(reader)?; - Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32))) - } - - pub fn words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { - match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? { - Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), - } - } - - pub fn put_words_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { - Ok(self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, fst.as_fst().as_bytes())?) - } - - pub fn put_sorted_document_ids_cache(self, writer: &mut heed::RwTxn, documents_ids: &[DocumentId]) -> MResult<()> { - Ok(self.main.put::<_, Str, CowSlice>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?) - } - - pub fn sorted_document_ids_cache<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult>> { - Ok(self.main.get::<_, Str, CowSlice>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?) - } - - pub fn put_schema(self, writer: &mut heed::RwTxn, schema: &Schema) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeBincode>(writer, SCHEMA_KEY, schema)?) - } - - pub fn schema(self, reader: &heed::RoTxn) -> MResult> { - Ok(self.main.get::<_, Str, SerdeBincode>(reader, SCHEMA_KEY)?) - } - - pub fn delete_schema(self, writer: &mut heed::RwTxn) -> MResult { - Ok(self.main.delete::<_, Str>(writer, SCHEMA_KEY)?) - } - - pub fn put_ranked_map(self, writer: &mut heed::RwTxn, ranked_map: &RankedMap) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeBincode>(writer, RANKED_MAP_KEY, &ranked_map)?) - } - - pub fn ranked_map(self, reader: &heed::RoTxn) -> MResult> { - Ok(self.main.get::<_, Str, SerdeBincode>(reader, RANKED_MAP_KEY)?) - } - - pub fn put_synonyms_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { - let bytes = fst.as_fst().as_bytes(); - Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?) - } - - pub(crate) fn synonyms_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { - match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? { - Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), - } - } - - pub fn synonyms(self, reader: &heed::RoTxn) -> MResult> { - let synonyms = self - .synonyms_fst(&reader)? - .stream() - .into_strs()?; - Ok(synonyms) - } - - pub fn put_stop_words_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { - let bytes = fst.as_fst().as_bytes(); - Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?) - } - - pub(crate) fn stop_words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { - match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? { - Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), - } - } - - pub fn stop_words(self, reader: &heed::RoTxn) -> MResult> { - let stop_word_list = self - .stop_words_fst(reader)? - .stream() - .into_strs()?; - Ok(stop_word_list) - } - - pub fn put_number_of_documents(self, writer: &mut heed::RwTxn, f: F) -> MResult - where - F: Fn(u64) -> u64, - { - let new = self.number_of_documents(&*writer).map(f)?; - self.main - .put::<_, Str, OwnedType>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?; - Ok(new) - } - - pub fn number_of_documents(self, reader: &heed::RoTxn) -> MResult { - match self - .main - .get::<_, Str, OwnedType>(reader, NUMBER_OF_DOCUMENTS_KEY)? { - Some(value) => Ok(value), - None => Ok(0), - } - } - - pub fn put_fields_distribution( - self, - writer: &mut heed::RwTxn, - fields_frequency: &FreqsMap, - ) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeFreqsMap>(writer, FIELDS_DISTRIBUTION_KEY, fields_frequency)?) - } - - pub fn fields_distribution(&self, reader: &heed::RoTxn) -> MResult> { - match self - .main - .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_DISTRIBUTION_KEY)? - { - Some(freqs) => Ok(Some(freqs)), - None => Ok(None), - } - } - - pub fn attributes_for_faceting<'txn>(&self, reader: &'txn heed::RoTxn) -> MResult>>> { - Ok(self.main.get::<_, Str, CowSet>(reader, ATTRIBUTES_FOR_FACETING_KEY)?) - } - - pub fn put_attributes_for_faceting(self, writer: &mut heed::RwTxn, attributes: &Set) -> MResult<()> { - Ok(self.main.put::<_, Str, CowSet>(writer, ATTRIBUTES_FOR_FACETING_KEY, attributes)?) - } - - pub fn delete_attributes_for_faceting(self, writer: &mut heed::RwTxn) -> MResult { - Ok(self.main.delete::<_, Str>(writer, ATTRIBUTES_FOR_FACETING_KEY)?) - } - - pub fn ranking_rules(&self, reader: &heed::RoTxn) -> MResult>> { - Ok(self.main.get::<_, Str, SerdeBincode>>(reader, RANKING_RULES_KEY)?) - } - - pub fn put_ranking_rules(self, writer: &mut heed::RwTxn, value: &[RankingRule]) -> MResult<()> { - Ok(self.main.put::<_, Str, SerdeBincode>>(writer, RANKING_RULES_KEY, &value.to_vec())?) - } - - pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn) -> MResult { - Ok(self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)?) - } - - pub fn distinct_attribute(&self, reader: &heed::RoTxn) -> MResult> { - match self.main.get::<_, Str, OwnedType>(reader, DISTINCT_ATTRIBUTE_KEY)? { - Some(value) => Ok(Some(FieldId(value.to_owned()))), - None => Ok(None), - } - } - - pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn, value: FieldId) -> MResult<()> { - Ok(self.main.put::<_, Str, OwnedType>(writer, DISTINCT_ATTRIBUTE_KEY, &value.0)?) - } - - pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn) -> MResult { - Ok(self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)?) - } - - pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> MResult<()> { - Ok(self.main.put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)?) - } - - pub fn customs<'txn>(self, reader: &'txn heed::RoTxn) -> MResult> { - Ok(self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)?) - } -} diff --git a/meilisearch-core/src/store/mod.rs b/meilisearch-core/src/store/mod.rs deleted file mode 100644 index fa5baa831..000000000 --- a/meilisearch-core/src/store/mod.rs +++ /dev/null @@ -1,522 +0,0 @@ -mod cow_set; -mod docs_words; -mod documents_ids; -mod documents_fields; -mod documents_fields_counts; -mod facets; -mod main; -mod postings_lists; -mod prefix_documents_cache; -mod prefix_postings_lists_cache; -mod synonyms; -mod updates; -mod updates_results; - -pub use self::cow_set::CowSet; -pub use self::docs_words::DocsWords; -pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields}; -pub use self::documents_fields_counts::{DocumentFieldsCountsIter, DocumentsFieldsCounts, DocumentsIdsIter}; -pub use self::documents_ids::{DocumentsIds, DiscoverIds}; -pub use self::facets::Facets; -pub use self::main::Main; -pub use self::postings_lists::PostingsLists; -pub use self::prefix_documents_cache::PrefixDocumentsCache; -pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache; -pub use self::synonyms::Synonyms; -pub use self::updates::Updates; -pub use self::updates_results::UpdatesResults; - -use std::borrow::Cow; -use std::collections::HashSet; -use std::convert::TryInto; -use std::{mem, ptr}; - -use heed::{BytesEncode, BytesDecode}; -use meilisearch_schema::{IndexedPos, FieldId}; -use sdset::{Set, SetBuf}; -use serde::de::{self, Deserialize}; -use zerocopy::{AsBytes, FromBytes}; - -use crate::criterion::Criteria; -use crate::database::{MainT, UpdateT}; -use crate::database::{UpdateEvent, UpdateEventsEmitter}; -use crate::serde::Deserializer; -use crate::settings::SettingsUpdate; -use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult}; - -type BEU32 = zerocopy::U32; -type BEU64 = zerocopy::U64; -pub type BEU16 = zerocopy::U16; - -#[derive(Debug, Copy, Clone, AsBytes, FromBytes)] -#[repr(C)] -pub struct DocumentFieldIndexedKey { - docid: BEU32, - indexed_pos: BEU16, -} - -impl DocumentFieldIndexedKey { - fn new(docid: DocumentId, indexed_pos: IndexedPos) -> DocumentFieldIndexedKey { - DocumentFieldIndexedKey { - docid: BEU32::new(docid.0), - indexed_pos: BEU16::new(indexed_pos.0), - } - } -} - -#[derive(Debug, Copy, Clone, AsBytes, FromBytes)] -#[repr(C)] -pub struct DocumentFieldStoredKey { - docid: BEU32, - field_id: BEU16, -} - -impl DocumentFieldStoredKey { - fn new(docid: DocumentId, field_id: FieldId) -> DocumentFieldStoredKey { - DocumentFieldStoredKey { - docid: BEU32::new(docid.0), - field_id: BEU16::new(field_id.0), - } - } -} - -#[derive(Default, Debug)] -pub struct Postings<'a> { - pub docids: Cow<'a, Set>, - pub matches: Cow<'a, Set>, -} - -pub struct PostingsCodec; - -impl<'a> BytesEncode<'a> for PostingsCodec { - type EItem = Postings<'a>; - - fn bytes_encode(item: &'a Self::EItem) -> Option> { - let u64_size = mem::size_of::(); - let docids_size = item.docids.len() * mem::size_of::(); - let matches_size = item.matches.len() * mem::size_of::(); - - let mut buffer = Vec::with_capacity(u64_size + docids_size + matches_size); - - let docids_len = item.docids.len() as u64; - buffer.extend_from_slice(&docids_len.to_be_bytes()); - buffer.extend_from_slice(item.docids.as_bytes()); - buffer.extend_from_slice(item.matches.as_bytes()); - - Some(Cow::Owned(buffer)) - } -} - -fn aligned_to(bytes: &[u8], align: usize) -> bool { - (bytes as *const _ as *const () as usize) % align == 0 -} - -fn from_bytes_to_set<'a, T: 'a>(bytes: &'a [u8]) -> Option>> -where T: Clone + FromBytes -{ - match zerocopy::LayoutVerified::<_, [T]>::new_slice(bytes) { - Some(layout) => Some(Cow::Borrowed(Set::new_unchecked(layout.into_slice()))), - None => { - let len = bytes.len(); - let elem_size = mem::size_of::(); - - // ensure that it is the alignment that is wrong - // and the length is valid - if len % elem_size == 0 && !aligned_to(bytes, mem::align_of::()) { - let elems = len / elem_size; - let mut vec = Vec::::with_capacity(elems); - - unsafe { - let dst = vec.as_mut_ptr() as *mut u8; - ptr::copy_nonoverlapping(bytes.as_ptr(), dst, len); - vec.set_len(elems); - } - - return Some(Cow::Owned(SetBuf::new_unchecked(vec))); - } - - None - } - } -} - -impl<'a> BytesDecode<'a> for PostingsCodec { - type DItem = Postings<'a>; - - fn bytes_decode(bytes: &'a [u8]) -> Option { - let u64_size = mem::size_of::(); - let docid_size = mem::size_of::(); - - let (len_bytes, bytes) = bytes.split_at(u64_size); - let docids_len = len_bytes.try_into().ok().map(u64::from_be_bytes)? as usize; - let docids_size = docids_len * docid_size; - - let docids_bytes = &bytes[..docids_size]; - let matches_bytes = &bytes[docids_size..]; - - let docids = from_bytes_to_set(docids_bytes)?; - let matches = from_bytes_to_set(matches_bytes)?; - - Some(Postings { docids, matches }) - } -} - -fn main_name(name: &str) -> String { - format!("store-{}", name) -} - -fn postings_lists_name(name: &str) -> String { - format!("store-{}-postings-lists", name) -} - -fn documents_fields_name(name: &str) -> String { - format!("store-{}-documents-fields", name) -} - -fn documents_fields_counts_name(name: &str) -> String { - format!("store-{}-documents-fields-counts", name) -} - -fn synonyms_name(name: &str) -> String { - format!("store-{}-synonyms", name) -} - -fn docs_words_name(name: &str) -> String { - format!("store-{}-docs-words", name) -} - -fn prefix_documents_cache_name(name: &str) -> String { - format!("store-{}-prefix-documents-cache", name) -} - -fn prefix_postings_lists_cache_name(name: &str) -> String { - format!("store-{}-prefix-postings-lists-cache", name) -} - -fn updates_name(name: &str) -> String { - format!("store-{}-updates", name) -} - -fn updates_results_name(name: &str) -> String { - format!("store-{}-updates-results", name) -} - -fn facets_name(name: &str) -> String { - format!("store-{}-facets", name) -} - -#[derive(Clone)] -pub struct Index { - pub main: Main, - pub postings_lists: PostingsLists, - pub documents_fields: DocumentsFields, - pub documents_fields_counts: DocumentsFieldsCounts, - pub facets: Facets, - pub synonyms: Synonyms, - pub docs_words: DocsWords, - pub prefix_documents_cache: PrefixDocumentsCache, - pub prefix_postings_lists_cache: PrefixPostingsListsCache, - - pub updates: Updates, - pub updates_results: UpdatesResults, - pub(crate) updates_notifier: UpdateEventsEmitter, -} - -impl Index { - pub fn document( - &self, - reader: &heed::RoTxn, - attributes: Option<&HashSet<&str>>, - document_id: DocumentId, - ) -> MResult> { - let schema = self.main.schema(reader)?; - let schema = schema.ok_or(Error::SchemaMissing)?; - - let attributes = match attributes { - Some(attributes) => Some(attributes.iter().filter_map(|name| schema.id(*name)).collect()), - None => None, - }; - - let mut deserializer = Deserializer { - document_id, - reader, - documents_fields: self.documents_fields, - schema: &schema, - fields: attributes.as_ref(), - }; - - Ok(Option::::deserialize(&mut deserializer)?) - } - - pub fn document_attribute( - &self, - reader: &heed::RoTxn, - document_id: DocumentId, - attribute: FieldId, - ) -> MResult> { - let bytes = self - .documents_fields - .document_attribute(reader, document_id, attribute)?; - match bytes { - Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)), - None => Ok(None), - } - } - - pub fn document_attribute_bytes<'txn>( - &self, - reader: &'txn heed::RoTxn, - document_id: DocumentId, - attribute: FieldId, - ) -> MResult> { - let bytes = self - .documents_fields - .document_attribute(reader, document_id, attribute)?; - match bytes { - Some(bytes) => Ok(Some(bytes)), - None => Ok(None), - } - } - - pub fn customs_update(&self, writer: &mut heed::RwTxn, customs: Vec) -> MResult { - let _ = self.updates_notifier.send(UpdateEvent::NewUpdate); - Ok(update::push_customs_update(writer, self.updates, self.updates_results, customs)?) - } - - pub fn settings_update(&self, writer: &mut heed::RwTxn, update: SettingsUpdate) -> MResult { - let _ = self.updates_notifier.send(UpdateEvent::NewUpdate); - Ok(update::push_settings_update(writer, self.updates, self.updates_results, update)?) - } - - pub fn documents_addition(&self) -> update::DocumentsAddition { - update::DocumentsAddition::new( - self.updates, - self.updates_results, - self.updates_notifier.clone(), - ) - } - - pub fn documents_partial_addition(&self) -> update::DocumentsAddition { - update::DocumentsAddition::new_partial( - self.updates, - self.updates_results, - self.updates_notifier.clone(), - ) - } - - pub fn documents_deletion(&self) -> update::DocumentsDeletion { - update::DocumentsDeletion::new( - self.updates, - self.updates_results, - self.updates_notifier.clone(), - ) - } - - pub fn clear_all(&self, writer: &mut heed::RwTxn) -> MResult { - let _ = self.updates_notifier.send(UpdateEvent::NewUpdate); - update::push_clear_all(writer, self.updates, self.updates_results) - } - - pub fn current_update_id(&self, reader: &heed::RoTxn) -> MResult> { - match self.updates.last_update(reader)? { - Some((id, _)) => Ok(Some(id)), - None => Ok(None), - } - } - - pub fn update_status( - &self, - reader: &heed::RoTxn, - update_id: u64, - ) -> MResult> { - update::update_status(reader, self.updates, self.updates_results, update_id) - } - - pub fn all_updates_status(&self, reader: &heed::RoTxn) -> MResult> { - let mut updates = Vec::new(); - let mut last_update_result_id = 0; - - // retrieve all updates results - if let Some((last_id, _)) = self.updates_results.last_update(reader)? { - updates.reserve(last_id as usize); - - for id in 0..=last_id { - if let Some(update) = self.update_status(reader, id)? { - updates.push(update); - last_update_result_id = id + 1; - } - } - } - - // retrieve all enqueued updates - if let Some((last_id, _)) = self.updates.last_update(reader)? { - for id in last_update_result_id..=last_id { - if let Some(update) = self.update_status(reader, id)? { - updates.push(update); - } - } - } - - Ok(updates) - } - - pub fn query_builder(&self) -> QueryBuilder { - QueryBuilder::new(self) - } - - pub fn query_builder_with_criteria<'c, 'f, 'd, 'i>( - &'i self, - criteria: Criteria<'c>, - ) -> QueryBuilder<'c, 'f, 'd, 'i> { - QueryBuilder::with_criteria(self, criteria) - } -} - -pub fn create( - env: &heed::Env, - update_env: &heed::Env, - name: &str, - updates_notifier: UpdateEventsEmitter, -) -> MResult { - // create all the store names - let main_name = main_name(name); - let postings_lists_name = postings_lists_name(name); - let documents_fields_name = documents_fields_name(name); - let documents_fields_counts_name = documents_fields_counts_name(name); - let synonyms_name = synonyms_name(name); - let docs_words_name = docs_words_name(name); - let prefix_documents_cache_name = prefix_documents_cache_name(name); - let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name); - let updates_name = updates_name(name); - let updates_results_name = updates_results_name(name); - let facets_name = facets_name(name); - - // open all the stores - let main = env.create_poly_database(Some(&main_name))?; - let postings_lists = env.create_database(Some(&postings_lists_name))?; - let documents_fields = env.create_database(Some(&documents_fields_name))?; - let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?; - let facets = env.create_database(Some(&facets_name))?; - let synonyms = env.create_database(Some(&synonyms_name))?; - let docs_words = env.create_database(Some(&docs_words_name))?; - let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?; - let prefix_postings_lists_cache = env.create_database(Some(&prefix_postings_lists_cache_name))?; - let updates = update_env.create_database(Some(&updates_name))?; - let updates_results = update_env.create_database(Some(&updates_results_name))?; - - Ok(Index { - main: Main { main }, - postings_lists: PostingsLists { postings_lists }, - documents_fields: DocumentsFields { documents_fields }, - documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts }, - synonyms: Synonyms { synonyms }, - docs_words: DocsWords { docs_words }, - prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache }, - prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache }, - facets: Facets { facets }, - - updates: Updates { updates }, - updates_results: UpdatesResults { updates_results }, - updates_notifier, - }) -} - -pub fn open( - env: &heed::Env, - update_env: &heed::Env, - name: &str, - updates_notifier: UpdateEventsEmitter, -) -> MResult> { - // create all the store names - let main_name = main_name(name); - let postings_lists_name = postings_lists_name(name); - let documents_fields_name = documents_fields_name(name); - let documents_fields_counts_name = documents_fields_counts_name(name); - let synonyms_name = synonyms_name(name); - let docs_words_name = docs_words_name(name); - let prefix_documents_cache_name = prefix_documents_cache_name(name); - let facets_name = facets_name(name); - let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name); - let updates_name = updates_name(name); - let updates_results_name = updates_results_name(name); - - // open all the stores - let main = match env.open_poly_database(Some(&main_name))? { - Some(main) => main, - None => return Ok(None), - }; - let postings_lists = match env.open_database(Some(&postings_lists_name))? { - Some(postings_lists) => postings_lists, - None => return Ok(None), - }; - let documents_fields = match env.open_database(Some(&documents_fields_name))? { - Some(documents_fields) => documents_fields, - None => return Ok(None), - }; - let documents_fields_counts = match env.open_database(Some(&documents_fields_counts_name))? { - Some(documents_fields_counts) => documents_fields_counts, - None => return Ok(None), - }; - let synonyms = match env.open_database(Some(&synonyms_name))? { - Some(synonyms) => synonyms, - None => return Ok(None), - }; - let docs_words = match env.open_database(Some(&docs_words_name))? { - Some(docs_words) => docs_words, - None => return Ok(None), - }; - let prefix_documents_cache = match env.open_database(Some(&prefix_documents_cache_name))? { - Some(prefix_documents_cache) => prefix_documents_cache, - None => return Ok(None), - }; - let facets = match env.open_database(Some(&facets_name))? { - Some(facets) => facets, - None => return Ok(None), - }; - let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? { - Some(prefix_postings_lists_cache) => prefix_postings_lists_cache, - None => return Ok(None), - }; - let updates = match update_env.open_database(Some(&updates_name))? { - Some(updates) => updates, - None => return Ok(None), - }; - let updates_results = match update_env.open_database(Some(&updates_results_name))? { - Some(updates_results) => updates_results, - None => return Ok(None), - }; - - Ok(Some(Index { - main: Main { main }, - postings_lists: PostingsLists { postings_lists }, - documents_fields: DocumentsFields { documents_fields }, - documents_fields_counts: DocumentsFieldsCounts { documents_fields_counts }, - synonyms: Synonyms { synonyms }, - docs_words: DocsWords { docs_words }, - prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache }, - facets: Facets { facets }, - prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache }, - updates: Updates { updates }, - updates_results: UpdatesResults { updates_results }, - updates_notifier, - })) -} - -pub fn clear( - writer: &mut heed::RwTxn, - update_writer: &mut heed::RwTxn, - index: &Index, -) -> MResult<()> { - // clear all the stores - index.main.clear(writer)?; - index.postings_lists.clear(writer)?; - index.documents_fields.clear(writer)?; - index.documents_fields_counts.clear(writer)?; - index.synonyms.clear(writer)?; - index.docs_words.clear(writer)?; - index.prefix_documents_cache.clear(writer)?; - index.prefix_postings_lists_cache.clear(writer)?; - index.updates.clear(update_writer)?; - index.updates_results.clear(update_writer)?; - Ok(()) -} diff --git a/meilisearch-core/src/store/postings_lists.rs b/meilisearch-core/src/store/postings_lists.rs deleted file mode 100644 index 3cf1a6a1f..000000000 --- a/meilisearch-core/src/store/postings_lists.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::borrow::Cow; - -use heed::Result as ZResult; -use heed::types::ByteSlice; -use sdset::{Set, SetBuf}; -use slice_group_by::GroupBy; - -use crate::database::MainT; -use crate::DocIndex; -use crate::store::{Postings, PostingsCodec}; - -#[derive(Copy, Clone)] -pub struct PostingsLists { - pub(crate) postings_lists: heed::Database, -} - -impl PostingsLists { - pub fn put_postings_list( - self, - writer: &mut heed::RwTxn, - word: &[u8], - matches: &Set, - ) -> ZResult<()> { - let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect(); - let docids = Cow::Owned(SetBuf::new_unchecked(docids)); - let matches = Cow::Borrowed(matches); - let postings = Postings { docids, matches }; - - self.postings_lists.put(writer, word, &postings) - } - - pub fn del_postings_list(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult { - self.postings_lists.delete(writer, word) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.postings_lists.clear(writer) - } - - pub fn postings_list<'txn>( - self, - reader: &'txn heed::RoTxn, - word: &[u8], - ) -> ZResult>> { - self.postings_lists.get(reader, word) - } -} diff --git a/meilisearch-core/src/store/prefix_documents_cache.rs b/meilisearch-core/src/store/prefix_documents_cache.rs deleted file mode 100644 index 2bb8700dc..000000000 --- a/meilisearch-core/src/store/prefix_documents_cache.rs +++ /dev/null @@ -1,80 +0,0 @@ -use std::borrow::Cow; - -use heed::types::{OwnedType, CowSlice}; -use heed::Result as ZResult; -use zerocopy::{AsBytes, FromBytes}; - -use super::{BEU64, BEU32}; -use crate::{DocumentId, Highlight}; -use crate::database::MainT; - -#[derive(Debug, Copy, Clone, AsBytes, FromBytes)] -#[repr(C)] -pub struct PrefixKey { - prefix: [u8; 4], - index: BEU64, - docid: BEU32, -} - -impl PrefixKey { - pub fn new(prefix: [u8; 4], index: u64, docid: u32) -> PrefixKey { - PrefixKey { - prefix, - index: BEU64::new(index), - docid: BEU32::new(docid), - } - } -} - -#[derive(Copy, Clone)] -pub struct PrefixDocumentsCache { - pub(crate) prefix_documents_cache: heed::Database, CowSlice>, -} - -impl PrefixDocumentsCache { - pub fn put_prefix_document( - self, - writer: &mut heed::RwTxn, - prefix: [u8; 4], - index: usize, - docid: DocumentId, - highlights: &[Highlight], - ) -> ZResult<()> { - let key = PrefixKey::new(prefix, index as u64, docid.0); - self.prefix_documents_cache.put(writer, &key, highlights) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.prefix_documents_cache.clear(writer) - } - - pub fn prefix_documents<'txn>( - self, - reader: &'txn heed::RoTxn, - prefix: [u8; 4], - ) -> ZResult> { - let start = PrefixKey::new(prefix, 0, 0); - let end = PrefixKey::new(prefix, u64::max_value(), u32::max_value()); - let iter = self.prefix_documents_cache.range(reader, &(start..=end))?; - Ok(PrefixDocumentsIter { iter }) - } -} - -pub struct PrefixDocumentsIter<'txn> { - iter: heed::RoRange<'txn, OwnedType, CowSlice>, -} - -impl<'txn> Iterator for PrefixDocumentsIter<'txn> { - type Item = ZResult<(DocumentId, Cow<'txn, [Highlight]>)>; - - fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok((key, highlights))) => { - let docid = DocumentId(key.docid.get()); - Some(Ok((docid, highlights))) - } - Some(Err(e)) => Some(Err(e)), - None => None, - } - } -} diff --git a/meilisearch-core/src/store/prefix_postings_lists_cache.rs b/meilisearch-core/src/store/prefix_postings_lists_cache.rs deleted file mode 100644 index bc0c58f52..000000000 --- a/meilisearch-core/src/store/prefix_postings_lists_cache.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::borrow::Cow; - -use heed::Result as ZResult; -use heed::types::OwnedType; -use sdset::{Set, SetBuf}; -use slice_group_by::GroupBy; - -use crate::database::MainT; -use crate::DocIndex; -use crate::store::{PostingsCodec, Postings}; - -#[derive(Copy, Clone)] -pub struct PrefixPostingsListsCache { - pub(crate) prefix_postings_lists_cache: heed::Database, PostingsCodec>, -} - -impl PrefixPostingsListsCache { - pub fn put_prefix_postings_list( - self, - writer: &mut heed::RwTxn, - prefix: [u8; 4], - matches: &Set, - ) -> ZResult<()> - { - let docids = matches.linear_group_by_key(|m| m.document_id).map(|g| g[0].document_id).collect(); - let docids = Cow::Owned(SetBuf::new_unchecked(docids)); - let matches = Cow::Borrowed(matches); - let postings = Postings { docids, matches }; - - self.prefix_postings_lists_cache.put(writer, &prefix, &postings) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.prefix_postings_lists_cache.clear(writer) - } - - pub fn prefix_postings_list<'txn>( - self, - reader: &'txn heed::RoTxn, - prefix: [u8; 4], - ) -> ZResult>> - { - self.prefix_postings_lists_cache.get(reader, &prefix) - } -} diff --git a/meilisearch-core/src/store/synonyms.rs b/meilisearch-core/src/store/synonyms.rs deleted file mode 100644 index bf7472f96..000000000 --- a/meilisearch-core/src/store/synonyms.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::borrow::Cow; - -use heed::Result as ZResult; -use heed::types::ByteSlice; - -use crate::database::MainT; -use crate::{FstSetCow, MResult}; - -#[derive(Copy, Clone)] -pub struct Synonyms { - pub(crate) synonyms: heed::Database, -} - -impl Synonyms { - pub fn put_synonyms(self, writer: &mut heed::RwTxn, word: &[u8], synonyms: &fst::Set) -> ZResult<()> - where A: AsRef<[u8]>, - { - let bytes = synonyms.as_fst().as_bytes(); - self.synonyms.put(writer, word, bytes) - } - - pub fn del_synonyms(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult { - self.synonyms.delete(writer, word) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.synonyms.clear(writer) - } - - pub(crate) fn synonyms_fst<'txn>(self, reader: &'txn heed::RoTxn, word: &[u8]) -> ZResult> { - match self.synonyms.get(reader, word)? { - Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), - None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), - } - } - - pub fn synonyms(self, reader: &heed::RoTxn, word: &[u8]) -> MResult> { - let synonyms = self - .synonyms_fst(&reader, word)? - .stream() - .into_strs()?; - Ok(synonyms) - } -} diff --git a/meilisearch-core/src/store/updates.rs b/meilisearch-core/src/store/updates.rs deleted file mode 100644 index a614303a3..000000000 --- a/meilisearch-core/src/store/updates.rs +++ /dev/null @@ -1,65 +0,0 @@ -use super::BEU64; -use crate::database::UpdateT; -use crate::update::Update; -use heed::types::{OwnedType, SerdeJson}; -use heed::Result as ZResult; - -#[derive(Copy, Clone)] -pub struct Updates { - pub(crate) updates: heed::Database, SerdeJson>, -} - -impl Updates { - // TODO do not trigger deserialize if possible - pub fn last_update(self, reader: &heed::RoTxn) -> ZResult> { - match self.updates.last(reader)? { - Some((key, data)) => Ok(Some((key.get(), data))), - None => Ok(None), - } - } - - // TODO do not trigger deserialize if possible - pub fn first_update(self, reader: &heed::RoTxn) -> ZResult> { - match self.updates.first(reader)? { - Some((key, data)) => Ok(Some((key.get(), data))), - None => Ok(None), - } - } - - // TODO do not trigger deserialize if possible - pub fn get(self, reader: &heed::RoTxn, update_id: u64) -> ZResult> { - let update_id = BEU64::new(update_id); - self.updates.get(reader, &update_id) - } - - pub fn put_update( - self, - writer: &mut heed::RwTxn, - update_id: u64, - update: &Update, - ) -> ZResult<()> { - // TODO prefer using serde_json? - let update_id = BEU64::new(update_id); - self.updates.put(writer, &update_id, update) - } - - pub fn del_update(self, writer: &mut heed::RwTxn, update_id: u64) -> ZResult { - let update_id = BEU64::new(update_id); - self.updates.delete(writer, &update_id) - } - - pub fn pop_front(self, writer: &mut heed::RwTxn) -> ZResult> { - match self.first_update(writer)? { - Some((update_id, update)) => { - let key = BEU64::new(update_id); - self.updates.delete(writer, &key)?; - Ok(Some((update_id, update))) - } - None => Ok(None), - } - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.updates.clear(writer) - } -} diff --git a/meilisearch-core/src/store/updates_results.rs b/meilisearch-core/src/store/updates_results.rs deleted file mode 100644 index ca631e316..000000000 --- a/meilisearch-core/src/store/updates_results.rs +++ /dev/null @@ -1,45 +0,0 @@ -use super::BEU64; -use crate::database::UpdateT; -use crate::update::ProcessedUpdateResult; -use heed::types::{OwnedType, SerdeJson}; -use heed::Result as ZResult; - -#[derive(Copy, Clone)] -pub struct UpdatesResults { - pub(crate) updates_results: heed::Database, SerdeJson>, -} - -impl UpdatesResults { - pub fn last_update( - self, - reader: &heed::RoTxn, - ) -> ZResult> { - match self.updates_results.last(reader)? { - Some((key, data)) => Ok(Some((key.get(), data))), - None => Ok(None), - } - } - - pub fn put_update_result( - self, - writer: &mut heed::RwTxn, - update_id: u64, - update_result: &ProcessedUpdateResult, - ) -> ZResult<()> { - let update_id = BEU64::new(update_id); - self.updates_results.put(writer, &update_id, update_result) - } - - pub fn update_result( - self, - reader: &heed::RoTxn, - update_id: u64, - ) -> ZResult> { - let update_id = BEU64::new(update_id); - self.updates_results.get(reader, &update_id) - } - - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { - self.updates_results.clear(writer) - } -} diff --git a/meilisearch-core/src/update/clear_all.rs b/meilisearch-core/src/update/clear_all.rs deleted file mode 100644 index 434e8a245..000000000 --- a/meilisearch-core/src/update/clear_all.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::database::{MainT, UpdateT}; -use crate::update::{next_update_id, Update}; -use crate::{store, MResult, RankedMap}; - -pub fn apply_clear_all( - writer: &mut heed::RwTxn, - index: &store::Index, -) -> MResult<()> { - index.main.put_words_fst(writer, &fst::Set::default())?; - index.main.put_external_docids(writer, &fst::Map::default())?; - index.main.put_internal_docids(writer, &sdset::SetBuf::default())?; - index.main.put_ranked_map(writer, &RankedMap::default())?; - index.main.put_number_of_documents(writer, |_| 0)?; - index.main.put_sorted_document_ids_cache(writer, &[])?; - index.documents_fields.clear(writer)?; - index.documents_fields_counts.clear(writer)?; - index.postings_lists.clear(writer)?; - index.docs_words.clear(writer)?; - index.prefix_documents_cache.clear(writer)?; - index.prefix_postings_lists_cache.clear(writer)?; - index.facets.clear(writer)?; - - Ok(()) -} - -pub fn push_clear_all( - writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, -) -> MResult { - let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; - let update = Update::clear_all(); - updates_store.put_update(writer, last_update_id, &update)?; - - Ok(last_update_id) -} diff --git a/meilisearch-core/src/update/customs_update.rs b/meilisearch-core/src/update/customs_update.rs deleted file mode 100644 index a3a66e61d..000000000 --- a/meilisearch-core/src/update/customs_update.rs +++ /dev/null @@ -1,26 +0,0 @@ - -use crate::database::{MainT, UpdateT}; -use crate::{store, MResult}; -use crate::update::{next_update_id, Update}; - -pub fn apply_customs_update( - writer: &mut heed::RwTxn, - main_store: store::Main, - customs: &[u8], -) -> MResult<()> { - main_store.put_customs(writer, customs) -} - -pub fn push_customs_update( - writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - customs: Vec, -) -> MResult { - let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; - - let update = Update::customs(customs); - updates_store.put_update(writer, last_update_id, &update)?; - - Ok(last_update_id) -} diff --git a/meilisearch-core/src/update/documents_addition.rs b/meilisearch-core/src/update/documents_addition.rs deleted file mode 100644 index 26bbd94b2..000000000 --- a/meilisearch-core/src/update/documents_addition.rs +++ /dev/null @@ -1,444 +0,0 @@ -use std::borrow::Cow; -use std::collections::{HashMap, BTreeMap}; - -use fst::{set::OpBuilder, SetBuilder}; -use indexmap::IndexMap; -use meilisearch_schema::{Schema, FieldId}; -use meilisearch_types::DocumentId; -use sdset::{duo::Union, SetOperation}; -use serde::Deserialize; -use serde_json::Value; - -use crate::database::{MainT, UpdateT}; -use crate::database::{UpdateEvent, UpdateEventsEmitter}; -use crate::facets; -use crate::raw_indexer::RawIndexer; -use crate::serde::Deserializer; -use crate::store::{self, DocumentsFields, DocumentsFieldsCounts, DiscoverIds}; -use crate::update::helpers::{index_value, value_to_number, extract_document_id}; -use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update}; -use crate::{Error, MResult, RankedMap}; - -pub struct DocumentsAddition { - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - updates_notifier: UpdateEventsEmitter, - // Whether the user explicitly set the primary key in the update - primary_key: Option, - documents: Vec, - is_partial: bool, -} - -impl DocumentsAddition { - pub fn new( - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - updates_notifier: UpdateEventsEmitter, - ) -> DocumentsAddition { - DocumentsAddition { - updates_store, - updates_results_store, - updates_notifier, - documents: Vec::new(), - is_partial: false, - primary_key: None, - } - } - - pub fn new_partial( - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - updates_notifier: UpdateEventsEmitter, - ) -> DocumentsAddition { - DocumentsAddition { - updates_store, - updates_results_store, - updates_notifier, - documents: Vec::new(), - is_partial: true, - primary_key: None, - } - } - - pub fn set_primary_key(&mut self, primary_key: String) { - self.primary_key = Some(primary_key); - } - - pub fn update_document(&mut self, document: D) { - self.documents.push(document); - } - - pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult - where - D: serde::Serialize, - { - let _ = self.updates_notifier.send(UpdateEvent::NewUpdate); - let update_id = push_documents_addition( - writer, - self.updates_store, - self.updates_results_store, - self.documents, - self.is_partial, - self.primary_key, - )?; - Ok(update_id) - } -} - -impl Extend for DocumentsAddition { - fn extend>(&mut self, iter: T) { - self.documents.extend(iter) - } -} - -pub fn push_documents_addition( - writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - addition: Vec, - is_partial: bool, - primary_key: Option, -) -> MResult { - let mut values = Vec::with_capacity(addition.len()); - for add in addition { - let vec = serde_json::to_vec(&add)?; - let add = serde_json::from_slice(&vec)?; - values.push(add); - } - - let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; - - let update = if is_partial { - Update::documents_partial(primary_key, values) - } else { - Update::documents_addition(primary_key, values) - }; - - updates_store.put_update(writer, last_update_id, &update)?; - - Ok(last_update_id) -} - -#[allow(clippy::too_many_arguments)] -fn index_document>( - writer: &mut heed::RwTxn, - documents_fields: DocumentsFields, - documents_fields_counts: DocumentsFieldsCounts, - ranked_map: &mut RankedMap, - indexer: &mut RawIndexer, - schema: &Schema, - field_id: FieldId, - document_id: DocumentId, - value: &Value, -) -> MResult<()> -{ - let serialized = serde_json::to_vec(value)?; - documents_fields.put_document_field(writer, document_id, field_id, &serialized)?; - - if let Some(indexed_pos) = schema.is_searchable(field_id) { - let number_of_words = index_value(indexer, document_id, indexed_pos, value); - if let Some(number_of_words) = number_of_words { - documents_fields_counts.put_document_field_count( - writer, - document_id, - indexed_pos, - number_of_words as u16, - )?; - } - } - - if schema.is_ranked(field_id) { - let number = value_to_number(value).unwrap_or_default(); - ranked_map.insert(document_id, field_id, number); - } - - Ok(()) -} - -pub fn apply_addition( - writer: &mut heed::RwTxn, - index: &store::Index, - new_documents: Vec>, - partial: bool, - primary_key: Option, -) -> MResult<()> -{ - let mut schema = match index.main.schema(writer)? { - Some(schema) => schema, - None => return Err(Error::SchemaMissing), - }; - - // Retrieve the documents ids related structures - let external_docids = index.main.external_docids(writer)?; - let internal_docids = index.main.internal_docids(writer)?; - let mut available_ids = DiscoverIds::new(&internal_docids); - - let primary_key = match schema.primary_key() { - Some(primary_key) => primary_key.to_string(), - None => { - let name = primary_key.ok_or(Error::MissingPrimaryKey)?; - schema.set_primary_key(&name)?; - name - } - }; - - // 1. store documents ids for future deletion - let mut documents_additions = HashMap::new(); - let mut new_external_docids = BTreeMap::new(); - let mut new_internal_docids = Vec::with_capacity(new_documents.len()); - - for mut document in new_documents { - let external_docids_get = |docid: &str| { - match (external_docids.get(docid), new_external_docids.get(docid)) { - (_, Some(&id)) - | (Some(id), _) => Some(id as u32), - (None, None) => None, - } - }; - - let (internal_docid, external_docid) = - extract_document_id( - &primary_key, - &document, - &external_docids_get, - &mut available_ids, - )?; - - new_external_docids.insert(external_docid, internal_docid.0 as u64); - new_internal_docids.push(internal_docid); - - if partial { - let mut deserializer = Deserializer { - document_id: internal_docid, - reader: writer, - documents_fields: index.documents_fields, - schema: &schema, - fields: None, - }; - - let old_document = Option::>::deserialize(&mut deserializer)?; - if let Some(old_document) = old_document { - for (key, value) in old_document { - document.entry(key).or_insert(value); - } - } - } - documents_additions.insert(internal_docid, document); - } - - // 2. remove the documents postings lists - let number_of_inserted_documents = documents_additions.len(); - let documents_ids = new_external_docids.iter().map(|(id, _)| id.clone()).collect(); - apply_documents_deletion(writer, index, documents_ids)?; - - let mut ranked_map = match index.main.ranked_map(writer)? { - Some(ranked_map) => ranked_map, - None => RankedMap::default(), - }; - - let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?; - - - let mut indexer = RawIndexer::new(&stop_words); - - // For each document in this update - for (document_id, document) in &documents_additions { - // For each key-value pair in the document. - for (attribute, value) in document { - let (field_id, _) = schema.insert_with_position(&attribute)?; - index_document( - writer, - index.documents_fields, - index.documents_fields_counts, - &mut ranked_map, - &mut indexer, - &schema, - field_id, - *document_id, - &value, - )?; - } - } - - write_documents_addition_index( - writer, - index, - &ranked_map, - number_of_inserted_documents, - indexer, - )?; - - index.main.put_schema(writer, &schema)?; - - let new_external_docids = fst::Map::from_iter(new_external_docids.iter().map(|(ext, id)| (ext, *id as u64)))?; - let new_internal_docids = sdset::SetBuf::from_dirty(new_internal_docids); - index.main.merge_external_docids(writer, &new_external_docids)?; - index.main.merge_internal_docids(writer, &new_internal_docids)?; - - // recompute all facet attributes after document update. - if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? { - let docids = index.main.internal_docids(writer)?; - let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?; - index.facets.add(writer, facet_map)?; - } - - // update is finished; update sorted document id cache with new state - let mut document_ids = index.main.internal_docids(writer)?.to_vec(); - super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?; - - Ok(()) -} - -pub fn apply_documents_partial_addition( - writer: &mut heed::RwTxn, - index: &store::Index, - new_documents: Vec>, - primary_key: Option, -) -> MResult<()> { - apply_addition(writer, index, new_documents, true, primary_key) -} - -pub fn apply_documents_addition( - writer: &mut heed::RwTxn, - index: &store::Index, - new_documents: Vec>, - primary_key: Option, -) -> MResult<()> { - apply_addition(writer, index, new_documents, false, primary_key) -} - -pub fn reindex_all_documents(writer: &mut heed::RwTxn, index: &store::Index) -> MResult<()> { - let schema = match index.main.schema(writer)? { - Some(schema) => schema, - None => return Err(Error::SchemaMissing), - }; - - let mut ranked_map = RankedMap::default(); - - // 1. retrieve all documents ids - let mut documents_ids_to_reindex = Vec::new(); - for result in index.documents_fields_counts.documents_ids(writer)? { - let document_id = result?; - documents_ids_to_reindex.push(document_id); - } - - // 2. remove the documents posting lists - index.main.put_words_fst(writer, &fst::Set::default())?; - index.main.put_ranked_map(writer, &ranked_map)?; - index.main.put_number_of_documents(writer, |_| 0)?; - index.facets.clear(writer)?; - index.postings_lists.clear(writer)?; - index.docs_words.clear(writer)?; - - let stop_words = index.main - .stop_words_fst(writer)? - .map_data(Cow::into_owned) - .unwrap(); - - let number_of_inserted_documents = documents_ids_to_reindex.len(); - let mut indexer = RawIndexer::new(&stop_words); - let mut ram_store = HashMap::new(); - - if let Some(ref attributes_for_facetting) = index.main.attributes_for_faceting(writer)? { - let facet_map = facets::facet_map_from_docids(writer, &index, &documents_ids_to_reindex, &attributes_for_facetting)?; - index.facets.add(writer, facet_map)?; - } - // ^-- https://github.com/meilisearch/MeiliSearch/pull/631#issuecomment-626624470 --v - for document_id in &documents_ids_to_reindex { - for result in index.documents_fields.document_fields(writer, *document_id)? { - let (field_id, bytes) = result?; - let value: Value = serde_json::from_slice(bytes)?; - ram_store.insert((document_id, field_id), value); - } - - // For each key-value pair in the document. - for ((document_id, field_id), value) in ram_store.drain() { - index_document( - writer, - index.documents_fields, - index.documents_fields_counts, - &mut ranked_map, - &mut indexer, - &schema, - field_id, - *document_id, - &value, - )?; - } - } - - // 4. write the new index in the main store - write_documents_addition_index( - writer, - index, - &ranked_map, - number_of_inserted_documents, - indexer, - )?; - - index.main.put_schema(writer, &schema)?; - - // recompute all facet attributes after document update. - if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? { - let docids = index.main.internal_docids(writer)?; - let facet_map = facets::facet_map_from_docids(writer, index, &docids, attributes_for_facetting.as_ref())?; - index.facets.add(writer, facet_map)?; - } - - // update is finished; update sorted document id cache with new state - let mut document_ids = index.main.internal_docids(writer)?.to_vec(); - super::cache_document_ids_sorted(writer, &ranked_map, index, &mut document_ids)?; - - Ok(()) -} - -pub fn write_documents_addition_index>( - writer: &mut heed::RwTxn, - index: &store::Index, - ranked_map: &RankedMap, - number_of_inserted_documents: usize, - indexer: RawIndexer, -) -> MResult<()> -{ - let indexed = indexer.build(); - let mut delta_words_builder = SetBuilder::memory(); - - for (word, delta_set) in indexed.words_doc_indexes { - delta_words_builder.insert(&word).unwrap(); - - let set = match index.postings_lists.postings_list(writer, &word)? { - Some(postings) => Union::new(&postings.matches, &delta_set).into_set_buf(), - None => delta_set, - }; - - index.postings_lists.put_postings_list(writer, &word, &set)?; - } - - for (id, words) in indexed.docs_words { - index.docs_words.put_doc_words(writer, id, &words)?; - } - - let delta_words = delta_words_builder.into_set(); - - let words_fst = index.main.words_fst(writer)?; - let words = if !words_fst.is_empty() { - let op = OpBuilder::new() - .add(words_fst.stream()) - .add(delta_words.stream()) - .r#union(); - - let mut words_builder = SetBuilder::memory(); - words_builder.extend_stream(op).unwrap(); - words_builder.into_set() - } else { - delta_words - }; - - index.main.put_words_fst(writer, &words)?; - index.main.put_ranked_map(writer, ranked_map)?; - index.main.put_number_of_documents(writer, |old| old + number_of_inserted_documents as u64)?; - - compute_short_prefixes(writer, &words, index)?; - - Ok(()) -} diff --git a/meilisearch-core/src/update/documents_deletion.rs b/meilisearch-core/src/update/documents_deletion.rs deleted file mode 100644 index def6251c8..000000000 --- a/meilisearch-core/src/update/documents_deletion.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::collections::{BTreeSet, HashMap, HashSet}; - -use fst::{SetBuilder, Streamer}; -use sdset::{duo::DifferenceByKey, SetBuf, SetOperation}; - -use crate::database::{MainT, UpdateT}; -use crate::database::{UpdateEvent, UpdateEventsEmitter}; -use crate::facets; -use crate::store; -use crate::update::{next_update_id, compute_short_prefixes, Update}; -use crate::{DocumentId, Error, MResult, RankedMap, MainWriter, Index}; - -pub struct DocumentsDeletion { - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - updates_notifier: UpdateEventsEmitter, - external_docids: Vec, -} - -impl DocumentsDeletion { - pub fn new( - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - updates_notifier: UpdateEventsEmitter, - ) -> DocumentsDeletion { - DocumentsDeletion { - updates_store, - updates_results_store, - updates_notifier, - external_docids: Vec::new(), - } - } - - pub fn delete_document_by_external_docid(&mut self, document_id: String) { - self.external_docids.push(document_id); - } - - pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult { - let _ = self.updates_notifier.send(UpdateEvent::NewUpdate); - let update_id = push_documents_deletion( - writer, - self.updates_store, - self.updates_results_store, - self.external_docids, - )?; - Ok(update_id) - } -} - -impl Extend for DocumentsDeletion { - fn extend>(&mut self, iter: T) { - self.external_docids.extend(iter) - } -} - -pub fn push_documents_deletion( - writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - external_docids: Vec, -) -> MResult { - let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; - - let update = Update::documents_deletion(external_docids); - updates_store.put_update(writer, last_update_id, &update)?; - - Ok(last_update_id) -} - -pub fn apply_documents_deletion( - writer: &mut heed::RwTxn, - index: &store::Index, - external_docids: Vec, -) -> MResult<()> -{ - let (external_docids, internal_docids) = { - let new_external_docids = SetBuf::from_dirty(external_docids); - let mut internal_docids = Vec::new(); - - let old_external_docids = index.main.external_docids(writer)?; - for external_docid in new_external_docids.as_slice() { - if let Some(id) = old_external_docids.get(external_docid) { - internal_docids.push(DocumentId(id as u32)); - } - } - - let new_external_docids = fst::Map::from_iter(new_external_docids.into_iter().map(|k| (k, 0))).unwrap(); - (new_external_docids, SetBuf::from_dirty(internal_docids)) - }; - - let schema = match index.main.schema(writer)? { - Some(schema) => schema, - None => return Err(Error::SchemaMissing), - }; - - let mut ranked_map = match index.main.ranked_map(writer)? { - Some(ranked_map) => ranked_map, - None => RankedMap::default(), - }; - - // facet filters deletion - if let Some(attributes_for_facetting) = index.main.attributes_for_faceting(writer)? { - let facet_map = facets::facet_map_from_docids(writer, &index, &internal_docids, &attributes_for_facetting)?; - index.facets.remove(writer, facet_map)?; - } - - // collect the ranked attributes according to the schema - let ranked_fields = schema.ranked(); - - let mut words_document_ids = HashMap::new(); - for id in internal_docids.iter().cloned() { - // remove all the ranked attributes from the ranked_map - for ranked_attr in ranked_fields { - ranked_map.remove(id, *ranked_attr); - } - - let words = index.docs_words.doc_words(writer, id)?; - if !words.is_empty() { - let mut stream = words.stream(); - while let Some(word) = stream.next() { - let word = word.to_vec(); - words_document_ids - .entry(word) - .or_insert_with(Vec::new) - .push(id); - } - } - } - - let mut deleted_documents = HashSet::new(); - let mut removed_words = BTreeSet::new(); - for (word, document_ids) in words_document_ids { - let document_ids = SetBuf::from_dirty(document_ids); - - if let Some(postings) = index.postings_lists.postings_list(writer, &word)? { - let op = DifferenceByKey::new(&postings.matches, &document_ids, |d| d.document_id, |id| *id); - let doc_indexes = op.into_set_buf(); - - if !doc_indexes.is_empty() { - index.postings_lists.put_postings_list(writer, &word, &doc_indexes)?; - } else { - index.postings_lists.del_postings_list(writer, &word)?; - removed_words.insert(word); - } - } - - for id in document_ids { - index.documents_fields_counts.del_all_document_fields_counts(writer, id)?; - if index.documents_fields.del_all_document_fields(writer, id)? != 0 { - deleted_documents.insert(id); - } - } - } - - let deleted_documents_len = deleted_documents.len() as u64; - for id in &deleted_documents { - index.docs_words.del_doc_words(writer, *id)?; - } - - let removed_words = fst::Set::from_iter(removed_words).unwrap(); - let words = { - let words_set = index.main.words_fst(writer)?; - let op = fst::set::OpBuilder::new() - .add(words_set.stream()) - .add(removed_words.stream()) - .difference(); - - let mut words_builder = SetBuilder::memory(); - words_builder.extend_stream(op).unwrap(); - words_builder.into_set() - }; - - index.main.put_words_fst(writer, &words)?; - index.main.put_ranked_map(writer, &ranked_map)?; - index.main.put_number_of_documents(writer, |old| old - deleted_documents_len)?; - - // We apply the changes to the user and internal ids - index.main.remove_external_docids(writer, &external_docids)?; - index.main.remove_internal_docids(writer, &internal_docids)?; - - compute_short_prefixes(writer, &words, index)?; - - // update is finished; update sorted document id cache with new state - document_cache_remove_deleted(writer, index, &ranked_map, &deleted_documents)?; - - Ok(()) -} - -/// rebuilds the document id cache by either removing deleted documents from the existing cache, -/// and generating a new one from docs in store -fn document_cache_remove_deleted(writer: &mut MainWriter, index: &Index, ranked_map: &RankedMap, documents_to_delete: &HashSet) -> MResult<()> { - let new_cache = match index.main.sorted_document_ids_cache(writer)? { - // only keep documents that are not in the list of deleted documents. Order is preserved, - // no need to resort - Some(old_cache) => { - old_cache.iter().filter(|docid| !documents_to_delete.contains(docid)).cloned().collect::>() - } - // couldn't find cached documents, try building a new cache from documents in store - None => { - let mut document_ids = index.main.internal_docids(writer)?.to_vec(); - super::cache_document_ids_sorted(writer, ranked_map, index, &mut document_ids)?; - document_ids - } - }; - index.main.put_sorted_document_ids_cache(writer, &new_cache)?; - Ok(()) -} diff --git a/meilisearch-core/src/update/helpers.rs b/meilisearch-core/src/update/helpers.rs deleted file mode 100644 index 8d9ff633c..000000000 --- a/meilisearch-core/src/update/helpers.rs +++ /dev/null @@ -1,142 +0,0 @@ -use std::fmt::Write as _; - -use indexmap::IndexMap; -use meilisearch_schema::IndexedPos; -use meilisearch_types::DocumentId; -use ordered_float::OrderedFloat; -use serde_json::Value; - -use crate::Number; -use crate::raw_indexer::RawIndexer; -use crate::serde::SerializerError; -use crate::store::DiscoverIds; - -/// Returns the number of words indexed or `None` if the type is unindexable. -pub fn index_value>( - indexer: &mut RawIndexer, - document_id: DocumentId, - indexed_pos: IndexedPos, - value: &Value, -) -> Option -{ - match value { - Value::Null => None, - Value::Bool(boolean) => { - let text = boolean.to_string(); - let number_of_words = indexer.index_text(document_id, indexed_pos, &text); - Some(number_of_words) - }, - Value::Number(number) => { - let text = number.to_string(); - Some(indexer.index_text(document_id, indexed_pos, &text)) - }, - Value::String(string) => { - Some(indexer.index_text(document_id, indexed_pos, &string)) - }, - Value::Array(_) => { - let text = value_to_string(value); - Some(indexer.index_text(document_id, indexed_pos, &text)) - }, - Value::Object(_) => { - let text = value_to_string(value); - Some(indexer.index_text(document_id, indexed_pos, &text)) - }, - } -} - -/// Transforms the JSON Value type into a String. -pub fn value_to_string(value: &Value) -> String { - fn internal_value_to_string(string: &mut String, value: &Value) { - match value { - Value::Null => (), - Value::Bool(boolean) => { let _ = write!(string, "{}", &boolean); }, - Value::Number(number) => { let _ = write!(string, "{}", &number); }, - Value::String(text) => string.push_str(&text), - Value::Array(array) => { - for value in array { - internal_value_to_string(string, value); - let _ = string.write_str(". "); - } - }, - Value::Object(object) => { - for (key, value) in object { - string.push_str(key); - let _ = string.write_str(". "); - internal_value_to_string(string, value); - let _ = string.write_str(". "); - } - }, - } - } - - let mut string = String::new(); - internal_value_to_string(&mut string, value); - string -} - -/// Transforms the JSON Value type into a Number. -pub fn value_to_number(value: &Value) -> Option { - use std::str::FromStr; - - match value { - Value::Null => None, - Value::Bool(boolean) => Some(Number::Unsigned(*boolean as u64)), - Value::Number(number) => { - match (number.as_i64(), number.as_u64(), number.as_f64()) { - (Some(n), _, _) => Some(Number::Signed(n)), - (_, Some(n), _) => Some(Number::Unsigned(n)), - (_, _, Some(n)) => Some(Number::Float(OrderedFloat(n))), - (None, None, None) => None, - } - }, - Value::String(string) => Number::from_str(string).ok(), - Value::Array(_array) => None, - Value::Object(_object) => None, - } -} - -/// Validates a string representation to be a correct document id and returns -/// the corresponding id or generate a new one, this is the way we produce documents ids. -pub fn discover_document_id( - docid: &str, - external_docids_get: F, - available_docids: &mut DiscoverIds<'_>, -) -> Result -where - F: FnOnce(&str) -> Option -{ - if docid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') { - match external_docids_get(docid) { - Some(id) => Ok(DocumentId(id)), - None => { - let internal_id = available_docids.next().expect("no more ids available"); - Ok(internal_id) - }, - } - } else { - Err(SerializerError::InvalidDocumentIdFormat) - } -} - -/// Extracts and validates the document id of a document. -pub fn extract_document_id( - primary_key: &str, - document: &IndexMap, - external_docids_get: F, - available_docids: &mut DiscoverIds<'_>, -) -> Result<(DocumentId, String), SerializerError> -where - F: FnOnce(&str) -> Option -{ - match document.get(primary_key) { - Some(value) => { - let docid = match value { - Value::Number(number) => number.to_string(), - Value::String(string) => string.clone(), - _ => return Err(SerializerError::InvalidDocumentIdFormat), - }; - discover_document_id(&docid, external_docids_get, available_docids).map(|id| (id, docid)) - } - None => Err(SerializerError::DocumentIdNotFound), - } -} diff --git a/meilisearch-core/src/update/mod.rs b/meilisearch-core/src/update/mod.rs deleted file mode 100644 index bcc03ec3f..000000000 --- a/meilisearch-core/src/update/mod.rs +++ /dev/null @@ -1,391 +0,0 @@ -mod clear_all; -mod customs_update; -mod documents_addition; -mod documents_deletion; -mod settings_update; -mod helpers; - -pub use self::clear_all::{apply_clear_all, push_clear_all}; -pub use self::customs_update::{apply_customs_update, push_customs_update}; -pub use self::documents_addition::{apply_documents_addition, apply_documents_partial_addition, DocumentsAddition}; -pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion}; -pub use self::helpers::{index_value, value_to_string, value_to_number, discover_document_id, extract_document_id}; -pub use self::settings_update::{apply_settings_update, push_settings_update}; - -use std::cmp; -use std::time::Instant; - -use chrono::{DateTime, Utc}; -use fst::{IntoStreamer, Streamer}; -use heed::Result as ZResult; -use indexmap::IndexMap; -use log::debug; -use sdset::Set; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use meilisearch_error::ErrorCode; -use meilisearch_types::DocumentId; - -use crate::{store, MResult, RankedMap}; -use crate::database::{MainT, UpdateT}; -use crate::settings::SettingsUpdate; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Update { - data: UpdateData, - enqueued_at: DateTime, -} - -impl Update { - fn clear_all() -> Update { - Update { - data: UpdateData::ClearAll, - enqueued_at: Utc::now(), - } - } - - fn customs(data: Vec) -> Update { - Update { - data: UpdateData::Customs(data), - enqueued_at: Utc::now(), - } - } - - fn documents_addition(primary_key: Option, documents: Vec>) -> Update { - Update { - data: UpdateData::DocumentsAddition{ documents, primary_key }, - enqueued_at: Utc::now(), - } - } - - fn documents_partial(primary_key: Option, documents: Vec>) -> Update { - Update { - data: UpdateData::DocumentsPartial{ documents, primary_key }, - enqueued_at: Utc::now(), - } - } - - fn documents_deletion(data: Vec) -> Update { - Update { - data: UpdateData::DocumentsDeletion(data), - enqueued_at: Utc::now(), - } - } - - fn settings(data: SettingsUpdate) -> Update { - Update { - data: UpdateData::Settings(Box::new(data)), - enqueued_at: Utc::now(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateData { - ClearAll, - Customs(Vec), - // (primary key, documents) - DocumentsAddition { - primary_key: Option, - documents: Vec> - }, - DocumentsPartial { - primary_key: Option, - documents: Vec>, - }, - DocumentsDeletion(Vec), - Settings(Box) -} - -impl UpdateData { - pub fn update_type(&self) -> UpdateType { - match self { - UpdateData::ClearAll => UpdateType::ClearAll, - UpdateData::Customs(_) => UpdateType::Customs, - UpdateData::DocumentsAddition{ documents, .. } => UpdateType::DocumentsAddition { - number: documents.len(), - }, - UpdateData::DocumentsPartial{ documents, .. } => UpdateType::DocumentsPartial { - number: documents.len(), - }, - UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion { - number: deletion.len(), - }, - UpdateData::Settings(update) => UpdateType::Settings { - settings: update.clone(), - }, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "name")] -pub enum UpdateType { - ClearAll, - Customs, - DocumentsAddition { number: usize }, - DocumentsPartial { number: usize }, - DocumentsDeletion { number: usize }, - Settings { settings: Box }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessedUpdateResult { - pub update_id: u64, - #[serde(rename = "type")] - pub update_type: UpdateType, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_link: Option, - pub duration: f64, // in seconds - pub enqueued_at: DateTime, - pub processed_at: DateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnqueuedUpdateResult { - pub update_id: u64, - #[serde(rename = "type")] - pub update_type: UpdateType, - pub enqueued_at: DateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", tag = "status")] -pub enum UpdateStatus { - Enqueued { - #[serde(flatten)] - content: EnqueuedUpdateResult, - }, - Failed { - #[serde(flatten)] - content: ProcessedUpdateResult, - }, - Processed { - #[serde(flatten)] - content: ProcessedUpdateResult, - }, -} - -pub fn update_status( - update_reader: &heed::RoTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - update_id: u64, -) -> MResult> { - match updates_results_store.update_result(update_reader, update_id)? { - Some(result) => { - if result.error.is_some() { - Ok(Some(UpdateStatus::Failed { content: result })) - } else { - Ok(Some(UpdateStatus::Processed { content: result })) - } - }, - None => match updates_store.get(update_reader, update_id)? { - Some(update) => Ok(Some(UpdateStatus::Enqueued { - content: EnqueuedUpdateResult { - update_id, - update_type: update.data.update_type(), - enqueued_at: update.enqueued_at, - }, - })), - None => Ok(None), - }, - } -} - -pub fn next_update_id( - update_writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, -) -> ZResult { - let last_update = updates_store.last_update(update_writer)?; - let last_update = last_update.map(|(n, _)| n); - - let last_update_results_id = updates_results_store.last_update(update_writer)?; - let last_update_results_id = last_update_results_id.map(|(n, _)| n); - - let max_update_id = cmp::max(last_update, last_update_results_id); - let new_update_id = max_update_id.map_or(0, |n| n + 1); - - Ok(new_update_id) -} - -pub fn update_task( - writer: &mut heed::RwTxn, - index: &store::Index, - update_id: u64, - update: Update, -) -> MResult { - debug!("Processing update number {}", update_id); - - let Update { enqueued_at, data } = update; - - let (update_type, result, duration) = match data { - UpdateData::ClearAll => { - let start = Instant::now(); - - let update_type = UpdateType::ClearAll; - let result = apply_clear_all(writer, index); - - (update_type, result, start.elapsed()) - } - UpdateData::Customs(customs) => { - let start = Instant::now(); - - let update_type = UpdateType::Customs; - let result = apply_customs_update(writer, index.main, &customs).map_err(Into::into); - - (update_type, result, start.elapsed()) - } - UpdateData::DocumentsAddition { documents, primary_key } => { - let start = Instant::now(); - - let update_type = UpdateType::DocumentsAddition { - number: documents.len(), - }; - - let result = apply_documents_addition(writer, index, documents, primary_key); - - (update_type, result, start.elapsed()) - } - UpdateData::DocumentsPartial{ documents, primary_key } => { - let start = Instant::now(); - - let update_type = UpdateType::DocumentsPartial { - number: documents.len(), - }; - - let result = apply_documents_partial_addition(writer, index, documents, primary_key); - - (update_type, result, start.elapsed()) - } - UpdateData::DocumentsDeletion(documents) => { - let start = Instant::now(); - - let update_type = UpdateType::DocumentsDeletion { - number: documents.len(), - }; - - let result = apply_documents_deletion(writer, index, documents); - - (update_type, result, start.elapsed()) - } - UpdateData::Settings(settings) => { - let start = Instant::now(); - - let update_type = UpdateType::Settings { - settings: settings.clone(), - }; - - let result = apply_settings_update( - writer, - index, - *settings, - ); - - (update_type, result, start.elapsed()) - } - }; - - debug!( - "Processed update number {} {:?} {:?}", - update_id, update_type, result - ); - - let status = ProcessedUpdateResult { - update_id, - update_type, - error: result.as_ref().map_err(|e| e.to_string()).err(), - error_code: result.as_ref().map_err(|e| e.error_name()).err(), - error_type: result.as_ref().map_err(|e| e.error_type()).err(), - error_link: result.as_ref().map_err(|e| e.error_url()).err(), - duration: duration.as_secs_f64(), - enqueued_at, - processed_at: Utc::now(), - }; - - Ok(status) -} - -fn compute_short_prefixes( - writer: &mut heed::RwTxn, - words_fst: &fst::Set, - index: &store::Index, -) -> MResult<()> -where A: AsRef<[u8]>, -{ - // clear the prefixes - let pplc_store = index.prefix_postings_lists_cache; - pplc_store.clear(writer)?; - - for prefix_len in 1..=2 { - // compute prefixes and store those in the PrefixPostingsListsCache store. - let mut previous_prefix: Option<([u8; 4], Vec<_>)> = None; - let mut stream = words_fst.into_stream(); - while let Some(input) = stream.next() { - - // We skip the prefixes that are shorter than the current length - // we want to cache (<). We must ignore the input when it is exactly the - // same word as the prefix because if we match exactly on it we need - // to consider it as an exact match and not as a prefix (=). - if input.len() <= prefix_len { continue } - - if let Some(postings_list) = index.postings_lists.postings_list(writer, input)?.map(|p| p.matches.into_owned()) { - let prefix = &input[..prefix_len]; - - let mut arr_prefix = [0; 4]; - arr_prefix[..prefix_len].copy_from_slice(prefix); - - match previous_prefix { - Some((ref mut prev_prefix, ref mut prev_pl)) if *prev_prefix != arr_prefix => { - prev_pl.sort_unstable(); - prev_pl.dedup(); - - if let Ok(prefix) = std::str::from_utf8(&prev_prefix[..prefix_len]) { - debug!("writing the prefix of {:?} of length {}", prefix, prev_pl.len()); - } - - let pls = Set::new_unchecked(&prev_pl); - pplc_store.put_prefix_postings_list(writer, *prev_prefix, &pls)?; - - *prev_prefix = arr_prefix; - prev_pl.clear(); - prev_pl.extend_from_slice(&postings_list); - }, - Some((_, ref mut prev_pl)) => prev_pl.extend_from_slice(&postings_list), - None => previous_prefix = Some((arr_prefix, postings_list.to_vec())), - } - } - } - - // write the last prefix postings lists - if let Some((prev_prefix, mut prev_pl)) = previous_prefix.take() { - prev_pl.sort_unstable(); - prev_pl.dedup(); - - let pls = Set::new_unchecked(&prev_pl); - pplc_store.put_prefix_postings_list(writer, prev_prefix, &pls)?; - } - } - - Ok(()) -} - -fn cache_document_ids_sorted( - writer: &mut heed::RwTxn, - ranked_map: &RankedMap, - index: &store::Index, - document_ids: &mut [DocumentId], -) -> MResult<()> { - crate::bucket_sort::placeholder_document_sort(document_ids, index, writer, ranked_map)?; - index.main.put_sorted_document_ids_cache(writer, &document_ids) -} diff --git a/meilisearch-core/src/update/settings_update.rs b/meilisearch-core/src/update/settings_update.rs deleted file mode 100644 index c9d40fa1b..000000000 --- a/meilisearch-core/src/update/settings_update.rs +++ /dev/null @@ -1,332 +0,0 @@ -use std::{borrow::Cow, collections::{BTreeMap, BTreeSet}}; - -use heed::Result as ZResult; -use fst::{SetBuilder, set::OpBuilder}; -use sdset::SetBuf; -use meilisearch_schema::Schema; -use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig}; - -use crate::database::{MainT, UpdateT}; -use crate::settings::{UpdateState, SettingsUpdate, RankingRule}; -use crate::update::documents_addition::reindex_all_documents; -use crate::update::{next_update_id, Update}; -use crate::{store, MResult, Error}; - -pub fn push_settings_update( - writer: &mut heed::RwTxn, - updates_store: store::Updates, - updates_results_store: store::UpdatesResults, - settings: SettingsUpdate, -) -> ZResult { - let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; - - let update = Update::settings(settings); - updates_store.put_update(writer, last_update_id, &update)?; - - Ok(last_update_id) -} - -pub fn apply_settings_update( - writer: &mut heed::RwTxn, - index: &store::Index, - settings: SettingsUpdate, -) -> MResult<()> { - let mut must_reindex = false; - - let mut schema = match index.main.schema(writer)? { - Some(schema) => schema, - None => { - match settings.primary_key.clone() { - UpdateState::Update(id) => Schema::with_primary_key(&id), - _ => return Err(Error::MissingPrimaryKey) - } - } - }; - - match settings.ranking_rules { - UpdateState::Update(v) => { - let ranked_field: Vec<&str> = v.iter().filter_map(RankingRule::field).collect(); - schema.update_ranked(&ranked_field)?; - index.main.put_ranking_rules(writer, &v)?; - must_reindex = true; - }, - UpdateState::Clear => { - index.main.delete_ranking_rules(writer)?; - schema.clear_ranked(); - must_reindex = true; - }, - UpdateState::Nothing => (), - } - - match settings.distinct_attribute { - UpdateState::Update(v) => { - let field_id = schema.insert(&v)?; - index.main.put_distinct_attribute(writer, field_id)?; - }, - UpdateState::Clear => { - index.main.delete_distinct_attribute(writer)?; - }, - UpdateState::Nothing => (), - } - - match settings.searchable_attributes.clone() { - UpdateState::Update(v) => { - if v.iter().any(|e| e == "*") || v.is_empty() { - schema.set_all_searchable(); - } else { - schema.update_searchable(v)?; - } - must_reindex = true; - }, - UpdateState::Clear => { - schema.set_all_searchable(); - must_reindex = true; - }, - UpdateState::Nothing => (), - } - match settings.displayed_attributes.clone() { - UpdateState::Update(v) => { - if v.contains("*") || v.is_empty() { - schema.set_all_displayed(); - } else { - schema.update_displayed(v)? - } - }, - UpdateState::Clear => { - schema.set_all_displayed(); - }, - UpdateState::Nothing => (), - } - - match settings.attributes_for_faceting { - UpdateState::Update(attrs) => { - apply_attributes_for_faceting_update(writer, index, &mut schema, &attrs)?; - must_reindex = true; - }, - UpdateState::Clear => { - index.main.delete_attributes_for_faceting(writer)?; - index.facets.clear(writer)?; - }, - UpdateState::Nothing => (), - } - - index.main.put_schema(writer, &schema)?; - - match settings.stop_words { - UpdateState::Update(stop_words) => { - if apply_stop_words_update(writer, index, stop_words)? { - must_reindex = true; - } - }, - UpdateState::Clear => { - if apply_stop_words_update(writer, index, BTreeSet::new())? { - must_reindex = true; - } - }, - UpdateState::Nothing => (), - } - - match settings.synonyms { - UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, synonyms)?, - UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?, - UpdateState::Nothing => (), - } - - if must_reindex { - reindex_all_documents(writer, index)?; - } - - Ok(()) -} - -fn apply_attributes_for_faceting_update( - writer: &mut heed::RwTxn, - index: &store::Index, - schema: &mut Schema, - attributes: &[String] - ) -> MResult<()> { - let mut attribute_ids = Vec::new(); - for name in attributes { - attribute_ids.push(schema.insert(name)?); - } - let attributes_for_faceting = SetBuf::from_dirty(attribute_ids); - index.main.put_attributes_for_faceting(writer, &attributes_for_faceting)?; - Ok(()) -} - -pub fn apply_stop_words_update( - writer: &mut heed::RwTxn, - index: &store::Index, - stop_words: BTreeSet, -) -> MResult -{ - let mut must_reindex = false; - - let old_stop_words: BTreeSet = index.main - .stop_words_fst(writer)? - .stream() - .into_strs()? - .into_iter() - .collect(); - - let deletion: BTreeSet = old_stop_words.difference(&stop_words).cloned().collect(); - let addition: BTreeSet = stop_words.difference(&old_stop_words).cloned().collect(); - - if !addition.is_empty() { - apply_stop_words_addition(writer, index, addition)?; - } - - if !deletion.is_empty() { - must_reindex = true; - apply_stop_words_deletion(writer, index, deletion)?; - } - - let words_fst = index.main.words_fst(writer)?; - if !words_fst.is_empty() { - let stop_words = fst::Set::from_iter(stop_words)?; - let op = OpBuilder::new() - .add(&words_fst) - .add(&stop_words) - .difference(); - - let mut builder = fst::SetBuilder::memory(); - builder.extend_stream(op)?; - let words_fst = builder.into_set(); - - index.main.put_words_fst(writer, &words_fst)?; - index.main.put_stop_words_fst(writer, &stop_words)?; - } - - Ok(must_reindex) -} - -fn apply_stop_words_addition( - writer: &mut heed::RwTxn, - index: &store::Index, - addition: BTreeSet, -) -> MResult<()> -{ - let main_store = index.main; - let postings_lists_store = index.postings_lists; - - let mut stop_words_builder = SetBuilder::memory(); - - for word in addition { - stop_words_builder.insert(&word)?; - // we remove every posting list associated to a new stop word - postings_lists_store.del_postings_list(writer, word.as_bytes())?; - } - - // create the new delta stop words fst - let delta_stop_words = stop_words_builder.into_set(); - - // we also need to remove all the stop words from the main fst - let words_fst = main_store.words_fst(writer)?; - if !words_fst.is_empty() { - let op = OpBuilder::new() - .add(&words_fst) - .add(&delta_stop_words) - .difference(); - - let mut word_fst_builder = SetBuilder::memory(); - word_fst_builder.extend_stream(op)?; - let word_fst = word_fst_builder.into_set(); - - main_store.put_words_fst(writer, &word_fst)?; - } - - // now we add all of these stop words from the main store - let stop_words_fst = main_store.stop_words_fst(writer)?; - - let op = OpBuilder::new() - .add(&stop_words_fst) - .add(&delta_stop_words) - .r#union(); - - let mut stop_words_builder = SetBuilder::memory(); - stop_words_builder.extend_stream(op)?; - let stop_words_fst = stop_words_builder.into_set(); - - main_store.put_stop_words_fst(writer, &stop_words_fst)?; - - Ok(()) -} - -fn apply_stop_words_deletion( - writer: &mut heed::RwTxn, - index: &store::Index, - deletion: BTreeSet, -) -> MResult<()> { - - let mut stop_words_builder = SetBuilder::memory(); - - for word in deletion { - stop_words_builder.insert(&word)?; - } - - // create the new delta stop words fst - let delta_stop_words = stop_words_builder.into_set(); - - // now we delete all of these stop words from the main store - let stop_words_fst = index.main.stop_words_fst(writer)?; - - let op = OpBuilder::new() - .add(&stop_words_fst) - .add(&delta_stop_words) - .difference(); - - let mut stop_words_builder = SetBuilder::memory(); - stop_words_builder.extend_stream(op)?; - let stop_words_fst = stop_words_builder.into_set(); - - Ok(index.main.put_stop_words_fst(writer, &stop_words_fst)?) -} - -pub fn apply_synonyms_update( - writer: &mut heed::RwTxn, - index: &store::Index, - synonyms: BTreeMap>, -) -> MResult<()> { - - let main_store = index.main; - let synonyms_store = index.synonyms; - let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?; - let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(&stop_words)); - - fn normalize>(analyzer: &Analyzer, text: &str) -> String { - analyzer.analyze(&text) - .tokens() - .fold(String::new(), |s, t| s + t.text()) - } - - // normalize synonyms and reorder them creating a BTreeMap - let synonyms: BTreeMap> = synonyms.into_iter().map( |(word, alternatives)| { - let word = normalize(&analyzer, &word); - let alternatives = alternatives.into_iter().map(|text| normalize(&analyzer, &text)).collect(); - - (word, alternatives) - }).collect(); - - // index synonyms, - // synyonyms have to be ordered by key before indexation - let mut synonyms_builder = SetBuilder::memory(); - synonyms_store.clear(writer)?; - for (word, alternatives) in synonyms { - synonyms_builder.insert(&word)?; - - let alternatives = { - let alternatives = SetBuf::from_dirty(alternatives); - let mut alternatives_builder = SetBuilder::memory(); - alternatives_builder.extend_iter(alternatives)?; - alternatives_builder.into_set() - }; - - synonyms_store.put_synonyms(writer, word.as_bytes(), &alternatives)?; - } - - let synonyms_set = synonyms_builder.into_set(); - - main_store.put_synonyms_fst(writer, &synonyms_set)?; - - Ok(()) -} diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml index 96172d0dd..b06ea4df6 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-error/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "meilisearch-error" -version = "0.20.0" +version = "0.21.0" authors = ["marin "] edition = "2018" [dependencies] -actix-http = "2.2.0" +actix-http = "=3.0.0-beta.6" diff --git a/meilisearch-error/src/lib.rs b/meilisearch-error/src/lib.rs index d0e00e9be..4cf801c72 100644 --- a/meilisearch-error/src/lib.rs +++ b/meilisearch-error/src/lib.rs @@ -81,7 +81,6 @@ pub enum Code { } impl Code { - /// ascociate a `Code` variant to the actual ErrCode fn err_code(&self) -> ErrCode { use Code::*; @@ -94,17 +93,23 @@ impl Code { // thrown when requesting an unexisting index IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND), InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST), - OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR), + OpenIndex => { + ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR) + } // invalid state error InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR), // thrown when no primary key has been set MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST), // error thrown when trying to set an already existing primary key - PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST), + PrimaryKeyAlreadyPresent => { + ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST) + } // invalid document - MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST), + MaxFieldsLimitExceeded => { + ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST) + } MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST), // error related to facets @@ -117,16 +122,26 @@ impl Code { DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND), Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR), InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN), - MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED), + MissingAuthorizationHeader => { + ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED) + } NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND), PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE), - RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST), + RetrieveDocument => { + ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST) + } SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST), - UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE), + UnsupportedMediaType => { + ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE) + } // error related to dump - DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT), - DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR), + DumpAlreadyInProgress => { + ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT) + } + DumpProcessFailed => { + ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR) + } } } diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 68b4a7645..51ba63b8e 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -1,86 +1,123 @@ [package] -name = "meilisearch-http" +authors = ["Quentin de Quelen ", "Clément Renault "] description = "MeiliSearch HTTP server" -version = "0.20.0" -license = "MIT" -authors = [ - "Quentin de Quelen ", - "Clément Renault ", -] edition = "2018" +license = "MIT" +name = "meilisearch-http" +version = "0.21.0" [[bin]] name = "meilisearch" path = "src/main.rs" -[features] -default = ["sentry"] +[build-dependencies] +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } +anyhow = { version = "*", optional = true } +cargo_toml = { version = "0.9.0", optional = true } +hex = { version = "0.4.3", optional = true } +reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true } +sha-1 = { version = "0.9.4", optional = true } +tempfile = { version = "3.1.0", optional = true } +vergen = "3.1.0" +zip = { version = "0.5.12", optional = true } [dependencies] -actix-cors = "0.5.4" -actix-http = "2.2.0" -actix-rt = "1.1.1" -actix-service = "1.0.6" -actix-web = { version = "3.3.2", features = ["rustls"] } -bytes = "1.0.0" +actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"} +actix-http = { version = "=3.0.0-beta.6" } +actix-service = "2.0.0" +actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] } +actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true } +anyhow = "1.0.36" +async-stream = "0.3.0" +async-trait = "0.1.42" +arc-swap = "1.2.0" +byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +bytes = "0.6.0" chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.0" +either = "1.6.1" env_logger = "0.8.2" flate2 = "1.0.19" -futures = "0.3.8" -http = "0.2.2" -indexmap = { version = "1.6.1", features = ["serde-1"] } -log = "0.4.11" -main_error = "0.1.1" -meilisearch-core = { path = "../meilisearch-core", version = "0.20.0" } -meilisearch-error = { path = "../meilisearch-error", version = "0.20.0" } -meilisearch-schema = { path = "../meilisearch-schema", version = "0.20.0" } +fst = "0.4.5" +futures = "0.3.7" +futures-util = "0.3.8" +grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } +http = "0.2.1" +indexmap = { version = "1.3.2", features = ["serde-1"] } +itertools = "0.10.0" +log = "0.4.8" +main_error = "0.1.0" +meilisearch-error = { path = "../meilisearch-error" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } +memmap = "0.7.0" +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.0" } mime = "0.3.16" +num_cpus = "1.13.0" once_cell = "1.5.2" -rand = "0.8.1" +oxidized-json-checker = "0.3.2" +parking_lot = "0.11.1" +rand = "0.7.3" +rayon = "1.5.0" regex = "1.4.2" -rustls = "0.18.0" -serde = { version = "1.0.118", features = ["derive"] } -serde_json = { version = "1.0.61", features = ["preserve_order"] } -serde_qs = "0.8.2" -sha2 = "0.9.2" -siphasher = "0.3.3" +rustls = "0.19" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.59", features = ["preserve_order"] } +sha2 = "0.9.1" +siphasher = "0.3.2" slice-group-by = "0.2.6" -structopt = "0.3.21" -tar = "0.4.30" +structopt = "0.3.20" +tar = "0.4.29" tempfile = "3.1.0" -tokio = { version = "0.2", features = ["macros"] } -ureq = { version = "2.0.0", features = ["tls"], default-features = false } -uuid = "0.8" -walkdir = "2.3.1" -whoami = "1.0.3" +thiserror = "1.0.24" +tokio = { version = "1", features = ["full"] } +uuid = { version = "0.8.2", features = ["serde"] } +walkdir = "2.3.2" +obkv = "0.1.1" +pin-project = "1.0.7" +whoami = { version = "1.1.2", optional = true } +reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } [dependencies.sentry] -version = "0.18.1" default-features = false features = [ - "with_client_implementation", - "with_panic", - "with_failure", - "with_device_info", - "with_rust_info", - "with_reqwest_transport", - "with_rustls", - "with_env_logger" + "backtrace", + "contexts", + "panic", + "reqwest", + "rustls", + "log", ] optional = true +version = "0.22.0" + [dev-dependencies] +actix-rt = "2.1.0" +assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } +mockall = "0.9.1" +paste = "1.0.5" serde_url_params = "0.2.0" tempdir = "0.3.7" -tokio = { version = "0.2", features = ["macros", "time"] } +urlencoding = "1.1.1" -[dev-dependencies.assert-json-diff] -git = "https://github.com/qdequele/assert-json-diff" -branch = "master" - -[build-dependencies] -vergen = "3.1.0" +[features] +mini-dashboard = [ + "actix-web-static-files", + "anyhow", + "cargo_toml", + "hex", + "reqwest", + "sha-1", + "tempfile", + "zip", +] +analytics = ["sentry", "whoami", "reqwest"] +default = ["analytics", "mini-dashboard"] [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" + +[package.metadata.mini-dashboard] +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip" +sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e" diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 2257407a8..557e04fe7 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -7,4 +7,83 @@ fn main() { // Generate the 'cargo:' key output generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); + + #[cfg(feature = "mini-dashboard")] + mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets"); +} + +#[cfg(feature = "mini-dashboard")] +mod mini_dashboard { + use std::env; + use std::fs::{create_dir_all, File, OpenOptions}; + use std::io::{Cursor, Read, Write}; + use std::path::PathBuf; + + use actix_web_static_files::resource_dir; + use anyhow::Context; + use cargo_toml::Manifest; + use reqwest::blocking::get; + use sha1::{Digest, Sha1}; + + pub fn setup_mini_dashboard() -> anyhow::Result<()> { + let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let cargo_toml = cargo_manifest_dir.join("Cargo.toml"); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + let sha1_path = out_dir.join(".mini-dashboard.sha1"); + let dashboard_dir = out_dir.join("mini-dashboard"); + + let manifest = Manifest::from_path(cargo_toml).unwrap(); + + let meta = &manifest + .package + .as_ref() + .context("package not specified in Cargo.toml")? + .metadata + .as_ref() + .context("no metadata specified in Cargo.toml")?["mini-dashboard"]; + + // Check if there already is a dashboard built, and if it is up to date. + if sha1_path.exists() && dashboard_dir.exists() { + let mut sha1_file = File::open(&sha1_path)?; + let mut sha1 = String::new(); + sha1_file.read_to_string(&mut sha1)?; + if sha1 == meta["sha1"].as_str().unwrap() { + // Nothing to do. + return Ok(()); + } + } + + let url = meta["assets-url"].as_str().unwrap(); + + let dashboard_assets_bytes = get(url)?.bytes()?; + + let mut hasher = Sha1::new(); + hasher.update(&dashboard_assets_bytes); + let sha1 = hex::encode(hasher.finalize()); + + assert_eq!( + meta["sha1"].as_str().unwrap(), + sha1, + "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml" + ); + + create_dir_all(&dashboard_dir)?; + let cursor = Cursor::new(&dashboard_assets_bytes); + let mut zip = zip::read::ZipArchive::new(cursor)?; + zip.extract(&dashboard_dir)?; + resource_dir(&dashboard_dir).build()?; + + // Write the sha1 for the dashboard back to file. + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(sha1_path)?; + + file.write_all(sha1.as_bytes())?; + file.flush()?; + + Ok(()) + } } diff --git a/meilisearch-http/public/bulma.min.css b/meilisearch-http/public/bulma.min.css deleted file mode 100644 index d0570ff03..000000000 --- a/meilisearch-http/public/bulma.min.css +++ /dev/null @@ -1 +0,0 @@ -/*! bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */@-webkit-keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}@keyframes spinAround{from{transform:rotate(0)}to{transform:rotate(359deg)}}.breadcrumb,.button,.delete,.file,.is-unselectable,.modal-close,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.tabs{-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.navbar-link:not(.is-arrowless)::after,.select:not(.is-multiple):not(.is-loading)::after{border:3px solid transparent;border-radius:2px;border-right:0;border-top:0;content:" ";display:block;height:.625em;margin-top:-.4375em;pointer-events:none;position:absolute;top:50%;transform:rotate(-45deg);transform-origin:center;width:.625em}.block:not(:last-child),.box:not(:last-child),.breadcrumb:not(:last-child),.content:not(:last-child),.highlight:not(:last-child),.level:not(:last-child),.list:not(:last-child),.message:not(:last-child),.notification:not(:last-child),.pagination:not(:last-child),.progress:not(:last-child),.subtitle:not(:last-child),.table-container:not(:last-child),.table:not(:last-child),.tabs:not(:last-child),.title:not(:last-child){margin-bottom:1.5rem}.delete,.modal-close{-moz-appearance:none;-webkit-appearance:none;background-color:rgba(10,10,10,.2);border:none;border-radius:290486px;cursor:pointer;pointer-events:auto;display:inline-block;flex-grow:0;flex-shrink:0;font-size:0;height:20px;max-height:20px;max-width:20px;min-height:20px;min-width:20px;outline:0;position:relative;vertical-align:top;width:20px}.delete::after,.delete::before,.modal-close::after,.modal-close::before{background-color:#fff;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.delete::before,.modal-close::before{height:2px;width:50%}.delete::after,.modal-close::after{height:50%;width:2px}.delete:focus,.delete:hover,.modal-close:focus,.modal-close:hover{background-color:rgba(10,10,10,.3)}.delete:active,.modal-close:active{background-color:rgba(10,10,10,.4)}.is-small.delete,.is-small.modal-close{height:16px;max-height:16px;max-width:16px;min-height:16px;min-width:16px;width:16px}.is-medium.delete,.is-medium.modal-close{height:24px;max-height:24px;max-width:24px;min-height:24px;min-width:24px;width:24px}.is-large.delete,.is-large.modal-close{height:32px;max-height:32px;max-width:32px;min-height:32px;min-width:32px;width:32px}.button.is-loading::after,.control.is-loading::after,.loader,.select.is-loading::after{-webkit-animation:spinAround .5s infinite linear;animation:spinAround .5s infinite linear;border:2px solid #dbdbdb;border-radius:290486px;border-right-color:transparent;border-top-color:transparent;content:"";display:block;height:1em;position:relative;width:1em}.hero-video,.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img,.is-overlay,.modal,.modal-background{bottom:0;left:0;position:absolute;right:0;top:0}.button,.file-cta,.file-name,.input,.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous,.select select,.textarea{-moz-appearance:none;-webkit-appearance:none;align-items:center;border:1px solid transparent;border-radius:4px;box-shadow:none;display:inline-flex;font-size:1rem;height:2.5em;justify-content:flex-start;line-height:1.5;padding-bottom:calc(.5em - 1px);padding-left:calc(.75em - 1px);padding-right:calc(.75em - 1px);padding-top:calc(.5em - 1px);position:relative;vertical-align:top}.button:active,.button:focus,.file-cta:active,.file-cta:focus,.file-name:active,.file-name:focus,.input:active,.input:focus,.is-active.button,.is-active.file-cta,.is-active.file-name,.is-active.input,.is-active.pagination-ellipsis,.is-active.pagination-link,.is-active.pagination-next,.is-active.pagination-previous,.is-active.textarea,.is-focused.button,.is-focused.file-cta,.is-focused.file-name,.is-focused.input,.is-focused.pagination-ellipsis,.is-focused.pagination-link,.is-focused.pagination-next,.is-focused.pagination-previous,.is-focused.textarea,.pagination-ellipsis:active,.pagination-ellipsis:focus,.pagination-link:active,.pagination-link:focus,.pagination-next:active,.pagination-next:focus,.pagination-previous:active,.pagination-previous:focus,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{outline:0}.button[disabled],.file-cta[disabled],.file-name[disabled],.input[disabled],.pagination-ellipsis[disabled],.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .button,fieldset[disabled] .file-cta,fieldset[disabled] .file-name,fieldset[disabled] .input,fieldset[disabled] .pagination-ellipsis,fieldset[disabled] .pagination-link,fieldset[disabled] .pagination-next,fieldset[disabled] .pagination-previous,fieldset[disabled] .select select,fieldset[disabled] .textarea{cursor:not-allowed}/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */blockquote,body,dd,dl,dt,fieldset,figure,h1,h2,h3,h4,h5,h6,hr,html,iframe,legend,li,ol,p,pre,textarea,ul{margin:0;padding:0}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:400}ul{list-style:none}button,input,select,textarea{margin:0}html{box-sizing:border-box}*,::after,::before{box-sizing:inherit}img,video{height:auto;max-width:100%}iframe{border:0}table{border-collapse:collapse;border-spacing:0}td,th{padding:0}td:not([align]),th:not([align]){text-align:left}html{background-color:#fff;font-size:16px;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;min-width:300px;overflow-x:hidden;overflow-y:scroll;text-rendering:optimizeLegibility;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%;-ms-text-size-adjust:100%;text-size-adjust:100%}article,aside,figure,footer,header,hgroup,section{display:block}body,button,input,select,textarea{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif}code,pre{-moz-osx-font-smoothing:auto;-webkit-font-smoothing:auto;font-family:monospace}body{color:#4a4a4a;font-size:1em;font-weight:400;line-height:1.5}a{color:#3273dc;cursor:pointer;text-decoration:none}a strong{color:currentColor}a:hover{color:#363636}code{background-color:#f5f5f5;color:#f14668;font-size:.875em;font-weight:400;padding:.25em .5em .25em}hr{background-color:#f5f5f5;border:none;display:block;height:2px;margin:1.5rem 0}img{height:auto;max-width:100%}input[type=checkbox],input[type=radio]{vertical-align:baseline}small{font-size:.875em}span{font-style:inherit;font-weight:inherit}strong{color:#363636;font-weight:700}fieldset{border:none}pre{-webkit-overflow-scrolling:touch;background-color:#f5f5f5;color:#4a4a4a;font-size:.875em;overflow-x:auto;padding:1.25rem 1.5rem;white-space:pre;word-wrap:normal}pre code{background-color:transparent;color:currentColor;font-size:1em;padding:0}table td,table th{vertical-align:top}table td:not([align]),table th:not([align]){text-align:left}table th{color:#363636}.is-clearfix::after{clear:both;content:" ";display:table}.is-pulled-left{float:left!important}.is-pulled-right{float:right!important}.is-clipped{overflow:hidden!important}.is-size-1{font-size:3rem!important}.is-size-2{font-size:2.5rem!important}.is-size-3{font-size:2rem!important}.is-size-4{font-size:1.5rem!important}.is-size-5{font-size:1.25rem!important}.is-size-6{font-size:1rem!important}.is-size-7{font-size:.75rem!important}@media screen and (max-width:768px){.is-size-1-mobile{font-size:3rem!important}.is-size-2-mobile{font-size:2.5rem!important}.is-size-3-mobile{font-size:2rem!important}.is-size-4-mobile{font-size:1.5rem!important}.is-size-5-mobile{font-size:1.25rem!important}.is-size-6-mobile{font-size:1rem!important}.is-size-7-mobile{font-size:.75rem!important}}@media screen and (min-width:769px),print{.is-size-1-tablet{font-size:3rem!important}.is-size-2-tablet{font-size:2.5rem!important}.is-size-3-tablet{font-size:2rem!important}.is-size-4-tablet{font-size:1.5rem!important}.is-size-5-tablet{font-size:1.25rem!important}.is-size-6-tablet{font-size:1rem!important}.is-size-7-tablet{font-size:.75rem!important}}@media screen and (max-width:1023px){.is-size-1-touch{font-size:3rem!important}.is-size-2-touch{font-size:2.5rem!important}.is-size-3-touch{font-size:2rem!important}.is-size-4-touch{font-size:1.5rem!important}.is-size-5-touch{font-size:1.25rem!important}.is-size-6-touch{font-size:1rem!important}.is-size-7-touch{font-size:.75rem!important}}@media screen and (min-width:1024px){.is-size-1-desktop{font-size:3rem!important}.is-size-2-desktop{font-size:2.5rem!important}.is-size-3-desktop{font-size:2rem!important}.is-size-4-desktop{font-size:1.5rem!important}.is-size-5-desktop{font-size:1.25rem!important}.is-size-6-desktop{font-size:1rem!important}.is-size-7-desktop{font-size:.75rem!important}}@media screen and (min-width:1216px){.is-size-1-widescreen{font-size:3rem!important}.is-size-2-widescreen{font-size:2.5rem!important}.is-size-3-widescreen{font-size:2rem!important}.is-size-4-widescreen{font-size:1.5rem!important}.is-size-5-widescreen{font-size:1.25rem!important}.is-size-6-widescreen{font-size:1rem!important}.is-size-7-widescreen{font-size:.75rem!important}}@media screen and (min-width:1408px){.is-size-1-fullhd{font-size:3rem!important}.is-size-2-fullhd{font-size:2.5rem!important}.is-size-3-fullhd{font-size:2rem!important}.is-size-4-fullhd{font-size:1.5rem!important}.is-size-5-fullhd{font-size:1.25rem!important}.is-size-6-fullhd{font-size:1rem!important}.is-size-7-fullhd{font-size:.75rem!important}}.has-text-centered{text-align:center!important}.has-text-justified{text-align:justify!important}.has-text-left{text-align:left!important}.has-text-right{text-align:right!important}@media screen and (max-width:768px){.has-text-centered-mobile{text-align:center!important}}@media screen and (min-width:769px),print{.has-text-centered-tablet{text-align:center!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-centered-tablet-only{text-align:center!important}}@media screen and (max-width:1023px){.has-text-centered-touch{text-align:center!important}}@media screen and (min-width:1024px){.has-text-centered-desktop{text-align:center!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-centered-desktop-only{text-align:center!important}}@media screen and (min-width:1216px){.has-text-centered-widescreen{text-align:center!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-centered-widescreen-only{text-align:center!important}}@media screen and (min-width:1408px){.has-text-centered-fullhd{text-align:center!important}}@media screen and (max-width:768px){.has-text-justified-mobile{text-align:justify!important}}@media screen and (min-width:769px),print{.has-text-justified-tablet{text-align:justify!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-justified-tablet-only{text-align:justify!important}}@media screen and (max-width:1023px){.has-text-justified-touch{text-align:justify!important}}@media screen and (min-width:1024px){.has-text-justified-desktop{text-align:justify!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-justified-desktop-only{text-align:justify!important}}@media screen and (min-width:1216px){.has-text-justified-widescreen{text-align:justify!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-justified-widescreen-only{text-align:justify!important}}@media screen and (min-width:1408px){.has-text-justified-fullhd{text-align:justify!important}}@media screen and (max-width:768px){.has-text-left-mobile{text-align:left!important}}@media screen and (min-width:769px),print{.has-text-left-tablet{text-align:left!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-left-tablet-only{text-align:left!important}}@media screen and (max-width:1023px){.has-text-left-touch{text-align:left!important}}@media screen and (min-width:1024px){.has-text-left-desktop{text-align:left!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-left-desktop-only{text-align:left!important}}@media screen and (min-width:1216px){.has-text-left-widescreen{text-align:left!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-left-widescreen-only{text-align:left!important}}@media screen and (min-width:1408px){.has-text-left-fullhd{text-align:left!important}}@media screen and (max-width:768px){.has-text-right-mobile{text-align:right!important}}@media screen and (min-width:769px),print{.has-text-right-tablet{text-align:right!important}}@media screen and (min-width:769px) and (max-width:1023px){.has-text-right-tablet-only{text-align:right!important}}@media screen and (max-width:1023px){.has-text-right-touch{text-align:right!important}}@media screen and (min-width:1024px){.has-text-right-desktop{text-align:right!important}}@media screen and (min-width:1024px) and (max-width:1215px){.has-text-right-desktop-only{text-align:right!important}}@media screen and (min-width:1216px){.has-text-right-widescreen{text-align:right!important}}@media screen and (min-width:1216px) and (max-width:1407px){.has-text-right-widescreen-only{text-align:right!important}}@media screen and (min-width:1408px){.has-text-right-fullhd{text-align:right!important}}.is-capitalized{text-transform:capitalize!important}.is-lowercase{text-transform:lowercase!important}.is-uppercase{text-transform:uppercase!important}.is-italic{font-style:italic!important}.has-text-white{color:#fff!important}a.has-text-white:focus,a.has-text-white:hover{color:#e6e6e6!important}.has-background-white{background-color:#fff!important}.has-text-black{color:#0a0a0a!important}a.has-text-black:focus,a.has-text-black:hover{color:#000!important}.has-background-black{background-color:#0a0a0a!important}.has-text-light{color:#f5f5f5!important}a.has-text-light:focus,a.has-text-light:hover{color:#dbdbdb!important}.has-background-light{background-color:#f5f5f5!important}.has-text-dark{color:#363636!important}a.has-text-dark:focus,a.has-text-dark:hover{color:#1c1c1c!important}.has-background-dark{background-color:#363636!important}.has-text-primary{color:#00d1b2!important}a.has-text-primary:focus,a.has-text-primary:hover{color:#009e86!important}.has-background-primary{background-color:#00d1b2!important}.has-text-link{color:#3273dc!important}a.has-text-link:focus,a.has-text-link:hover{color:#205bbc!important}.has-background-link{background-color:#3273dc!important}.has-text-info{color:#3298dc!important}a.has-text-info:focus,a.has-text-info:hover{color:#207dbc!important}.has-background-info{background-color:#3298dc!important}.has-text-success{color:#48c774!important}a.has-text-success:focus,a.has-text-success:hover{color:#34a85c!important}.has-background-success{background-color:#48c774!important}.has-text-warning{color:#ffdd57!important}a.has-text-warning:focus,a.has-text-warning:hover{color:#ffd324!important}.has-background-warning{background-color:#ffdd57!important}.has-text-danger{color:#f14668!important}a.has-text-danger:focus,a.has-text-danger:hover{color:#ee1742!important}.has-background-danger{background-color:#f14668!important}.has-text-black-bis{color:#121212!important}.has-background-black-bis{background-color:#121212!important}.has-text-black-ter{color:#242424!important}.has-background-black-ter{background-color:#242424!important}.has-text-grey-darker{color:#363636!important}.has-background-grey-darker{background-color:#363636!important}.has-text-grey-dark{color:#4a4a4a!important}.has-background-grey-dark{background-color:#4a4a4a!important}.has-text-grey{color:#7a7a7a!important}.has-background-grey{background-color:#7a7a7a!important}.has-text-grey-light{color:#b5b5b5!important}.has-background-grey-light{background-color:#b5b5b5!important}.has-text-grey-lighter{color:#dbdbdb!important}.has-background-grey-lighter{background-color:#dbdbdb!important}.has-text-white-ter{color:#f5f5f5!important}.has-background-white-ter{background-color:#f5f5f5!important}.has-text-white-bis{color:#fafafa!important}.has-background-white-bis{background-color:#fafafa!important}.has-text-weight-light{font-weight:300!important}.has-text-weight-normal{font-weight:400!important}.has-text-weight-medium{font-weight:500!important}.has-text-weight-semibold{font-weight:600!important}.has-text-weight-bold{font-weight:700!important}.is-family-primary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-secondary{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-sans-serif{font-family:BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif!important}.is-family-monospace{font-family:monospace!important}.is-family-code{font-family:monospace!important}.is-block{display:block!important}@media screen and (max-width:768px){.is-block-mobile{display:block!important}}@media screen and (min-width:769px),print{.is-block-tablet{display:block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-block-tablet-only{display:block!important}}@media screen and (max-width:1023px){.is-block-touch{display:block!important}}@media screen and (min-width:1024px){.is-block-desktop{display:block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-block-desktop-only{display:block!important}}@media screen and (min-width:1216px){.is-block-widescreen{display:block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-block-widescreen-only{display:block!important}}@media screen and (min-width:1408px){.is-block-fullhd{display:block!important}}.is-flex{display:flex!important}@media screen and (max-width:768px){.is-flex-mobile{display:flex!important}}@media screen and (min-width:769px),print{.is-flex-tablet{display:flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-flex-tablet-only{display:flex!important}}@media screen and (max-width:1023px){.is-flex-touch{display:flex!important}}@media screen and (min-width:1024px){.is-flex-desktop{display:flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-flex-desktop-only{display:flex!important}}@media screen and (min-width:1216px){.is-flex-widescreen{display:flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-flex-widescreen-only{display:flex!important}}@media screen and (min-width:1408px){.is-flex-fullhd{display:flex!important}}.is-inline{display:inline!important}@media screen and (max-width:768px){.is-inline-mobile{display:inline!important}}@media screen and (min-width:769px),print{.is-inline-tablet{display:inline!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-tablet-only{display:inline!important}}@media screen and (max-width:1023px){.is-inline-touch{display:inline!important}}@media screen and (min-width:1024px){.is-inline-desktop{display:inline!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-desktop-only{display:inline!important}}@media screen and (min-width:1216px){.is-inline-widescreen{display:inline!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-widescreen-only{display:inline!important}}@media screen and (min-width:1408px){.is-inline-fullhd{display:inline!important}}.is-inline-block{display:inline-block!important}@media screen and (max-width:768px){.is-inline-block-mobile{display:inline-block!important}}@media screen and (min-width:769px),print{.is-inline-block-tablet{display:inline-block!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-block-tablet-only{display:inline-block!important}}@media screen and (max-width:1023px){.is-inline-block-touch{display:inline-block!important}}@media screen and (min-width:1024px){.is-inline-block-desktop{display:inline-block!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-block-desktop-only{display:inline-block!important}}@media screen and (min-width:1216px){.is-inline-block-widescreen{display:inline-block!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-block-widescreen-only{display:inline-block!important}}@media screen and (min-width:1408px){.is-inline-block-fullhd{display:inline-block!important}}.is-inline-flex{display:inline-flex!important}@media screen and (max-width:768px){.is-inline-flex-mobile{display:inline-flex!important}}@media screen and (min-width:769px),print{.is-inline-flex-tablet{display:inline-flex!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-inline-flex-tablet-only{display:inline-flex!important}}@media screen and (max-width:1023px){.is-inline-flex-touch{display:inline-flex!important}}@media screen and (min-width:1024px){.is-inline-flex-desktop{display:inline-flex!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-inline-flex-desktop-only{display:inline-flex!important}}@media screen and (min-width:1216px){.is-inline-flex-widescreen{display:inline-flex!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-inline-flex-widescreen-only{display:inline-flex!important}}@media screen and (min-width:1408px){.is-inline-flex-fullhd{display:inline-flex!important}}.is-hidden{display:none!important}.is-sr-only{border:none!important;clip:rect(0,0,0,0)!important;height:.01em!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:.01em!important}@media screen and (max-width:768px){.is-hidden-mobile{display:none!important}}@media screen and (min-width:769px),print{.is-hidden-tablet{display:none!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-hidden-tablet-only{display:none!important}}@media screen and (max-width:1023px){.is-hidden-touch{display:none!important}}@media screen and (min-width:1024px){.is-hidden-desktop{display:none!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-hidden-desktop-only{display:none!important}}@media screen and (min-width:1216px){.is-hidden-widescreen{display:none!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-hidden-widescreen-only{display:none!important}}@media screen and (min-width:1408px){.is-hidden-fullhd{display:none!important}}.is-invisible{visibility:hidden!important}@media screen and (max-width:768px){.is-invisible-mobile{visibility:hidden!important}}@media screen and (min-width:769px),print{.is-invisible-tablet{visibility:hidden!important}}@media screen and (min-width:769px) and (max-width:1023px){.is-invisible-tablet-only{visibility:hidden!important}}@media screen and (max-width:1023px){.is-invisible-touch{visibility:hidden!important}}@media screen and (min-width:1024px){.is-invisible-desktop{visibility:hidden!important}}@media screen and (min-width:1024px) and (max-width:1215px){.is-invisible-desktop-only{visibility:hidden!important}}@media screen and (min-width:1216px){.is-invisible-widescreen{visibility:hidden!important}}@media screen and (min-width:1216px) and (max-width:1407px){.is-invisible-widescreen-only{visibility:hidden!important}}@media screen and (min-width:1408px){.is-invisible-fullhd{visibility:hidden!important}}.is-marginless{margin:0!important}.is-paddingless{padding:0!important}.is-radiusless{border-radius:0!important}.is-shadowless{box-shadow:none!important}.is-relative{position:relative!important}.box{background-color:#fff;border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;display:block;padding:1.25rem}a.box:focus,a.box:hover{box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px #3273dc}a.box:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2),0 0 0 1px #3273dc}.button{background-color:#fff;border-color:#dbdbdb;border-width:1px;color:#363636;cursor:pointer;justify-content:center;padding-bottom:calc(.5em - 1px);padding-left:1em;padding-right:1em;padding-top:calc(.5em - 1px);text-align:center;white-space:nowrap}.button strong{color:inherit}.button .icon,.button .icon.is-large,.button .icon.is-medium,.button .icon.is-small{height:1.5em;width:1.5em}.button .icon:first-child:not(:last-child){margin-left:calc(-.5em - 1px);margin-right:.25em}.button .icon:last-child:not(:first-child){margin-left:.25em;margin-right:calc(-.5em - 1px)}.button .icon:first-child:last-child{margin-left:calc(-.5em - 1px);margin-right:calc(-.5em - 1px)}.button.is-hovered,.button:hover{border-color:#b5b5b5;color:#363636}.button.is-focused,.button:focus{border-color:#3273dc;color:#363636}.button.is-focused:not(:active),.button:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-active,.button:active{border-color:#4a4a4a;color:#363636}.button.is-text{background-color:transparent;border-color:transparent;color:#4a4a4a;text-decoration:underline}.button.is-text.is-focused,.button.is-text.is-hovered,.button.is-text:focus,.button.is-text:hover{background-color:#f5f5f5;color:#363636}.button.is-text.is-active,.button.is-text:active{background-color:#e8e8e8;color:#363636}.button.is-text[disabled],fieldset[disabled] .button.is-text{background-color:transparent;border-color:transparent;box-shadow:none}.button.is-white{background-color:#fff;border-color:transparent;color:#0a0a0a}.button.is-white.is-hovered,.button.is-white:hover{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.button.is-white.is-focused,.button.is-white:focus{border-color:transparent;color:#0a0a0a}.button.is-white.is-focused:not(:active),.button.is-white:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.button.is-white.is-active,.button.is-white:active{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.button.is-white[disabled],fieldset[disabled] .button.is-white{background-color:#fff;border-color:transparent;box-shadow:none}.button.is-white.is-inverted{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-hovered,.button.is-white.is-inverted:hover{background-color:#000}.button.is-white.is-inverted[disabled],fieldset[disabled] .button.is-white.is-inverted{background-color:#0a0a0a;border-color:transparent;box-shadow:none;color:#fff}.button.is-white.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-white.is-outlined.is-focused,.button.is-white.is-outlined.is-hovered,.button.is-white.is-outlined:focus,.button.is-white.is-outlined:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.button.is-white.is-outlined.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-outlined.is-loading.is-focused::after,.button.is-white.is-outlined.is-loading.is-hovered::after,.button.is-white.is-outlined.is-loading:focus::after,.button.is-white.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-white.is-outlined[disabled],fieldset[disabled] .button.is-white.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-white.is-inverted.is-outlined.is-focused,.button.is-white.is-inverted.is-outlined.is-hovered,.button.is-white.is-inverted.is-outlined:focus,.button.is-white.is-inverted.is-outlined:hover{background-color:#0a0a0a;color:#fff}.button.is-white.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-white.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-white.is-inverted.is-outlined.is-loading:focus::after,.button.is-white.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-white.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-white.is-inverted.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black{background-color:#0a0a0a;border-color:transparent;color:#fff}.button.is-black.is-hovered,.button.is-black:hover{background-color:#040404;border-color:transparent;color:#fff}.button.is-black.is-focused,.button.is-black:focus{border-color:transparent;color:#fff}.button.is-black.is-focused:not(:active),.button.is-black:focus:not(:active){box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.button.is-black.is-active,.button.is-black:active{background-color:#000;border-color:transparent;color:#fff}.button.is-black[disabled],fieldset[disabled] .button.is-black{background-color:#0a0a0a;border-color:transparent;box-shadow:none}.button.is-black.is-inverted{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-hovered,.button.is-black.is-inverted:hover{background-color:#f2f2f2}.button.is-black.is-inverted[disabled],fieldset[disabled] .button.is-black.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#0a0a0a}.button.is-black.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;color:#0a0a0a}.button.is-black.is-outlined.is-focused,.button.is-black.is-outlined.is-hovered,.button.is-black.is-outlined:focus,.button.is-black.is-outlined:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.button.is-black.is-outlined.is-loading::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-outlined.is-loading.is-focused::after,.button.is-black.is-outlined.is-loading.is-hovered::after,.button.is-black.is-outlined.is-loading:focus::after,.button.is-black.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-black.is-outlined[disabled],fieldset[disabled] .button.is-black.is-outlined{background-color:transparent;border-color:#0a0a0a;box-shadow:none;color:#0a0a0a}.button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-black.is-inverted.is-outlined.is-focused,.button.is-black.is-inverted.is-outlined.is-hovered,.button.is-black.is-inverted.is-outlined:focus,.button.is-black.is-inverted.is-outlined:hover{background-color:#fff;color:#0a0a0a}.button.is-black.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-black.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-black.is-inverted.is-outlined.is-loading:focus::after,.button.is-black.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #0a0a0a #0a0a0a!important}.button.is-black.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-black.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-light{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-hovered,.button.is-light:hover{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused,.button.is-light:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light.is-focused:not(:active),.button.is-light:focus:not(:active){box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.button.is-light.is-active,.button.is-light:active{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-light[disabled],fieldset[disabled] .button.is-light{background-color:#f5f5f5;border-color:transparent;box-shadow:none}.button.is-light.is-inverted{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-hovered,.button.is-light.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-light.is-inverted[disabled],fieldset[disabled] .button.is-light.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#f5f5f5}.button.is-light.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;color:#f5f5f5}.button.is-light.is-outlined.is-focused,.button.is-light.is-outlined.is-hovered,.button.is-light.is-outlined:focus,.button.is-light.is-outlined:hover{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.button.is-light.is-outlined.is-loading::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-outlined.is-loading.is-focused::after,.button.is-light.is-outlined.is-loading.is-hovered::after,.button.is-light.is-outlined.is-loading:focus::after,.button.is-light.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-light.is-outlined[disabled],fieldset[disabled] .button.is-light.is-outlined{background-color:transparent;border-color:#f5f5f5;box-shadow:none;color:#f5f5f5}.button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-light.is-inverted.is-outlined.is-focused,.button.is-light.is-inverted.is-outlined.is-hovered,.button.is-light.is-inverted.is-outlined:focus,.button.is-light.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#f5f5f5}.button.is-light.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-light.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-light.is-inverted.is-outlined.is-loading:focus::after,.button.is-light.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f5f5f5 #f5f5f5!important}.button.is-light.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-light.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-dark{background-color:#363636;border-color:transparent;color:#fff}.button.is-dark.is-hovered,.button.is-dark:hover{background-color:#2f2f2f;border-color:transparent;color:#fff}.button.is-dark.is-focused,.button.is-dark:focus{border-color:transparent;color:#fff}.button.is-dark.is-focused:not(:active),.button.is-dark:focus:not(:active){box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.button.is-dark.is-active,.button.is-dark:active{background-color:#292929;border-color:transparent;color:#fff}.button.is-dark[disabled],fieldset[disabled] .button.is-dark{background-color:#363636;border-color:transparent;box-shadow:none}.button.is-dark.is-inverted{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-hovered,.button.is-dark.is-inverted:hover{background-color:#f2f2f2}.button.is-dark.is-inverted[disabled],fieldset[disabled] .button.is-dark.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#363636}.button.is-dark.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined{background-color:transparent;border-color:#363636;color:#363636}.button.is-dark.is-outlined.is-focused,.button.is-dark.is-outlined.is-hovered,.button.is-dark.is-outlined:focus,.button.is-dark.is-outlined:hover{background-color:#363636;border-color:#363636;color:#fff}.button.is-dark.is-outlined.is-loading::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-outlined.is-loading.is-focused::after,.button.is-dark.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-outlined.is-loading:focus::after,.button.is-dark.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-dark.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-outlined{background-color:transparent;border-color:#363636;box-shadow:none;color:#363636}.button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-dark.is-inverted.is-outlined.is-focused,.button.is-dark.is-inverted.is-outlined.is-hovered,.button.is-dark.is-inverted.is-outlined:focus,.button.is-dark.is-inverted.is-outlined:hover{background-color:#fff;color:#363636}.button.is-dark.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-dark.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-dark.is-inverted.is-outlined.is-loading:focus::after,.button.is-dark.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #363636 #363636!important}.button.is-dark.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-dark.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary{background-color:#00d1b2;border-color:transparent;color:#fff}.button.is-primary.is-hovered,.button.is-primary:hover{background-color:#00c4a7;border-color:transparent;color:#fff}.button.is-primary.is-focused,.button.is-primary:focus{border-color:transparent;color:#fff}.button.is-primary.is-focused:not(:active),.button.is-primary:focus:not(:active){box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.button.is-primary.is-active,.button.is-primary:active{background-color:#00b89c;border-color:transparent;color:#fff}.button.is-primary[disabled],fieldset[disabled] .button.is-primary{background-color:#00d1b2;border-color:transparent;box-shadow:none}.button.is-primary.is-inverted{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-hovered,.button.is-primary.is-inverted:hover{background-color:#f2f2f2}.button.is-primary.is-inverted[disabled],fieldset[disabled] .button.is-primary.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#00d1b2}.button.is-primary.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;color:#00d1b2}.button.is-primary.is-outlined.is-focused,.button.is-primary.is-outlined.is-hovered,.button.is-primary.is-outlined:focus,.button.is-primary.is-outlined:hover{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.button.is-primary.is-outlined.is-loading::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-outlined.is-loading.is-focused::after,.button.is-primary.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-outlined.is-loading:focus::after,.button.is-primary.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-primary.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-outlined{background-color:transparent;border-color:#00d1b2;box-shadow:none;color:#00d1b2}.button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-primary.is-inverted.is-outlined.is-focused,.button.is-primary.is-inverted.is-outlined.is-hovered,.button.is-primary.is-inverted.is-outlined:focus,.button.is-primary.is-inverted.is-outlined:hover{background-color:#fff;color:#00d1b2}.button.is-primary.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-primary.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-primary.is-inverted.is-outlined.is-loading:focus::after,.button.is-primary.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #00d1b2 #00d1b2!important}.button.is-primary.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-primary.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-primary.is-light{background-color:#ebfffc;color:#00947e}.button.is-primary.is-light.is-hovered,.button.is-primary.is-light:hover{background-color:#defffa;border-color:transparent;color:#00947e}.button.is-primary.is-light.is-active,.button.is-primary.is-light:active{background-color:#d1fff8;border-color:transparent;color:#00947e}.button.is-link{background-color:#3273dc;border-color:transparent;color:#fff}.button.is-link.is-hovered,.button.is-link:hover{background-color:#276cda;border-color:transparent;color:#fff}.button.is-link.is-focused,.button.is-link:focus{border-color:transparent;color:#fff}.button.is-link.is-focused:not(:active),.button.is-link:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.button.is-link.is-active,.button.is-link:active{background-color:#2366d1;border-color:transparent;color:#fff}.button.is-link[disabled],fieldset[disabled] .button.is-link{background-color:#3273dc;border-color:transparent;box-shadow:none}.button.is-link.is-inverted{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-hovered,.button.is-link.is-inverted:hover{background-color:#f2f2f2}.button.is-link.is-inverted[disabled],fieldset[disabled] .button.is-link.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3273dc}.button.is-link.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;color:#3273dc}.button.is-link.is-outlined.is-focused,.button.is-link.is-outlined.is-hovered,.button.is-link.is-outlined:focus,.button.is-link.is-outlined:hover{background-color:#3273dc;border-color:#3273dc;color:#fff}.button.is-link.is-outlined.is-loading::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-outlined.is-loading.is-focused::after,.button.is-link.is-outlined.is-loading.is-hovered::after,.button.is-link.is-outlined.is-loading:focus::after,.button.is-link.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-link.is-outlined[disabled],fieldset[disabled] .button.is-link.is-outlined{background-color:transparent;border-color:#3273dc;box-shadow:none;color:#3273dc}.button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-link.is-inverted.is-outlined.is-focused,.button.is-link.is-inverted.is-outlined.is-hovered,.button.is-link.is-inverted.is-outlined:focus,.button.is-link.is-inverted.is-outlined:hover{background-color:#fff;color:#3273dc}.button.is-link.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-link.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-link.is-inverted.is-outlined.is-loading:focus::after,.button.is-link.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3273dc #3273dc!important}.button.is-link.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-link.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-link.is-light{background-color:#eef3fc;color:#2160c4}.button.is-link.is-light.is-hovered,.button.is-link.is-light:hover{background-color:#e3ecfa;border-color:transparent;color:#2160c4}.button.is-link.is-light.is-active,.button.is-link.is-light:active{background-color:#d8e4f8;border-color:transparent;color:#2160c4}.button.is-info{background-color:#3298dc;border-color:transparent;color:#fff}.button.is-info.is-hovered,.button.is-info:hover{background-color:#2793da;border-color:transparent;color:#fff}.button.is-info.is-focused,.button.is-info:focus{border-color:transparent;color:#fff}.button.is-info.is-focused:not(:active),.button.is-info:focus:not(:active){box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.button.is-info.is-active,.button.is-info:active{background-color:#238cd1;border-color:transparent;color:#fff}.button.is-info[disabled],fieldset[disabled] .button.is-info{background-color:#3298dc;border-color:transparent;box-shadow:none}.button.is-info.is-inverted{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-hovered,.button.is-info.is-inverted:hover{background-color:#f2f2f2}.button.is-info.is-inverted[disabled],fieldset[disabled] .button.is-info.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#3298dc}.button.is-info.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;color:#3298dc}.button.is-info.is-outlined.is-focused,.button.is-info.is-outlined.is-hovered,.button.is-info.is-outlined:focus,.button.is-info.is-outlined:hover{background-color:#3298dc;border-color:#3298dc;color:#fff}.button.is-info.is-outlined.is-loading::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-outlined.is-loading.is-focused::after,.button.is-info.is-outlined.is-loading.is-hovered::after,.button.is-info.is-outlined.is-loading:focus::after,.button.is-info.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-info.is-outlined[disabled],fieldset[disabled] .button.is-info.is-outlined{background-color:transparent;border-color:#3298dc;box-shadow:none;color:#3298dc}.button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-info.is-inverted.is-outlined.is-focused,.button.is-info.is-inverted.is-outlined.is-hovered,.button.is-info.is-inverted.is-outlined:focus,.button.is-info.is-inverted.is-outlined:hover{background-color:#fff;color:#3298dc}.button.is-info.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-info.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-info.is-inverted.is-outlined.is-loading:focus::after,.button.is-info.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #3298dc #3298dc!important}.button.is-info.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-info.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-info.is-light{background-color:#eef6fc;color:#1d72aa}.button.is-info.is-light.is-hovered,.button.is-info.is-light:hover{background-color:#e3f1fa;border-color:transparent;color:#1d72aa}.button.is-info.is-light.is-active,.button.is-info.is-light:active{background-color:#d8ebf8;border-color:transparent;color:#1d72aa}.button.is-success{background-color:#48c774;border-color:transparent;color:#fff}.button.is-success.is-hovered,.button.is-success:hover{background-color:#3ec46d;border-color:transparent;color:#fff}.button.is-success.is-focused,.button.is-success:focus{border-color:transparent;color:#fff}.button.is-success.is-focused:not(:active),.button.is-success:focus:not(:active){box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.button.is-success.is-active,.button.is-success:active{background-color:#3abb67;border-color:transparent;color:#fff}.button.is-success[disabled],fieldset[disabled] .button.is-success{background-color:#48c774;border-color:transparent;box-shadow:none}.button.is-success.is-inverted{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-hovered,.button.is-success.is-inverted:hover{background-color:#f2f2f2}.button.is-success.is-inverted[disabled],fieldset[disabled] .button.is-success.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#48c774}.button.is-success.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined{background-color:transparent;border-color:#48c774;color:#48c774}.button.is-success.is-outlined.is-focused,.button.is-success.is-outlined.is-hovered,.button.is-success.is-outlined:focus,.button.is-success.is-outlined:hover{background-color:#48c774;border-color:#48c774;color:#fff}.button.is-success.is-outlined.is-loading::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-outlined.is-loading.is-focused::after,.button.is-success.is-outlined.is-loading.is-hovered::after,.button.is-success.is-outlined.is-loading:focus::after,.button.is-success.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-success.is-outlined[disabled],fieldset[disabled] .button.is-success.is-outlined{background-color:transparent;border-color:#48c774;box-shadow:none;color:#48c774}.button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-success.is-inverted.is-outlined.is-focused,.button.is-success.is-inverted.is-outlined.is-hovered,.button.is-success.is-inverted.is-outlined:focus,.button.is-success.is-inverted.is-outlined:hover{background-color:#fff;color:#48c774}.button.is-success.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-success.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-success.is-inverted.is-outlined.is-loading:focus::after,.button.is-success.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #48c774 #48c774!important}.button.is-success.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-success.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-success.is-light{background-color:#effaf3;color:#257942}.button.is-success.is-light.is-hovered,.button.is-success.is-light:hover{background-color:#e6f7ec;border-color:transparent;color:#257942}.button.is-success.is-light.is-active,.button.is-success.is-light:active{background-color:#dcf4e4;border-color:transparent;color:#257942}.button.is-warning{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-hovered,.button.is-warning:hover{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused,.button.is-warning:focus{border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning.is-focused:not(:active),.button.is-warning:focus:not(:active){box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.button.is-warning.is-active,.button.is-warning:active{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.button.is-warning[disabled],fieldset[disabled] .button.is-warning{background-color:#ffdd57;border-color:transparent;box-shadow:none}.button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-hovered,.button.is-warning.is-inverted:hover{background-color:rgba(0,0,0,.7)}.button.is-warning.is-inverted[disabled],fieldset[disabled] .button.is-warning.is-inverted{background-color:rgba(0,0,0,.7);border-color:transparent;box-shadow:none;color:#ffdd57}.button.is-warning.is-loading::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;color:#ffdd57}.button.is-warning.is-outlined.is-focused,.button.is-warning.is-outlined.is-hovered,.button.is-warning.is-outlined:focus,.button.is-warning.is-outlined:hover{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.button.is-warning.is-outlined.is-loading::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-outlined.is-loading.is-focused::after,.button.is-warning.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-outlined.is-loading:focus::after,.button.is-warning.is-outlined.is-loading:hover::after{border-color:transparent transparent rgba(0,0,0,.7) rgba(0,0,0,.7)!important}.button.is-warning.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-outlined{background-color:transparent;border-color:#ffdd57;box-shadow:none;color:#ffdd57}.button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);color:rgba(0,0,0,.7)}.button.is-warning.is-inverted.is-outlined.is-focused,.button.is-warning.is-inverted.is-outlined.is-hovered,.button.is-warning.is-inverted.is-outlined:focus,.button.is-warning.is-inverted.is-outlined:hover{background-color:rgba(0,0,0,.7);color:#ffdd57}.button.is-warning.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-warning.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-warning.is-inverted.is-outlined.is-loading:focus::after,.button.is-warning.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #ffdd57 #ffdd57!important}.button.is-warning.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-warning.is-inverted.is-outlined{background-color:transparent;border-color:rgba(0,0,0,.7);box-shadow:none;color:rgba(0,0,0,.7)}.button.is-warning.is-light{background-color:#fffbeb;color:#947600}.button.is-warning.is-light.is-hovered,.button.is-warning.is-light:hover{background-color:#fff8de;border-color:transparent;color:#947600}.button.is-warning.is-light.is-active,.button.is-warning.is-light:active{background-color:#fff6d1;border-color:transparent;color:#947600}.button.is-danger{background-color:#f14668;border-color:transparent;color:#fff}.button.is-danger.is-hovered,.button.is-danger:hover{background-color:#f03a5f;border-color:transparent;color:#fff}.button.is-danger.is-focused,.button.is-danger:focus{border-color:transparent;color:#fff}.button.is-danger.is-focused:not(:active),.button.is-danger:focus:not(:active){box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.button.is-danger.is-active,.button.is-danger:active{background-color:#ef2e55;border-color:transparent;color:#fff}.button.is-danger[disabled],fieldset[disabled] .button.is-danger{background-color:#f14668;border-color:transparent;box-shadow:none}.button.is-danger.is-inverted{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-hovered,.button.is-danger.is-inverted:hover{background-color:#f2f2f2}.button.is-danger.is-inverted[disabled],fieldset[disabled] .button.is-danger.is-inverted{background-color:#fff;border-color:transparent;box-shadow:none;color:#f14668}.button.is-danger.is-loading::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;color:#f14668}.button.is-danger.is-outlined.is-focused,.button.is-danger.is-outlined.is-hovered,.button.is-danger.is-outlined:focus,.button.is-danger.is-outlined:hover{background-color:#f14668;border-color:#f14668;color:#fff}.button.is-danger.is-outlined.is-loading::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-outlined.is-loading.is-focused::after,.button.is-danger.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-outlined.is-loading:focus::after,.button.is-danger.is-outlined.is-loading:hover::after{border-color:transparent transparent #fff #fff!important}.button.is-danger.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-outlined{background-color:transparent;border-color:#f14668;box-shadow:none;color:#f14668}.button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;color:#fff}.button.is-danger.is-inverted.is-outlined.is-focused,.button.is-danger.is-inverted.is-outlined.is-hovered,.button.is-danger.is-inverted.is-outlined:focus,.button.is-danger.is-inverted.is-outlined:hover{background-color:#fff;color:#f14668}.button.is-danger.is-inverted.is-outlined.is-loading.is-focused::after,.button.is-danger.is-inverted.is-outlined.is-loading.is-hovered::after,.button.is-danger.is-inverted.is-outlined.is-loading:focus::after,.button.is-danger.is-inverted.is-outlined.is-loading:hover::after{border-color:transparent transparent #f14668 #f14668!important}.button.is-danger.is-inverted.is-outlined[disabled],fieldset[disabled] .button.is-danger.is-inverted.is-outlined{background-color:transparent;border-color:#fff;box-shadow:none;color:#fff}.button.is-danger.is-light{background-color:#feecf0;color:#cc0f35}.button.is-danger.is-light.is-hovered,.button.is-danger.is-light:hover{background-color:#fde0e6;border-color:transparent;color:#cc0f35}.button.is-danger.is-light.is-active,.button.is-danger.is-light:active{background-color:#fcd4dc;border-color:transparent;color:#cc0f35}.button.is-small{border-radius:2px;font-size:.75rem}.button.is-normal{font-size:1rem}.button.is-medium{font-size:1.25rem}.button.is-large{font-size:1.5rem}.button[disabled],fieldset[disabled] .button{background-color:#fff;border-color:#dbdbdb;box-shadow:none;opacity:.5}.button.is-fullwidth{display:flex;width:100%}.button.is-loading{color:transparent!important;pointer-events:none}.button.is-loading::after{position:absolute;left:calc(50% - (1em / 2));top:calc(50% - (1em / 2));position:absolute!important}.button.is-static{background-color:#f5f5f5;border-color:#dbdbdb;color:#7a7a7a;box-shadow:none;pointer-events:none}.button.is-rounded{border-radius:290486px;padding-left:calc(1em + .25em);padding-right:calc(1em + .25em)}.buttons{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.buttons .button{margin-bottom:.5rem}.buttons .button:not(:last-child):not(.is-fullwidth){margin-right:.5rem}.buttons:last-child{margin-bottom:-.5rem}.buttons:not(:last-child){margin-bottom:1rem}.buttons.are-small .button:not(.is-normal):not(.is-medium):not(.is-large){border-radius:2px;font-size:.75rem}.buttons.are-medium .button:not(.is-small):not(.is-normal):not(.is-large){font-size:1.25rem}.buttons.are-large .button:not(.is-small):not(.is-normal):not(.is-medium){font-size:1.5rem}.buttons.has-addons .button:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.buttons.has-addons .button:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0;margin-right:-1px}.buttons.has-addons .button:last-child{margin-right:0}.buttons.has-addons .button.is-hovered,.buttons.has-addons .button:hover{z-index:2}.buttons.has-addons .button.is-active,.buttons.has-addons .button.is-focused,.buttons.has-addons .button.is-selected,.buttons.has-addons .button:active,.buttons.has-addons .button:focus{z-index:3}.buttons.has-addons .button.is-active:hover,.buttons.has-addons .button.is-focused:hover,.buttons.has-addons .button.is-selected:hover,.buttons.has-addons .button:active:hover,.buttons.has-addons .button:focus:hover{z-index:4}.buttons.has-addons .button.is-expanded{flex-grow:1;flex-shrink:1}.buttons.is-centered{justify-content:center}.buttons.is-centered:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.buttons.is-right{justify-content:flex-end}.buttons.is-right:not(.has-addons) .button:not(.is-fullwidth){margin-left:.25rem;margin-right:.25rem}.container{flex-grow:1;margin:0 auto;position:relative;width:auto}.container.is-fluid{max-width:none;padding-left:32px;padding-right:32px;width:100%}@media screen and (min-width:1024px){.container{max-width:960px}}@media screen and (max-width:1215px){.container.is-widescreen{max-width:1152px}}@media screen and (max-width:1407px){.container.is-fullhd{max-width:1344px}}@media screen and (min-width:1216px){.container{max-width:1152px}}@media screen and (min-width:1408px){.container{max-width:1344px}}.content li+li{margin-top:.25em}.content blockquote:not(:last-child),.content dl:not(:last-child),.content ol:not(:last-child),.content p:not(:last-child),.content pre:not(:last-child),.content table:not(:last-child),.content ul:not(:last-child){margin-bottom:1em}.content h1,.content h2,.content h3,.content h4,.content h5,.content h6{color:#363636;font-weight:600;line-height:1.125}.content h1{font-size:2em;margin-bottom:.5em}.content h1:not(:first-child){margin-top:1em}.content h2{font-size:1.75em;margin-bottom:.5714em}.content h2:not(:first-child){margin-top:1.1428em}.content h3{font-size:1.5em;margin-bottom:.6666em}.content h3:not(:first-child){margin-top:1.3333em}.content h4{font-size:1.25em;margin-bottom:.8em}.content h5{font-size:1.125em;margin-bottom:.8888em}.content h6{font-size:1em;margin-bottom:1em}.content blockquote{background-color:#f5f5f5;border-left:5px solid #dbdbdb;padding:1.25em 1.5em}.content ol{list-style-position:outside;margin-left:2em;margin-top:1em}.content ol:not([type]){list-style-type:decimal}.content ol:not([type]).is-lower-alpha{list-style-type:lower-alpha}.content ol:not([type]).is-lower-roman{list-style-type:lower-roman}.content ol:not([type]).is-upper-alpha{list-style-type:upper-alpha}.content ol:not([type]).is-upper-roman{list-style-type:upper-roman}.content ul{list-style:disc outside;margin-left:2em;margin-top:1em}.content ul ul{list-style-type:circle;margin-top:.5em}.content ul ul ul{list-style-type:square}.content dd{margin-left:2em}.content figure{margin-left:2em;margin-right:2em;text-align:center}.content figure:not(:first-child){margin-top:2em}.content figure:not(:last-child){margin-bottom:2em}.content figure img{display:inline-block}.content figure figcaption{font-style:italic}.content pre{-webkit-overflow-scrolling:touch;overflow-x:auto;padding:1.25em 1.5em;white-space:pre;word-wrap:normal}.content sub,.content sup{font-size:75%}.content table{width:100%}.content table td,.content table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.content table th{color:#363636}.content table th:not([align]){text-align:left}.content table thead td,.content table thead th{border-width:0 0 2px;color:#363636}.content table tfoot td,.content table tfoot th{border-width:2px 0 0;color:#363636}.content table tbody tr:last-child td,.content table tbody tr:last-child th{border-bottom-width:0}.content .tabs li+li{margin-top:0}.content.is-small{font-size:.75rem}.content.is-medium{font-size:1.25rem}.content.is-large{font-size:1.5rem}.icon{align-items:center;display:inline-flex;justify-content:center;height:1.5rem;width:1.5rem}.icon.is-small{height:1rem;width:1rem}.icon.is-medium{height:2rem;width:2rem}.icon.is-large{height:3rem;width:3rem}.image{display:block;position:relative}.image img{display:block;height:auto;width:100%}.image img.is-rounded{border-radius:290486px}.image.is-fullwidth{width:100%}.image.is-16by9 .has-ratio,.image.is-16by9 img,.image.is-1by1 .has-ratio,.image.is-1by1 img,.image.is-1by2 .has-ratio,.image.is-1by2 img,.image.is-1by3 .has-ratio,.image.is-1by3 img,.image.is-2by1 .has-ratio,.image.is-2by1 img,.image.is-2by3 .has-ratio,.image.is-2by3 img,.image.is-3by1 .has-ratio,.image.is-3by1 img,.image.is-3by2 .has-ratio,.image.is-3by2 img,.image.is-3by4 .has-ratio,.image.is-3by4 img,.image.is-3by5 .has-ratio,.image.is-3by5 img,.image.is-4by3 .has-ratio,.image.is-4by3 img,.image.is-4by5 .has-ratio,.image.is-4by5 img,.image.is-5by3 .has-ratio,.image.is-5by3 img,.image.is-5by4 .has-ratio,.image.is-5by4 img,.image.is-9by16 .has-ratio,.image.is-9by16 img,.image.is-square .has-ratio,.image.is-square img{height:100%;width:100%}.image.is-1by1,.image.is-square{padding-top:100%}.image.is-5by4{padding-top:80%}.image.is-4by3{padding-top:75%}.image.is-3by2{padding-top:66.6666%}.image.is-5by3{padding-top:60%}.image.is-16by9{padding-top:56.25%}.image.is-2by1{padding-top:50%}.image.is-3by1{padding-top:33.3333%}.image.is-4by5{padding-top:125%}.image.is-3by4{padding-top:133.3333%}.image.is-2by3{padding-top:150%}.image.is-3by5{padding-top:166.6666%}.image.is-9by16{padding-top:177.7777%}.image.is-1by2{padding-top:200%}.image.is-1by3{padding-top:300%}.image.is-16x16{height:16px;width:16px}.image.is-24x24{height:24px;width:24px}.image.is-32x32{height:32px;width:32px}.image.is-48x48{height:48px;width:48px}.image.is-64x64{height:64px;width:64px}.image.is-96x96{height:96px;width:96px}.image.is-128x128{height:128px;width:128px}.notification{background-color:#f5f5f5;border-radius:4px;padding:1.25rem 2.5rem 1.25rem 1.5rem;position:relative}.notification a:not(.button):not(.dropdown-item){color:currentColor;text-decoration:underline}.notification strong{color:currentColor}.notification code,.notification pre{background:#fff}.notification pre code{background:0 0}.notification>.delete{position:absolute;right:.5rem;top:.5rem}.notification .content,.notification .subtitle,.notification .title{color:currentColor}.notification.is-white{background-color:#fff;color:#0a0a0a}.notification.is-black{background-color:#0a0a0a;color:#fff}.notification.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.notification.is-dark{background-color:#363636;color:#fff}.notification.is-primary{background-color:#00d1b2;color:#fff}.notification.is-link{background-color:#3273dc;color:#fff}.notification.is-info{background-color:#3298dc;color:#fff}.notification.is-success{background-color:#48c774;color:#fff}.notification.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.notification.is-danger{background-color:#f14668;color:#fff}.progress{-moz-appearance:none;-webkit-appearance:none;border:none;border-radius:290486px;display:block;height:1rem;overflow:hidden;padding:0;width:100%}.progress::-webkit-progress-bar{background-color:#ededed}.progress::-webkit-progress-value{background-color:#4a4a4a}.progress::-moz-progress-bar{background-color:#4a4a4a}.progress::-ms-fill{background-color:#4a4a4a;border:none}.progress.is-white::-webkit-progress-value{background-color:#fff}.progress.is-white::-moz-progress-bar{background-color:#fff}.progress.is-white::-ms-fill{background-color:#fff}.progress.is-white:indeterminate{background-image:linear-gradient(to right,#fff 30%,#ededed 30%)}.progress.is-black::-webkit-progress-value{background-color:#0a0a0a}.progress.is-black::-moz-progress-bar{background-color:#0a0a0a}.progress.is-black::-ms-fill{background-color:#0a0a0a}.progress.is-black:indeterminate{background-image:linear-gradient(to right,#0a0a0a 30%,#ededed 30%)}.progress.is-light::-webkit-progress-value{background-color:#f5f5f5}.progress.is-light::-moz-progress-bar{background-color:#f5f5f5}.progress.is-light::-ms-fill{background-color:#f5f5f5}.progress.is-light:indeterminate{background-image:linear-gradient(to right,#f5f5f5 30%,#ededed 30%)}.progress.is-dark::-webkit-progress-value{background-color:#363636}.progress.is-dark::-moz-progress-bar{background-color:#363636}.progress.is-dark::-ms-fill{background-color:#363636}.progress.is-dark:indeterminate{background-image:linear-gradient(to right,#363636 30%,#ededed 30%)}.progress.is-primary::-webkit-progress-value{background-color:#00d1b2}.progress.is-primary::-moz-progress-bar{background-color:#00d1b2}.progress.is-primary::-ms-fill{background-color:#00d1b2}.progress.is-primary:indeterminate{background-image:linear-gradient(to right,#00d1b2 30%,#ededed 30%)}.progress.is-link::-webkit-progress-value{background-color:#3273dc}.progress.is-link::-moz-progress-bar{background-color:#3273dc}.progress.is-link::-ms-fill{background-color:#3273dc}.progress.is-link:indeterminate{background-image:linear-gradient(to right,#3273dc 30%,#ededed 30%)}.progress.is-info::-webkit-progress-value{background-color:#3298dc}.progress.is-info::-moz-progress-bar{background-color:#3298dc}.progress.is-info::-ms-fill{background-color:#3298dc}.progress.is-info:indeterminate{background-image:linear-gradient(to right,#3298dc 30%,#ededed 30%)}.progress.is-success::-webkit-progress-value{background-color:#48c774}.progress.is-success::-moz-progress-bar{background-color:#48c774}.progress.is-success::-ms-fill{background-color:#48c774}.progress.is-success:indeterminate{background-image:linear-gradient(to right,#48c774 30%,#ededed 30%)}.progress.is-warning::-webkit-progress-value{background-color:#ffdd57}.progress.is-warning::-moz-progress-bar{background-color:#ffdd57}.progress.is-warning::-ms-fill{background-color:#ffdd57}.progress.is-warning:indeterminate{background-image:linear-gradient(to right,#ffdd57 30%,#ededed 30%)}.progress.is-danger::-webkit-progress-value{background-color:#f14668}.progress.is-danger::-moz-progress-bar{background-color:#f14668}.progress.is-danger::-ms-fill{background-color:#f14668}.progress.is-danger:indeterminate{background-image:linear-gradient(to right,#f14668 30%,#ededed 30%)}.progress:indeterminate{-webkit-animation-duration:1.5s;animation-duration:1.5s;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;-webkit-animation-name:moveIndeterminate;animation-name:moveIndeterminate;-webkit-animation-timing-function:linear;animation-timing-function:linear;background-color:#ededed;background-image:linear-gradient(to right,#4a4a4a 30%,#ededed 30%);background-position:top left;background-repeat:no-repeat;background-size:150% 150%}.progress:indeterminate::-webkit-progress-bar{background-color:transparent}.progress:indeterminate::-moz-progress-bar{background-color:transparent}.progress.is-small{height:.75rem}.progress.is-medium{height:1.25rem}.progress.is-large{height:1.5rem}@-webkit-keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}@keyframes moveIndeterminate{from{background-position:200% 0}to{background-position:-200% 0}}.table{background-color:#fff;color:#363636}.table td,.table th{border:1px solid #dbdbdb;border-width:0 0 1px;padding:.5em .75em;vertical-align:top}.table td.is-white,.table th.is-white{background-color:#fff;border-color:#fff;color:#0a0a0a}.table td.is-black,.table th.is-black{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.table td.is-light,.table th.is-light{background-color:#f5f5f5;border-color:#f5f5f5;color:rgba(0,0,0,.7)}.table td.is-dark,.table th.is-dark{background-color:#363636;border-color:#363636;color:#fff}.table td.is-primary,.table th.is-primary{background-color:#00d1b2;border-color:#00d1b2;color:#fff}.table td.is-link,.table th.is-link{background-color:#3273dc;border-color:#3273dc;color:#fff}.table td.is-info,.table th.is-info{background-color:#3298dc;border-color:#3298dc;color:#fff}.table td.is-success,.table th.is-success{background-color:#48c774;border-color:#48c774;color:#fff}.table td.is-warning,.table th.is-warning{background-color:#ffdd57;border-color:#ffdd57;color:rgba(0,0,0,.7)}.table td.is-danger,.table th.is-danger{background-color:#f14668;border-color:#f14668;color:#fff}.table td.is-narrow,.table th.is-narrow{white-space:nowrap;width:1%}.table td.is-selected,.table th.is-selected{background-color:#00d1b2;color:#fff}.table td.is-selected a,.table td.is-selected strong,.table th.is-selected a,.table th.is-selected strong{color:currentColor}.table th{color:#363636}.table th:not([align]){text-align:left}.table tr.is-selected{background-color:#00d1b2;color:#fff}.table tr.is-selected a,.table tr.is-selected strong{color:currentColor}.table tr.is-selected td,.table tr.is-selected th{border-color:#fff;color:currentColor}.table thead{background-color:transparent}.table thead td,.table thead th{border-width:0 0 2px;color:#363636}.table tfoot{background-color:transparent}.table tfoot td,.table tfoot th{border-width:2px 0 0;color:#363636}.table tbody{background-color:transparent}.table tbody tr:last-child td,.table tbody tr:last-child th{border-bottom-width:0}.table.is-bordered td,.table.is-bordered th{border-width:1px}.table.is-bordered tr:last-child td,.table.is-bordered tr:last-child th{border-bottom-width:1px}.table.is-fullwidth{width:100%}.table.is-hoverable tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover{background-color:#fafafa}.table.is-hoverable.is-striped tbody tr:not(.is-selected):hover:nth-child(even){background-color:#f5f5f5}.table.is-narrow td,.table.is-narrow th{padding:.25em .5em}.table.is-striped tbody tr:not(.is-selected):nth-child(even){background-color:#fafafa}.table-container{-webkit-overflow-scrolling:touch;overflow:auto;overflow-y:hidden;max-width:100%}.tags{align-items:center;display:flex;flex-wrap:wrap;justify-content:flex-start}.tags .tag{margin-bottom:.5rem}.tags .tag:not(:last-child){margin-right:.5rem}.tags:last-child{margin-bottom:-.5rem}.tags:not(:last-child){margin-bottom:1rem}.tags.are-medium .tag:not(.is-normal):not(.is-large){font-size:1rem}.tags.are-large .tag:not(.is-normal):not(.is-medium){font-size:1.25rem}.tags.is-centered{justify-content:center}.tags.is-centered .tag{margin-right:.25rem;margin-left:.25rem}.tags.is-right{justify-content:flex-end}.tags.is-right .tag:not(:first-child){margin-left:.5rem}.tags.is-right .tag:not(:last-child){margin-right:0}.tags.has-addons .tag{margin-right:0}.tags.has-addons .tag:not(:first-child){margin-left:0;border-bottom-left-radius:0;border-top-left-radius:0}.tags.has-addons .tag:not(:last-child){border-bottom-right-radius:0;border-top-right-radius:0}.tag:not(body){align-items:center;background-color:#f5f5f5;border-radius:4px;color:#4a4a4a;display:inline-flex;font-size:.75rem;height:2em;justify-content:center;line-height:1.5;padding-left:.75em;padding-right:.75em;white-space:nowrap}.tag:not(body) .delete{margin-left:.25rem;margin-right:-.375rem}.tag:not(body).is-white{background-color:#fff;color:#0a0a0a}.tag:not(body).is-black{background-color:#0a0a0a;color:#fff}.tag:not(body).is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.tag:not(body).is-dark{background-color:#363636;color:#fff}.tag:not(body).is-primary{background-color:#00d1b2;color:#fff}.tag:not(body).is-primary.is-light{background-color:#ebfffc;color:#00947e}.tag:not(body).is-link{background-color:#3273dc;color:#fff}.tag:not(body).is-link.is-light{background-color:#eef3fc;color:#2160c4}.tag:not(body).is-info{background-color:#3298dc;color:#fff}.tag:not(body).is-info.is-light{background-color:#eef6fc;color:#1d72aa}.tag:not(body).is-success{background-color:#48c774;color:#fff}.tag:not(body).is-success.is-light{background-color:#effaf3;color:#257942}.tag:not(body).is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.tag:not(body).is-warning.is-light{background-color:#fffbeb;color:#947600}.tag:not(body).is-danger{background-color:#f14668;color:#fff}.tag:not(body).is-danger.is-light{background-color:#feecf0;color:#cc0f35}.tag:not(body).is-normal{font-size:.75rem}.tag:not(body).is-medium{font-size:1rem}.tag:not(body).is-large{font-size:1.25rem}.tag:not(body) .icon:first-child:not(:last-child){margin-left:-.375em;margin-right:.1875em}.tag:not(body) .icon:last-child:not(:first-child){margin-left:.1875em;margin-right:-.375em}.tag:not(body) .icon:first-child:last-child{margin-left:-.375em;margin-right:-.375em}.tag:not(body).is-delete{margin-left:1px;padding:0;position:relative;width:2em}.tag:not(body).is-delete::after,.tag:not(body).is-delete::before{background-color:currentColor;content:"";display:block;left:50%;position:absolute;top:50%;transform:translateX(-50%) translateY(-50%) rotate(45deg);transform-origin:center center}.tag:not(body).is-delete::before{height:1px;width:50%}.tag:not(body).is-delete::after{height:50%;width:1px}.tag:not(body).is-delete:focus,.tag:not(body).is-delete:hover{background-color:#e8e8e8}.tag:not(body).is-delete:active{background-color:#dbdbdb}.tag:not(body).is-rounded{border-radius:290486px}a.tag:hover{text-decoration:underline}.subtitle,.title{word-break:break-word}.subtitle em,.subtitle span,.title em,.title span{font-weight:inherit}.subtitle sub,.title sub{font-size:.75em}.subtitle sup,.title sup{font-size:.75em}.subtitle .tag,.title .tag{vertical-align:middle}.title{color:#363636;font-size:2rem;font-weight:600;line-height:1.125}.title strong{color:inherit;font-weight:inherit}.title+.highlight{margin-top:-.75rem}.title:not(.is-spaced)+.subtitle{margin-top:-1.25rem}.title.is-1{font-size:3rem}.title.is-2{font-size:2.5rem}.title.is-3{font-size:2rem}.title.is-4{font-size:1.5rem}.title.is-5{font-size:1.25rem}.title.is-6{font-size:1rem}.title.is-7{font-size:.75rem}.subtitle{color:#4a4a4a;font-size:1.25rem;font-weight:400;line-height:1.25}.subtitle strong{color:#363636;font-weight:600}.subtitle:not(.is-spaced)+.title{margin-top:-1.25rem}.subtitle.is-1{font-size:3rem}.subtitle.is-2{font-size:2.5rem}.subtitle.is-3{font-size:2rem}.subtitle.is-4{font-size:1.5rem}.subtitle.is-5{font-size:1.25rem}.subtitle.is-6{font-size:1rem}.subtitle.is-7{font-size:.75rem}.heading{display:block;font-size:11px;letter-spacing:1px;margin-bottom:5px;text-transform:uppercase}.highlight{font-weight:400;max-width:100%;overflow:hidden;padding:0}.highlight pre{overflow:auto;max-width:100%}.number{align-items:center;background-color:#f5f5f5;border-radius:290486px;display:inline-flex;font-size:1.25rem;height:2em;justify-content:center;margin-right:1.5rem;min-width:2.5em;padding:.25rem .5rem;text-align:center;vertical-align:top}.input,.select select,.textarea{background-color:#fff;border-color:#dbdbdb;border-radius:4px;color:#363636}.input::-moz-placeholder,.select select::-moz-placeholder,.textarea::-moz-placeholder{color:rgba(54,54,54,.3)}.input::-webkit-input-placeholder,.select select::-webkit-input-placeholder,.textarea::-webkit-input-placeholder{color:rgba(54,54,54,.3)}.input:-moz-placeholder,.select select:-moz-placeholder,.textarea:-moz-placeholder{color:rgba(54,54,54,.3)}.input:-ms-input-placeholder,.select select:-ms-input-placeholder,.textarea:-ms-input-placeholder{color:rgba(54,54,54,.3)}.input:hover,.is-hovered.input,.is-hovered.textarea,.select select.is-hovered,.select select:hover,.textarea:hover{border-color:#b5b5b5}.input:active,.input:focus,.is-active.input,.is-active.textarea,.is-focused.input,.is-focused.textarea,.select select.is-active,.select select.is-focused,.select select:active,.select select:focus,.textarea:active,.textarea:focus{border-color:#3273dc;box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.input[disabled],.select fieldset[disabled] select,.select select[disabled],.textarea[disabled],fieldset[disabled] .input,fieldset[disabled] .select select,fieldset[disabled] .textarea{background-color:#f5f5f5;border-color:#f5f5f5;box-shadow:none;color:#7a7a7a}.input[disabled]::-moz-placeholder,.select fieldset[disabled] select::-moz-placeholder,.select select[disabled]::-moz-placeholder,.textarea[disabled]::-moz-placeholder,fieldset[disabled] .input::-moz-placeholder,fieldset[disabled] .select select::-moz-placeholder,fieldset[disabled] .textarea::-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]::-webkit-input-placeholder,.select fieldset[disabled] select::-webkit-input-placeholder,.select select[disabled]::-webkit-input-placeholder,.textarea[disabled]::-webkit-input-placeholder,fieldset[disabled] .input::-webkit-input-placeholder,fieldset[disabled] .select select::-webkit-input-placeholder,fieldset[disabled] .textarea::-webkit-input-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-moz-placeholder,.select fieldset[disabled] select:-moz-placeholder,.select select[disabled]:-moz-placeholder,.textarea[disabled]:-moz-placeholder,fieldset[disabled] .input:-moz-placeholder,fieldset[disabled] .select select:-moz-placeholder,fieldset[disabled] .textarea:-moz-placeholder{color:rgba(122,122,122,.3)}.input[disabled]:-ms-input-placeholder,.select fieldset[disabled] select:-ms-input-placeholder,.select select[disabled]:-ms-input-placeholder,.textarea[disabled]:-ms-input-placeholder,fieldset[disabled] .input:-ms-input-placeholder,fieldset[disabled] .select select:-ms-input-placeholder,fieldset[disabled] .textarea:-ms-input-placeholder{color:rgba(122,122,122,.3)}.input,.textarea{box-shadow:inset 0 .0625em .125em rgba(10,10,10,.05);max-width:100%;width:100%}.input[readonly],.textarea[readonly]{box-shadow:none}.is-white.input,.is-white.textarea{border-color:#fff}.is-white.input:active,.is-white.input:focus,.is-white.is-active.input,.is-white.is-active.textarea,.is-white.is-focused.input,.is-white.is-focused.textarea,.is-white.textarea:active,.is-white.textarea:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.is-black.input,.is-black.textarea{border-color:#0a0a0a}.is-black.input:active,.is-black.input:focus,.is-black.is-active.input,.is-black.is-active.textarea,.is-black.is-focused.input,.is-black.is-focused.textarea,.is-black.textarea:active,.is-black.textarea:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.is-light.input,.is-light.textarea{border-color:#f5f5f5}.is-light.input:active,.is-light.input:focus,.is-light.is-active.input,.is-light.is-active.textarea,.is-light.is-focused.input,.is-light.is-focused.textarea,.is-light.textarea:active,.is-light.textarea:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.is-dark.input,.is-dark.textarea{border-color:#363636}.is-dark.input:active,.is-dark.input:focus,.is-dark.is-active.input,.is-dark.is-active.textarea,.is-dark.is-focused.input,.is-dark.is-focused.textarea,.is-dark.textarea:active,.is-dark.textarea:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.is-primary.input,.is-primary.textarea{border-color:#00d1b2}.is-primary.input:active,.is-primary.input:focus,.is-primary.is-active.input,.is-primary.is-active.textarea,.is-primary.is-focused.input,.is-primary.is-focused.textarea,.is-primary.textarea:active,.is-primary.textarea:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.is-link.input,.is-link.textarea{border-color:#3273dc}.is-link.input:active,.is-link.input:focus,.is-link.is-active.input,.is-link.is-active.textarea,.is-link.is-focused.input,.is-link.is-focused.textarea,.is-link.textarea:active,.is-link.textarea:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.is-info.input,.is-info.textarea{border-color:#3298dc}.is-info.input:active,.is-info.input:focus,.is-info.is-active.input,.is-info.is-active.textarea,.is-info.is-focused.input,.is-info.is-focused.textarea,.is-info.textarea:active,.is-info.textarea:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.is-success.input,.is-success.textarea{border-color:#48c774}.is-success.input:active,.is-success.input:focus,.is-success.is-active.input,.is-success.is-active.textarea,.is-success.is-focused.input,.is-success.is-focused.textarea,.is-success.textarea:active,.is-success.textarea:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.is-warning.input,.is-warning.textarea{border-color:#ffdd57}.is-warning.input:active,.is-warning.input:focus,.is-warning.is-active.input,.is-warning.is-active.textarea,.is-warning.is-focused.input,.is-warning.is-focused.textarea,.is-warning.textarea:active,.is-warning.textarea:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.is-danger.input,.is-danger.textarea{border-color:#f14668}.is-danger.input:active,.is-danger.input:focus,.is-danger.is-active.input,.is-danger.is-active.textarea,.is-danger.is-focused.input,.is-danger.is-focused.textarea,.is-danger.textarea:active,.is-danger.textarea:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.is-small.input,.is-small.textarea{border-radius:2px;font-size:.75rem}.is-medium.input,.is-medium.textarea{font-size:1.25rem}.is-large.input,.is-large.textarea{font-size:1.5rem}.is-fullwidth.input,.is-fullwidth.textarea{display:block;width:100%}.is-inline.input,.is-inline.textarea{display:inline;width:auto}.input.is-rounded{border-radius:290486px;padding-left:calc(calc(.75em - 1px) + .375em);padding-right:calc(calc(.75em - 1px) + .375em)}.input.is-static{background-color:transparent;border-color:transparent;box-shadow:none;padding-left:0;padding-right:0}.textarea{display:block;max-width:100%;min-width:100%;padding:calc(.75em - 1px);resize:vertical}.textarea:not([rows]){max-height:40em;min-height:8em}.textarea[rows]{height:initial}.textarea.has-fixed-size{resize:none}.checkbox,.radio{cursor:pointer;display:inline-block;line-height:1.25;position:relative}.checkbox input,.radio input{cursor:pointer}.checkbox:hover,.radio:hover{color:#363636}.checkbox[disabled],.radio[disabled],fieldset[disabled] .checkbox,fieldset[disabled] .radio{color:#7a7a7a;cursor:not-allowed}.radio+.radio{margin-left:.5em}.select{display:inline-block;max-width:100%;position:relative;vertical-align:top}.select:not(.is-multiple){height:2.5em}.select:not(.is-multiple):not(.is-loading)::after{border-color:#3273dc;right:1.125em;z-index:4}.select.is-rounded select{border-radius:290486px;padding-left:1em}.select select{cursor:pointer;display:block;font-size:1em;max-width:100%;outline:0}.select select::-ms-expand{display:none}.select select[disabled]:hover,fieldset[disabled] .select select:hover{border-color:#f5f5f5}.select select:not([multiple]){padding-right:2.5em}.select select[multiple]{height:auto;padding:0}.select select[multiple] option{padding:.5em 1em}.select:not(.is-multiple):not(.is-loading):hover::after{border-color:#363636}.select.is-white:not(:hover)::after{border-color:#fff}.select.is-white select{border-color:#fff}.select.is-white select.is-hovered,.select.is-white select:hover{border-color:#f2f2f2}.select.is-white select.is-active,.select.is-white select.is-focused,.select.is-white select:active,.select.is-white select:focus{box-shadow:0 0 0 .125em rgba(255,255,255,.25)}.select.is-black:not(:hover)::after{border-color:#0a0a0a}.select.is-black select{border-color:#0a0a0a}.select.is-black select.is-hovered,.select.is-black select:hover{border-color:#000}.select.is-black select.is-active,.select.is-black select.is-focused,.select.is-black select:active,.select.is-black select:focus{box-shadow:0 0 0 .125em rgba(10,10,10,.25)}.select.is-light:not(:hover)::after{border-color:#f5f5f5}.select.is-light select{border-color:#f5f5f5}.select.is-light select.is-hovered,.select.is-light select:hover{border-color:#e8e8e8}.select.is-light select.is-active,.select.is-light select.is-focused,.select.is-light select:active,.select.is-light select:focus{box-shadow:0 0 0 .125em rgba(245,245,245,.25)}.select.is-dark:not(:hover)::after{border-color:#363636}.select.is-dark select{border-color:#363636}.select.is-dark select.is-hovered,.select.is-dark select:hover{border-color:#292929}.select.is-dark select.is-active,.select.is-dark select.is-focused,.select.is-dark select:active,.select.is-dark select:focus{box-shadow:0 0 0 .125em rgba(54,54,54,.25)}.select.is-primary:not(:hover)::after{border-color:#00d1b2}.select.is-primary select{border-color:#00d1b2}.select.is-primary select.is-hovered,.select.is-primary select:hover{border-color:#00b89c}.select.is-primary select.is-active,.select.is-primary select.is-focused,.select.is-primary select:active,.select.is-primary select:focus{box-shadow:0 0 0 .125em rgba(0,209,178,.25)}.select.is-link:not(:hover)::after{border-color:#3273dc}.select.is-link select{border-color:#3273dc}.select.is-link select.is-hovered,.select.is-link select:hover{border-color:#2366d1}.select.is-link select.is-active,.select.is-link select.is-focused,.select.is-link select:active,.select.is-link select:focus{box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.select.is-info:not(:hover)::after{border-color:#3298dc}.select.is-info select{border-color:#3298dc}.select.is-info select.is-hovered,.select.is-info select:hover{border-color:#238cd1}.select.is-info select.is-active,.select.is-info select.is-focused,.select.is-info select:active,.select.is-info select:focus{box-shadow:0 0 0 .125em rgba(50,152,220,.25)}.select.is-success:not(:hover)::after{border-color:#48c774}.select.is-success select{border-color:#48c774}.select.is-success select.is-hovered,.select.is-success select:hover{border-color:#3abb67}.select.is-success select.is-active,.select.is-success select.is-focused,.select.is-success select:active,.select.is-success select:focus{box-shadow:0 0 0 .125em rgba(72,199,116,.25)}.select.is-warning:not(:hover)::after{border-color:#ffdd57}.select.is-warning select{border-color:#ffdd57}.select.is-warning select.is-hovered,.select.is-warning select:hover{border-color:#ffd83d}.select.is-warning select.is-active,.select.is-warning select.is-focused,.select.is-warning select:active,.select.is-warning select:focus{box-shadow:0 0 0 .125em rgba(255,221,87,.25)}.select.is-danger:not(:hover)::after{border-color:#f14668}.select.is-danger select{border-color:#f14668}.select.is-danger select.is-hovered,.select.is-danger select:hover{border-color:#ef2e55}.select.is-danger select.is-active,.select.is-danger select.is-focused,.select.is-danger select:active,.select.is-danger select:focus{box-shadow:0 0 0 .125em rgba(241,70,104,.25)}.select.is-small{border-radius:2px;font-size:.75rem}.select.is-medium{font-size:1.25rem}.select.is-large{font-size:1.5rem}.select.is-disabled::after{border-color:#7a7a7a}.select.is-fullwidth{width:100%}.select.is-fullwidth select{width:100%}.select.is-loading::after{margin-top:0;position:absolute;right:.625em;top:.625em;transform:none}.select.is-loading.is-small:after{font-size:.75rem}.select.is-loading.is-medium:after{font-size:1.25rem}.select.is-loading.is-large:after{font-size:1.5rem}.file{align-items:stretch;display:flex;justify-content:flex-start;position:relative}.file.is-white .file-cta{background-color:#fff;border-color:transparent;color:#0a0a0a}.file.is-white.is-hovered .file-cta,.file.is-white:hover .file-cta{background-color:#f9f9f9;border-color:transparent;color:#0a0a0a}.file.is-white.is-focused .file-cta,.file.is-white:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,255,255,.25);color:#0a0a0a}.file.is-white.is-active .file-cta,.file.is-white:active .file-cta{background-color:#f2f2f2;border-color:transparent;color:#0a0a0a}.file.is-black .file-cta{background-color:#0a0a0a;border-color:transparent;color:#fff}.file.is-black.is-hovered .file-cta,.file.is-black:hover .file-cta{background-color:#040404;border-color:transparent;color:#fff}.file.is-black.is-focused .file-cta,.file.is-black:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(10,10,10,.25);color:#fff}.file.is-black.is-active .file-cta,.file.is-black:active .file-cta{background-color:#000;border-color:transparent;color:#fff}.file.is-light .file-cta{background-color:#f5f5f5;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-hovered .file-cta,.file.is-light:hover .file-cta{background-color:#eee;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-light.is-focused .file-cta,.file.is-light:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(245,245,245,.25);color:rgba(0,0,0,.7)}.file.is-light.is-active .file-cta,.file.is-light:active .file-cta{background-color:#e8e8e8;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-dark .file-cta{background-color:#363636;border-color:transparent;color:#fff}.file.is-dark.is-hovered .file-cta,.file.is-dark:hover .file-cta{background-color:#2f2f2f;border-color:transparent;color:#fff}.file.is-dark.is-focused .file-cta,.file.is-dark:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(54,54,54,.25);color:#fff}.file.is-dark.is-active .file-cta,.file.is-dark:active .file-cta{background-color:#292929;border-color:transparent;color:#fff}.file.is-primary .file-cta{background-color:#00d1b2;border-color:transparent;color:#fff}.file.is-primary.is-hovered .file-cta,.file.is-primary:hover .file-cta{background-color:#00c4a7;border-color:transparent;color:#fff}.file.is-primary.is-focused .file-cta,.file.is-primary:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(0,209,178,.25);color:#fff}.file.is-primary.is-active .file-cta,.file.is-primary:active .file-cta{background-color:#00b89c;border-color:transparent;color:#fff}.file.is-link .file-cta{background-color:#3273dc;border-color:transparent;color:#fff}.file.is-link.is-hovered .file-cta,.file.is-link:hover .file-cta{background-color:#276cda;border-color:transparent;color:#fff}.file.is-link.is-focused .file-cta,.file.is-link:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,115,220,.25);color:#fff}.file.is-link.is-active .file-cta,.file.is-link:active .file-cta{background-color:#2366d1;border-color:transparent;color:#fff}.file.is-info .file-cta{background-color:#3298dc;border-color:transparent;color:#fff}.file.is-info.is-hovered .file-cta,.file.is-info:hover .file-cta{background-color:#2793da;border-color:transparent;color:#fff}.file.is-info.is-focused .file-cta,.file.is-info:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(50,152,220,.25);color:#fff}.file.is-info.is-active .file-cta,.file.is-info:active .file-cta{background-color:#238cd1;border-color:transparent;color:#fff}.file.is-success .file-cta{background-color:#48c774;border-color:transparent;color:#fff}.file.is-success.is-hovered .file-cta,.file.is-success:hover .file-cta{background-color:#3ec46d;border-color:transparent;color:#fff}.file.is-success.is-focused .file-cta,.file.is-success:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(72,199,116,.25);color:#fff}.file.is-success.is-active .file-cta,.file.is-success:active .file-cta{background-color:#3abb67;border-color:transparent;color:#fff}.file.is-warning .file-cta{background-color:#ffdd57;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-hovered .file-cta,.file.is-warning:hover .file-cta{background-color:#ffdb4a;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-warning.is-focused .file-cta,.file.is-warning:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(255,221,87,.25);color:rgba(0,0,0,.7)}.file.is-warning.is-active .file-cta,.file.is-warning:active .file-cta{background-color:#ffd83d;border-color:transparent;color:rgba(0,0,0,.7)}.file.is-danger .file-cta{background-color:#f14668;border-color:transparent;color:#fff}.file.is-danger.is-hovered .file-cta,.file.is-danger:hover .file-cta{background-color:#f03a5f;border-color:transparent;color:#fff}.file.is-danger.is-focused .file-cta,.file.is-danger:focus .file-cta{border-color:transparent;box-shadow:0 0 .5em rgba(241,70,104,.25);color:#fff}.file.is-danger.is-active .file-cta,.file.is-danger:active .file-cta{background-color:#ef2e55;border-color:transparent;color:#fff}.file.is-small{font-size:.75rem}.file.is-medium{font-size:1.25rem}.file.is-medium .file-icon .fa{font-size:21px}.file.is-large{font-size:1.5rem}.file.is-large .file-icon .fa{font-size:28px}.file.has-name .file-cta{border-bottom-right-radius:0;border-top-right-radius:0}.file.has-name .file-name{border-bottom-left-radius:0;border-top-left-radius:0}.file.has-name.is-empty .file-cta{border-radius:4px}.file.has-name.is-empty .file-name{display:none}.file.is-boxed .file-label{flex-direction:column}.file.is-boxed .file-cta{flex-direction:column;height:auto;padding:1em 3em}.file.is-boxed .file-name{border-width:0 1px 1px}.file.is-boxed .file-icon{height:1.5em;width:1.5em}.file.is-boxed .file-icon .fa{font-size:21px}.file.is-boxed.is-small .file-icon .fa{font-size:14px}.file.is-boxed.is-medium .file-icon .fa{font-size:28px}.file.is-boxed.is-large .file-icon .fa{font-size:35px}.file.is-boxed.has-name .file-cta{border-radius:4px 4px 0 0}.file.is-boxed.has-name .file-name{border-radius:0 0 4px 4px;border-width:0 1px 1px}.file.is-centered{justify-content:center}.file.is-fullwidth .file-label{width:100%}.file.is-fullwidth .file-name{flex-grow:1;max-width:none}.file.is-right{justify-content:flex-end}.file.is-right .file-cta{border-radius:0 4px 4px 0}.file.is-right .file-name{border-radius:4px 0 0 4px;border-width:1px 0 1px 1px;order:-1}.file-label{align-items:stretch;display:flex;cursor:pointer;justify-content:flex-start;overflow:hidden;position:relative}.file-label:hover .file-cta{background-color:#eee;color:#363636}.file-label:hover .file-name{border-color:#d5d5d5}.file-label:active .file-cta{background-color:#e8e8e8;color:#363636}.file-label:active .file-name{border-color:#cfcfcf}.file-input{height:100%;left:0;opacity:0;outline:0;position:absolute;top:0;width:100%}.file-cta,.file-name{border-color:#dbdbdb;border-radius:4px;font-size:1em;padding-left:1em;padding-right:1em;white-space:nowrap}.file-cta{background-color:#f5f5f5;color:#4a4a4a}.file-name{border-color:#dbdbdb;border-style:solid;border-width:1px 1px 1px 0;display:block;max-width:16em;overflow:hidden;text-align:left;text-overflow:ellipsis}.file-icon{align-items:center;display:flex;height:1em;justify-content:center;margin-right:.5em;width:1em}.file-icon .fa{font-size:14px}.label{color:#363636;display:block;font-size:1rem;font-weight:700}.label:not(:last-child){margin-bottom:.5em}.label.is-small{font-size:.75rem}.label.is-medium{font-size:1.25rem}.label.is-large{font-size:1.5rem}.help{display:block;font-size:.75rem;margin-top:.25rem}.help.is-white{color:#fff}.help.is-black{color:#0a0a0a}.help.is-light{color:#f5f5f5}.help.is-dark{color:#363636}.help.is-primary{color:#00d1b2}.help.is-link{color:#3273dc}.help.is-info{color:#3298dc}.help.is-success{color:#48c774}.help.is-warning{color:#ffdd57}.help.is-danger{color:#f14668}.field:not(:last-child){margin-bottom:.75rem}.field.has-addons{display:flex;justify-content:flex-start}.field.has-addons .control:not(:last-child){margin-right:-1px}.field.has-addons .control:not(:first-child):not(:last-child) .button,.field.has-addons .control:not(:first-child):not(:last-child) .input,.field.has-addons .control:not(:first-child):not(:last-child) .select select{border-radius:0}.field.has-addons .control:first-child:not(:only-child) .button,.field.has-addons .control:first-child:not(:only-child) .input,.field.has-addons .control:first-child:not(:only-child) .select select{border-bottom-right-radius:0;border-top-right-radius:0}.field.has-addons .control:last-child:not(:only-child) .button,.field.has-addons .control:last-child:not(:only-child) .input,.field.has-addons .control:last-child:not(:only-child) .select select{border-bottom-left-radius:0;border-top-left-radius:0}.field.has-addons .control .button:not([disabled]).is-hovered,.field.has-addons .control .button:not([disabled]):hover,.field.has-addons .control .input:not([disabled]).is-hovered,.field.has-addons .control .input:not([disabled]):hover,.field.has-addons .control .select select:not([disabled]).is-hovered,.field.has-addons .control .select select:not([disabled]):hover{z-index:2}.field.has-addons .control .button:not([disabled]).is-active,.field.has-addons .control .button:not([disabled]).is-focused,.field.has-addons .control .button:not([disabled]):active,.field.has-addons .control .button:not([disabled]):focus,.field.has-addons .control .input:not([disabled]).is-active,.field.has-addons .control .input:not([disabled]).is-focused,.field.has-addons .control .input:not([disabled]):active,.field.has-addons .control .input:not([disabled]):focus,.field.has-addons .control .select select:not([disabled]).is-active,.field.has-addons .control .select select:not([disabled]).is-focused,.field.has-addons .control .select select:not([disabled]):active,.field.has-addons .control .select select:not([disabled]):focus{z-index:3}.field.has-addons .control .button:not([disabled]).is-active:hover,.field.has-addons .control .button:not([disabled]).is-focused:hover,.field.has-addons .control .button:not([disabled]):active:hover,.field.has-addons .control .button:not([disabled]):focus:hover,.field.has-addons .control .input:not([disabled]).is-active:hover,.field.has-addons .control .input:not([disabled]).is-focused:hover,.field.has-addons .control .input:not([disabled]):active:hover,.field.has-addons .control .input:not([disabled]):focus:hover,.field.has-addons .control .select select:not([disabled]).is-active:hover,.field.has-addons .control .select select:not([disabled]).is-focused:hover,.field.has-addons .control .select select:not([disabled]):active:hover,.field.has-addons .control .select select:not([disabled]):focus:hover{z-index:4}.field.has-addons .control.is-expanded{flex-grow:1;flex-shrink:1}.field.has-addons.has-addons-centered{justify-content:center}.field.has-addons.has-addons-right{justify-content:flex-end}.field.has-addons.has-addons-fullwidth .control{flex-grow:1;flex-shrink:0}.field.is-grouped{display:flex;justify-content:flex-start}.field.is-grouped>.control{flex-shrink:0}.field.is-grouped>.control:not(:last-child){margin-bottom:0;margin-right:.75rem}.field.is-grouped>.control.is-expanded{flex-grow:1;flex-shrink:1}.field.is-grouped.is-grouped-centered{justify-content:center}.field.is-grouped.is-grouped-right{justify-content:flex-end}.field.is-grouped.is-grouped-multiline{flex-wrap:wrap}.field.is-grouped.is-grouped-multiline>.control:last-child,.field.is-grouped.is-grouped-multiline>.control:not(:last-child){margin-bottom:.75rem}.field.is-grouped.is-grouped-multiline:last-child{margin-bottom:-.75rem}.field.is-grouped.is-grouped-multiline:not(:last-child){margin-bottom:0}@media screen and (min-width:769px),print{.field.is-horizontal{display:flex}}.field-label .label{font-size:inherit}@media screen and (max-width:768px){.field-label{margin-bottom:.5rem}}@media screen and (min-width:769px),print{.field-label{flex-basis:0;flex-grow:1;flex-shrink:0;margin-right:1.5rem;text-align:right}.field-label.is-small{font-size:.75rem;padding-top:.375em}.field-label.is-normal{padding-top:.375em}.field-label.is-medium{font-size:1.25rem;padding-top:.375em}.field-label.is-large{font-size:1.5rem;padding-top:.375em}}.field-body .field .field{margin-bottom:0}@media screen and (min-width:769px),print{.field-body{display:flex;flex-basis:0;flex-grow:5;flex-shrink:1}.field-body .field{margin-bottom:0}.field-body>.field{flex-shrink:1}.field-body>.field:not(.is-narrow){flex-grow:1}.field-body>.field:not(:last-child){margin-right:.75rem}}.control{box-sizing:border-box;clear:both;font-size:1rem;position:relative;text-align:left}.control.has-icons-left .input:focus~.icon,.control.has-icons-left .select:focus~.icon,.control.has-icons-right .input:focus~.icon,.control.has-icons-right .select:focus~.icon{color:#4a4a4a}.control.has-icons-left .input.is-small~.icon,.control.has-icons-left .select.is-small~.icon,.control.has-icons-right .input.is-small~.icon,.control.has-icons-right .select.is-small~.icon{font-size:.75rem}.control.has-icons-left .input.is-medium~.icon,.control.has-icons-left .select.is-medium~.icon,.control.has-icons-right .input.is-medium~.icon,.control.has-icons-right .select.is-medium~.icon{font-size:1.25rem}.control.has-icons-left .input.is-large~.icon,.control.has-icons-left .select.is-large~.icon,.control.has-icons-right .input.is-large~.icon,.control.has-icons-right .select.is-large~.icon{font-size:1.5rem}.control.has-icons-left .icon,.control.has-icons-right .icon{color:#dbdbdb;height:2.5em;pointer-events:none;position:absolute;top:0;width:2.5em;z-index:4}.control.has-icons-left .input,.control.has-icons-left .select select{padding-left:2.5em}.control.has-icons-left .icon.is-left{left:0}.control.has-icons-right .input,.control.has-icons-right .select select{padding-right:2.5em}.control.has-icons-right .icon.is-right{right:0}.control.is-loading::after{position:absolute!important;right:.625em;top:.625em;z-index:4}.control.is-loading.is-small:after{font-size:.75rem}.control.is-loading.is-medium:after{font-size:1.25rem}.control.is-loading.is-large:after{font-size:1.5rem}.breadcrumb{font-size:1rem;white-space:nowrap}.breadcrumb a{align-items:center;color:#3273dc;display:flex;justify-content:center;padding:0 .75em}.breadcrumb a:hover{color:#363636}.breadcrumb li{align-items:center;display:flex}.breadcrumb li:first-child a{padding-left:0}.breadcrumb li.is-active a{color:#363636;cursor:default;pointer-events:none}.breadcrumb li+li::before{color:#b5b5b5;content:"\0002f"}.breadcrumb ol,.breadcrumb ul{align-items:flex-start;display:flex;flex-wrap:wrap;justify-content:flex-start}.breadcrumb .icon:first-child{margin-right:.5em}.breadcrumb .icon:last-child{margin-left:.5em}.breadcrumb.is-centered ol,.breadcrumb.is-centered ul{justify-content:center}.breadcrumb.is-right ol,.breadcrumb.is-right ul{justify-content:flex-end}.breadcrumb.is-small{font-size:.75rem}.breadcrumb.is-medium{font-size:1.25rem}.breadcrumb.is-large{font-size:1.5rem}.breadcrumb.has-arrow-separator li+li::before{content:"\02192"}.breadcrumb.has-bullet-separator li+li::before{content:"\02022"}.breadcrumb.has-dot-separator li+li::before{content:"\000b7"}.breadcrumb.has-succeeds-separator li+li::before{content:"\0227B"}.card{background-color:#fff;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);color:#4a4a4a;max-width:100%;position:relative}.card-header{background-color:transparent;align-items:stretch;box-shadow:0 .125em .25em rgba(10,10,10,.1);display:flex}.card-header-title{align-items:center;color:#363636;display:flex;flex-grow:1;font-weight:700;padding:.75rem 1rem}.card-header-title.is-centered{justify-content:center}.card-header-icon{align-items:center;cursor:pointer;display:flex;justify-content:center;padding:.75rem 1rem}.card-image{display:block;position:relative}.card-content{background-color:transparent;padding:1.5rem}.card-footer{background-color:transparent;border-top:1px solid #ededed;align-items:stretch;display:flex}.card-footer-item{align-items:center;display:flex;flex-basis:0;flex-grow:1;flex-shrink:0;justify-content:center;padding:.75rem}.card-footer-item:not(:last-child){border-right:1px solid #ededed}.card .media:not(:last-child){margin-bottom:1.5rem}.dropdown{display:inline-flex;position:relative;vertical-align:top}.dropdown.is-active .dropdown-menu,.dropdown.is-hoverable:hover .dropdown-menu{display:block}.dropdown.is-right .dropdown-menu{left:auto;right:0}.dropdown.is-up .dropdown-menu{bottom:100%;padding-bottom:4px;padding-top:initial;top:auto}.dropdown-menu{display:none;left:0;min-width:12rem;padding-top:4px;position:absolute;top:100%;z-index:20}.dropdown-content{background-color:#fff;border-radius:4px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);padding-bottom:.5rem;padding-top:.5rem}.dropdown-item{color:#4a4a4a;display:block;font-size:.875rem;line-height:1.5;padding:.375rem 1rem;position:relative}a.dropdown-item,button.dropdown-item{padding-right:3rem;text-align:left;white-space:nowrap;width:100%}a.dropdown-item:hover,button.dropdown-item:hover{background-color:#f5f5f5;color:#0a0a0a}a.dropdown-item.is-active,button.dropdown-item.is-active{background-color:#3273dc;color:#fff}.dropdown-divider{background-color:#ededed;border:none;display:block;height:1px;margin:.5rem 0}.level{align-items:center;justify-content:space-between}.level code{border-radius:4px}.level img{display:inline-block;vertical-align:top}.level.is-mobile{display:flex}.level.is-mobile .level-left,.level.is-mobile .level-right{display:flex}.level.is-mobile .level-left+.level-right{margin-top:0}.level.is-mobile .level-item:not(:last-child){margin-bottom:0;margin-right:.75rem}.level.is-mobile .level-item:not(.is-narrow){flex-grow:1}@media screen and (min-width:769px),print{.level{display:flex}.level>.level-item:not(.is-narrow){flex-grow:1}}.level-item{align-items:center;display:flex;flex-basis:auto;flex-grow:0;flex-shrink:0;justify-content:center}.level-item .subtitle,.level-item .title{margin-bottom:0}@media screen and (max-width:768px){.level-item:not(:last-child){margin-bottom:.75rem}}.level-left,.level-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.level-left .level-item.is-flexible,.level-right .level-item.is-flexible{flex-grow:1}@media screen and (min-width:769px),print{.level-left .level-item:not(:last-child),.level-right .level-item:not(:last-child){margin-right:.75rem}}.level-left{align-items:center;justify-content:flex-start}@media screen and (max-width:768px){.level-left+.level-right{margin-top:1.5rem}}@media screen and (min-width:769px),print{.level-left{display:flex}}.level-right{align-items:center;justify-content:flex-end}@media screen and (min-width:769px),print{.level-right{display:flex}}.list{background-color:#fff;border-radius:4px;box-shadow:0 2px 3px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1)}.list-item{display:block;padding:.5em 1em}.list-item:not(a){color:#4a4a4a}.list-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-item:last-child{border-bottom-left-radius:4px;border-bottom-right-radius:4px}.list-item:not(:last-child){border-bottom:1px solid #dbdbdb}.list-item.is-active{background-color:#3273dc;color:#fff}a.list-item{background-color:#f5f5f5;cursor:pointer}.media{align-items:flex-start;display:flex;text-align:left}.media .content:not(:last-child){margin-bottom:.75rem}.media .media{border-top:1px solid rgba(219,219,219,.5);display:flex;padding-top:.75rem}.media .media .content:not(:last-child),.media .media .control:not(:last-child){margin-bottom:.5rem}.media .media .media{padding-top:.5rem}.media .media .media+.media{margin-top:.5rem}.media+.media{border-top:1px solid rgba(219,219,219,.5);margin-top:1rem;padding-top:1rem}.media.is-large+.media{margin-top:1.5rem;padding-top:1.5rem}.media-left,.media-right{flex-basis:auto;flex-grow:0;flex-shrink:0}.media-left{margin-right:1rem}.media-right{margin-left:1rem}.media-content{flex-basis:auto;flex-grow:1;flex-shrink:1;text-align:left}@media screen and (max-width:768px){.media-content{overflow-x:auto}}.menu{font-size:1rem}.menu.is-small{font-size:.75rem}.menu.is-medium{font-size:1.25rem}.menu.is-large{font-size:1.5rem}.menu-list{line-height:1.25}.menu-list a{border-radius:2px;color:#4a4a4a;display:block;padding:.5em .75em}.menu-list a:hover{background-color:#f5f5f5;color:#363636}.menu-list a.is-active{background-color:#3273dc;color:#fff}.menu-list li ul{border-left:1px solid #dbdbdb;margin:.75em;padding-left:.75em}.menu-label{color:#7a7a7a;font-size:.75em;letter-spacing:.1em;text-transform:uppercase}.menu-label:not(:first-child){margin-top:1em}.menu-label:not(:last-child){margin-bottom:1em}.message{background-color:#f5f5f5;border-radius:4px;font-size:1rem}.message strong{color:currentColor}.message a:not(.button):not(.tag):not(.dropdown-item){color:currentColor;text-decoration:underline}.message.is-small{font-size:.75rem}.message.is-medium{font-size:1.25rem}.message.is-large{font-size:1.5rem}.message.is-white{background-color:#fff}.message.is-white .message-header{background-color:#fff;color:#0a0a0a}.message.is-white .message-body{border-color:#fff}.message.is-black{background-color:#fafafa}.message.is-black .message-header{background-color:#0a0a0a;color:#fff}.message.is-black .message-body{border-color:#0a0a0a}.message.is-light{background-color:#fafafa}.message.is-light .message-header{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.message.is-light .message-body{border-color:#f5f5f5}.message.is-dark{background-color:#fafafa}.message.is-dark .message-header{background-color:#363636;color:#fff}.message.is-dark .message-body{border-color:#363636}.message.is-primary{background-color:#ebfffc}.message.is-primary .message-header{background-color:#00d1b2;color:#fff}.message.is-primary .message-body{border-color:#00d1b2;color:#00947e}.message.is-link{background-color:#eef3fc}.message.is-link .message-header{background-color:#3273dc;color:#fff}.message.is-link .message-body{border-color:#3273dc;color:#2160c4}.message.is-info{background-color:#eef6fc}.message.is-info .message-header{background-color:#3298dc;color:#fff}.message.is-info .message-body{border-color:#3298dc;color:#1d72aa}.message.is-success{background-color:#effaf3}.message.is-success .message-header{background-color:#48c774;color:#fff}.message.is-success .message-body{border-color:#48c774;color:#257942}.message.is-warning{background-color:#fffbeb}.message.is-warning .message-header{background-color:#ffdd57;color:rgba(0,0,0,.7)}.message.is-warning .message-body{border-color:#ffdd57;color:#947600}.message.is-danger{background-color:#feecf0}.message.is-danger .message-header{background-color:#f14668;color:#fff}.message.is-danger .message-body{border-color:#f14668;color:#cc0f35}.message-header{align-items:center;background-color:#4a4a4a;border-radius:4px 4px 0 0;color:#fff;display:flex;font-weight:700;justify-content:space-between;line-height:1.25;padding:.75em 1em;position:relative}.message-header .delete{flex-grow:0;flex-shrink:0;margin-left:.75em}.message-header+.message-body{border-width:0;border-top-left-radius:0;border-top-right-radius:0}.message-body{border-color:#dbdbdb;border-radius:4px;border-style:solid;border-width:0 0 0 4px;color:#4a4a4a;padding:1.25em 1.5em}.message-body code,.message-body pre{background-color:#fff}.message-body pre code{background-color:transparent}.modal{align-items:center;display:none;flex-direction:column;justify-content:center;overflow:hidden;position:fixed;z-index:40}.modal.is-active{display:flex}.modal-background{background-color:rgba(10,10,10,.86)}.modal-card,.modal-content{margin:0 20px;max-height:calc(100vh - 160px);overflow:auto;position:relative;width:100%}@media screen and (min-width:769px),print{.modal-card,.modal-content{margin:0 auto;max-height:calc(100vh - 40px);width:640px}}.modal-close{background:0 0;height:40px;position:fixed;right:20px;top:20px;width:40px}.modal-card{display:flex;flex-direction:column;max-height:calc(100vh - 40px);overflow:hidden;-ms-overflow-y:visible}.modal-card-foot,.modal-card-head{align-items:center;background-color:#f5f5f5;display:flex;flex-shrink:0;justify-content:flex-start;padding:20px;position:relative}.modal-card-head{border-bottom:1px solid #dbdbdb;border-top-left-radius:6px;border-top-right-radius:6px}.modal-card-title{color:#363636;flex-grow:1;flex-shrink:0;font-size:1.5rem;line-height:1}.modal-card-foot{border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:1px solid #dbdbdb}.modal-card-foot .button:not(:last-child){margin-right:.5em}.modal-card-body{-webkit-overflow-scrolling:touch;background-color:#fff;flex-grow:1;flex-shrink:1;overflow:auto;padding:20px}.navbar{background-color:#fff;min-height:3.25rem;position:relative;z-index:30}.navbar.is-white{background-color:#fff;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link,.navbar.is-white .navbar-brand>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link.is-active,.navbar.is-white .navbar-brand .navbar-link:focus,.navbar.is-white .navbar-brand .navbar-link:hover,.navbar.is-white .navbar-brand>a.navbar-item.is-active,.navbar.is-white .navbar-brand>a.navbar-item:focus,.navbar.is-white .navbar-brand>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-brand .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-burger{color:#0a0a0a}@media screen and (min-width:1024px){.navbar.is-white .navbar-end .navbar-link,.navbar.is-white .navbar-end>.navbar-item,.navbar.is-white .navbar-start .navbar-link,.navbar.is-white .navbar-start>.navbar-item{color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link.is-active,.navbar.is-white .navbar-end .navbar-link:focus,.navbar.is-white .navbar-end .navbar-link:hover,.navbar.is-white .navbar-end>a.navbar-item.is-active,.navbar.is-white .navbar-end>a.navbar-item:focus,.navbar.is-white .navbar-end>a.navbar-item:hover,.navbar.is-white .navbar-start .navbar-link.is-active,.navbar.is-white .navbar-start .navbar-link:focus,.navbar.is-white .navbar-start .navbar-link:hover,.navbar.is-white .navbar-start>a.navbar-item.is-active,.navbar.is-white .navbar-start>a.navbar-item:focus,.navbar.is-white .navbar-start>a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-end .navbar-link::after,.navbar.is-white .navbar-start .navbar-link::after{border-color:#0a0a0a}.navbar.is-white .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-white .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-white .navbar-item.has-dropdown:hover .navbar-link{background-color:#f2f2f2;color:#0a0a0a}.navbar.is-white .navbar-dropdown a.navbar-item.is-active{background-color:#fff;color:#0a0a0a}}.navbar.is-black{background-color:#0a0a0a;color:#fff}.navbar.is-black .navbar-brand .navbar-link,.navbar.is-black .navbar-brand>.navbar-item{color:#fff}.navbar.is-black .navbar-brand .navbar-link.is-active,.navbar.is-black .navbar-brand .navbar-link:focus,.navbar.is-black .navbar-brand .navbar-link:hover,.navbar.is-black .navbar-brand>a.navbar-item.is-active,.navbar.is-black .navbar-brand>a.navbar-item:focus,.navbar.is-black .navbar-brand>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-black .navbar-end .navbar-link,.navbar.is-black .navbar-end>.navbar-item,.navbar.is-black .navbar-start .navbar-link,.navbar.is-black .navbar-start>.navbar-item{color:#fff}.navbar.is-black .navbar-end .navbar-link.is-active,.navbar.is-black .navbar-end .navbar-link:focus,.navbar.is-black .navbar-end .navbar-link:hover,.navbar.is-black .navbar-end>a.navbar-item.is-active,.navbar.is-black .navbar-end>a.navbar-item:focus,.navbar.is-black .navbar-end>a.navbar-item:hover,.navbar.is-black .navbar-start .navbar-link.is-active,.navbar.is-black .navbar-start .navbar-link:focus,.navbar.is-black .navbar-start .navbar-link:hover,.navbar.is-black .navbar-start>a.navbar-item.is-active,.navbar.is-black .navbar-start>a.navbar-item:focus,.navbar.is-black .navbar-start>a.navbar-item:hover{background-color:#000;color:#fff}.navbar.is-black .navbar-end .navbar-link::after,.navbar.is-black .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-black .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-black .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-black .navbar-item.has-dropdown:hover .navbar-link{background-color:#000;color:#fff}.navbar.is-black .navbar-dropdown a.navbar-item.is-active{background-color:#0a0a0a;color:#fff}}.navbar.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link,.navbar.is-light .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link.is-active,.navbar.is-light .navbar-brand .navbar-link:focus,.navbar.is-light .navbar-brand .navbar-link:hover,.navbar.is-light .navbar-brand>a.navbar-item.is-active,.navbar.is-light .navbar-brand>a.navbar-item:focus,.navbar.is-light .navbar-brand>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-light .navbar-end .navbar-link,.navbar.is-light .navbar-end>.navbar-item,.navbar.is-light .navbar-start .navbar-link,.navbar.is-light .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link.is-active,.navbar.is-light .navbar-end .navbar-link:focus,.navbar.is-light .navbar-end .navbar-link:hover,.navbar.is-light .navbar-end>a.navbar-item.is-active,.navbar.is-light .navbar-end>a.navbar-item:focus,.navbar.is-light .navbar-end>a.navbar-item:hover,.navbar.is-light .navbar-start .navbar-link.is-active,.navbar.is-light .navbar-start .navbar-link:focus,.navbar.is-light .navbar-start .navbar-link:hover,.navbar.is-light .navbar-start>a.navbar-item.is-active,.navbar.is-light .navbar-start>a.navbar-item:focus,.navbar.is-light .navbar-start>a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-end .navbar-link::after,.navbar.is-light .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-light .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-light .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-light .navbar-item.has-dropdown:hover .navbar-link{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.navbar.is-light .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:rgba(0,0,0,.7)}}.navbar.is-dark{background-color:#363636;color:#fff}.navbar.is-dark .navbar-brand .navbar-link,.navbar.is-dark .navbar-brand>.navbar-item{color:#fff}.navbar.is-dark .navbar-brand .navbar-link.is-active,.navbar.is-dark .navbar-brand .navbar-link:focus,.navbar.is-dark .navbar-brand .navbar-link:hover,.navbar.is-dark .navbar-brand>a.navbar-item.is-active,.navbar.is-dark .navbar-brand>a.navbar-item:focus,.navbar.is-dark .navbar-brand>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-dark .navbar-end .navbar-link,.navbar.is-dark .navbar-end>.navbar-item,.navbar.is-dark .navbar-start .navbar-link,.navbar.is-dark .navbar-start>.navbar-item{color:#fff}.navbar.is-dark .navbar-end .navbar-link.is-active,.navbar.is-dark .navbar-end .navbar-link:focus,.navbar.is-dark .navbar-end .navbar-link:hover,.navbar.is-dark .navbar-end>a.navbar-item.is-active,.navbar.is-dark .navbar-end>a.navbar-item:focus,.navbar.is-dark .navbar-end>a.navbar-item:hover,.navbar.is-dark .navbar-start .navbar-link.is-active,.navbar.is-dark .navbar-start .navbar-link:focus,.navbar.is-dark .navbar-start .navbar-link:hover,.navbar.is-dark .navbar-start>a.navbar-item.is-active,.navbar.is-dark .navbar-start>a.navbar-item:focus,.navbar.is-dark .navbar-start>a.navbar-item:hover{background-color:#292929;color:#fff}.navbar.is-dark .navbar-end .navbar-link::after,.navbar.is-dark .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-dark .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-dark .navbar-item.has-dropdown:hover .navbar-link{background-color:#292929;color:#fff}.navbar.is-dark .navbar-dropdown a.navbar-item.is-active{background-color:#363636;color:#fff}}.navbar.is-primary{background-color:#00d1b2;color:#fff}.navbar.is-primary .navbar-brand .navbar-link,.navbar.is-primary .navbar-brand>.navbar-item{color:#fff}.navbar.is-primary .navbar-brand .navbar-link.is-active,.navbar.is-primary .navbar-brand .navbar-link:focus,.navbar.is-primary .navbar-brand .navbar-link:hover,.navbar.is-primary .navbar-brand>a.navbar-item.is-active,.navbar.is-primary .navbar-brand>a.navbar-item:focus,.navbar.is-primary .navbar-brand>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-primary .navbar-end .navbar-link,.navbar.is-primary .navbar-end>.navbar-item,.navbar.is-primary .navbar-start .navbar-link,.navbar.is-primary .navbar-start>.navbar-item{color:#fff}.navbar.is-primary .navbar-end .navbar-link.is-active,.navbar.is-primary .navbar-end .navbar-link:focus,.navbar.is-primary .navbar-end .navbar-link:hover,.navbar.is-primary .navbar-end>a.navbar-item.is-active,.navbar.is-primary .navbar-end>a.navbar-item:focus,.navbar.is-primary .navbar-end>a.navbar-item:hover,.navbar.is-primary .navbar-start .navbar-link.is-active,.navbar.is-primary .navbar-start .navbar-link:focus,.navbar.is-primary .navbar-start .navbar-link:hover,.navbar.is-primary .navbar-start>a.navbar-item.is-active,.navbar.is-primary .navbar-start>a.navbar-item:focus,.navbar.is-primary .navbar-start>a.navbar-item:hover{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-end .navbar-link::after,.navbar.is-primary .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-primary .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-primary .navbar-item.has-dropdown:hover .navbar-link{background-color:#00b89c;color:#fff}.navbar.is-primary .navbar-dropdown a.navbar-item.is-active{background-color:#00d1b2;color:#fff}}.navbar.is-link{background-color:#3273dc;color:#fff}.navbar.is-link .navbar-brand .navbar-link,.navbar.is-link .navbar-brand>.navbar-item{color:#fff}.navbar.is-link .navbar-brand .navbar-link.is-active,.navbar.is-link .navbar-brand .navbar-link:focus,.navbar.is-link .navbar-brand .navbar-link:hover,.navbar.is-link .navbar-brand>a.navbar-item.is-active,.navbar.is-link .navbar-brand>a.navbar-item:focus,.navbar.is-link .navbar-brand>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-link .navbar-end .navbar-link,.navbar.is-link .navbar-end>.navbar-item,.navbar.is-link .navbar-start .navbar-link,.navbar.is-link .navbar-start>.navbar-item{color:#fff}.navbar.is-link .navbar-end .navbar-link.is-active,.navbar.is-link .navbar-end .navbar-link:focus,.navbar.is-link .navbar-end .navbar-link:hover,.navbar.is-link .navbar-end>a.navbar-item.is-active,.navbar.is-link .navbar-end>a.navbar-item:focus,.navbar.is-link .navbar-end>a.navbar-item:hover,.navbar.is-link .navbar-start .navbar-link.is-active,.navbar.is-link .navbar-start .navbar-link:focus,.navbar.is-link .navbar-start .navbar-link:hover,.navbar.is-link .navbar-start>a.navbar-item.is-active,.navbar.is-link .navbar-start>a.navbar-item:focus,.navbar.is-link .navbar-start>a.navbar-item:hover{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-end .navbar-link::after,.navbar.is-link .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-link .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-link .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-link .navbar-item.has-dropdown:hover .navbar-link{background-color:#2366d1;color:#fff}.navbar.is-link .navbar-dropdown a.navbar-item.is-active{background-color:#3273dc;color:#fff}}.navbar.is-info{background-color:#3298dc;color:#fff}.navbar.is-info .navbar-brand .navbar-link,.navbar.is-info .navbar-brand>.navbar-item{color:#fff}.navbar.is-info .navbar-brand .navbar-link.is-active,.navbar.is-info .navbar-brand .navbar-link:focus,.navbar.is-info .navbar-brand .navbar-link:hover,.navbar.is-info .navbar-brand>a.navbar-item.is-active,.navbar.is-info .navbar-brand>a.navbar-item:focus,.navbar.is-info .navbar-brand>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-info .navbar-end .navbar-link,.navbar.is-info .navbar-end>.navbar-item,.navbar.is-info .navbar-start .navbar-link,.navbar.is-info .navbar-start>.navbar-item{color:#fff}.navbar.is-info .navbar-end .navbar-link.is-active,.navbar.is-info .navbar-end .navbar-link:focus,.navbar.is-info .navbar-end .navbar-link:hover,.navbar.is-info .navbar-end>a.navbar-item.is-active,.navbar.is-info .navbar-end>a.navbar-item:focus,.navbar.is-info .navbar-end>a.navbar-item:hover,.navbar.is-info .navbar-start .navbar-link.is-active,.navbar.is-info .navbar-start .navbar-link:focus,.navbar.is-info .navbar-start .navbar-link:hover,.navbar.is-info .navbar-start>a.navbar-item.is-active,.navbar.is-info .navbar-start>a.navbar-item:focus,.navbar.is-info .navbar-start>a.navbar-item:hover{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-end .navbar-link::after,.navbar.is-info .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-info .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-info .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-info .navbar-item.has-dropdown:hover .navbar-link{background-color:#238cd1;color:#fff}.navbar.is-info .navbar-dropdown a.navbar-item.is-active{background-color:#3298dc;color:#fff}}.navbar.is-success{background-color:#48c774;color:#fff}.navbar.is-success .navbar-brand .navbar-link,.navbar.is-success .navbar-brand>.navbar-item{color:#fff}.navbar.is-success .navbar-brand .navbar-link.is-active,.navbar.is-success .navbar-brand .navbar-link:focus,.navbar.is-success .navbar-brand .navbar-link:hover,.navbar.is-success .navbar-brand>a.navbar-item.is-active,.navbar.is-success .navbar-brand>a.navbar-item:focus,.navbar.is-success .navbar-brand>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-success .navbar-end .navbar-link,.navbar.is-success .navbar-end>.navbar-item,.navbar.is-success .navbar-start .navbar-link,.navbar.is-success .navbar-start>.navbar-item{color:#fff}.navbar.is-success .navbar-end .navbar-link.is-active,.navbar.is-success .navbar-end .navbar-link:focus,.navbar.is-success .navbar-end .navbar-link:hover,.navbar.is-success .navbar-end>a.navbar-item.is-active,.navbar.is-success .navbar-end>a.navbar-item:focus,.navbar.is-success .navbar-end>a.navbar-item:hover,.navbar.is-success .navbar-start .navbar-link.is-active,.navbar.is-success .navbar-start .navbar-link:focus,.navbar.is-success .navbar-start .navbar-link:hover,.navbar.is-success .navbar-start>a.navbar-item.is-active,.navbar.is-success .navbar-start>a.navbar-item:focus,.navbar.is-success .navbar-start>a.navbar-item:hover{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-end .navbar-link::after,.navbar.is-success .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-success .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-success .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-success .navbar-item.has-dropdown:hover .navbar-link{background-color:#3abb67;color:#fff}.navbar.is-success .navbar-dropdown a.navbar-item.is-active{background-color:#48c774;color:#fff}}.navbar.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link,.navbar.is-warning .navbar-brand>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link.is-active,.navbar.is-warning .navbar-brand .navbar-link:focus,.navbar.is-warning .navbar-brand .navbar-link:hover,.navbar.is-warning .navbar-brand>a.navbar-item.is-active,.navbar.is-warning .navbar-brand>a.navbar-item:focus,.navbar.is-warning .navbar-brand>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-brand .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-burger{color:rgba(0,0,0,.7)}@media screen and (min-width:1024px){.navbar.is-warning .navbar-end .navbar-link,.navbar.is-warning .navbar-end>.navbar-item,.navbar.is-warning .navbar-start .navbar-link,.navbar.is-warning .navbar-start>.navbar-item{color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link.is-active,.navbar.is-warning .navbar-end .navbar-link:focus,.navbar.is-warning .navbar-end .navbar-link:hover,.navbar.is-warning .navbar-end>a.navbar-item.is-active,.navbar.is-warning .navbar-end>a.navbar-item:focus,.navbar.is-warning .navbar-end>a.navbar-item:hover,.navbar.is-warning .navbar-start .navbar-link.is-active,.navbar.is-warning .navbar-start .navbar-link:focus,.navbar.is-warning .navbar-start .navbar-link:hover,.navbar.is-warning .navbar-start>a.navbar-item.is-active,.navbar.is-warning .navbar-start>a.navbar-item:focus,.navbar.is-warning .navbar-start>a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-end .navbar-link::after,.navbar.is-warning .navbar-start .navbar-link::after{border-color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-warning .navbar-item.has-dropdown:hover .navbar-link{background-color:#ffd83d;color:rgba(0,0,0,.7)}.navbar.is-warning .navbar-dropdown a.navbar-item.is-active{background-color:#ffdd57;color:rgba(0,0,0,.7)}}.navbar.is-danger{background-color:#f14668;color:#fff}.navbar.is-danger .navbar-brand .navbar-link,.navbar.is-danger .navbar-brand>.navbar-item{color:#fff}.navbar.is-danger .navbar-brand .navbar-link.is-active,.navbar.is-danger .navbar-brand .navbar-link:focus,.navbar.is-danger .navbar-brand .navbar-link:hover,.navbar.is-danger .navbar-brand>a.navbar-item.is-active,.navbar.is-danger .navbar-brand>a.navbar-item:focus,.navbar.is-danger .navbar-brand>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-brand .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-burger{color:#fff}@media screen and (min-width:1024px){.navbar.is-danger .navbar-end .navbar-link,.navbar.is-danger .navbar-end>.navbar-item,.navbar.is-danger .navbar-start .navbar-link,.navbar.is-danger .navbar-start>.navbar-item{color:#fff}.navbar.is-danger .navbar-end .navbar-link.is-active,.navbar.is-danger .navbar-end .navbar-link:focus,.navbar.is-danger .navbar-end .navbar-link:hover,.navbar.is-danger .navbar-end>a.navbar-item.is-active,.navbar.is-danger .navbar-end>a.navbar-item:focus,.navbar.is-danger .navbar-end>a.navbar-item:hover,.navbar.is-danger .navbar-start .navbar-link.is-active,.navbar.is-danger .navbar-start .navbar-link:focus,.navbar.is-danger .navbar-start .navbar-link:hover,.navbar.is-danger .navbar-start>a.navbar-item.is-active,.navbar.is-danger .navbar-start>a.navbar-item:focus,.navbar.is-danger .navbar-start>a.navbar-item:hover{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-end .navbar-link::after,.navbar.is-danger .navbar-start .navbar-link::after{border-color:#fff}.navbar.is-danger .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:focus .navbar-link,.navbar.is-danger .navbar-item.has-dropdown:hover .navbar-link{background-color:#ef2e55;color:#fff}.navbar.is-danger .navbar-dropdown a.navbar-item.is-active{background-color:#f14668;color:#fff}}.navbar>.container{align-items:stretch;display:flex;min-height:3.25rem;width:100%}.navbar.has-shadow{box-shadow:0 2px 0 0 #f5f5f5}.navbar.is-fixed-bottom,.navbar.is-fixed-top{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom{bottom:0}.navbar.is-fixed-bottom.has-shadow{box-shadow:0 -2px 0 0 #f5f5f5}.navbar.is-fixed-top{top:0}body.has-navbar-fixed-top,html.has-navbar-fixed-top{padding-top:3.25rem}body.has-navbar-fixed-bottom,html.has-navbar-fixed-bottom{padding-bottom:3.25rem}.navbar-brand,.navbar-tabs{align-items:stretch;display:flex;flex-shrink:0;min-height:3.25rem}.navbar-brand a.navbar-item:focus,.navbar-brand a.navbar-item:hover{background-color:transparent}.navbar-tabs{-webkit-overflow-scrolling:touch;max-width:100vw;overflow-x:auto;overflow-y:hidden}.navbar-burger{color:#4a4a4a;cursor:pointer;display:block;height:3.25rem;position:relative;width:3.25rem;margin-left:auto}.navbar-burger span{background-color:currentColor;display:block;height:1px;left:calc(50% - 8px);position:absolute;transform-origin:center;transition-duration:86ms;transition-property:background-color,opacity,transform;transition-timing-function:ease-out;width:16px}.navbar-burger span:nth-child(1){top:calc(50% - 6px)}.navbar-burger span:nth-child(2){top:calc(50% - 1px)}.navbar-burger span:nth-child(3){top:calc(50% + 4px)}.navbar-burger:hover{background-color:rgba(0,0,0,.05)}.navbar-burger.is-active span:nth-child(1){transform:translateY(5px) rotate(45deg)}.navbar-burger.is-active span:nth-child(2){opacity:0}.navbar-burger.is-active span:nth-child(3){transform:translateY(-5px) rotate(-45deg)}.navbar-menu{display:none}.navbar-item,.navbar-link{color:#4a4a4a;display:block;line-height:1.5;padding:.5rem .75rem;position:relative}.navbar-item .icon:only-child,.navbar-link .icon:only-child{margin-left:-.25rem;margin-right:-.25rem}.navbar-link,a.navbar-item{cursor:pointer}.navbar-link.is-active,.navbar-link:focus,.navbar-link:focus-within,.navbar-link:hover,a.navbar-item.is-active,a.navbar-item:focus,a.navbar-item:focus-within,a.navbar-item:hover{background-color:#fafafa;color:#3273dc}.navbar-item{display:block;flex-grow:0;flex-shrink:0}.navbar-item img{max-height:1.75rem}.navbar-item.has-dropdown{padding:0}.navbar-item.is-expanded{flex-grow:1;flex-shrink:1}.navbar-item.is-tab{border-bottom:1px solid transparent;min-height:3.25rem;padding-bottom:calc(.5rem - 1px)}.navbar-item.is-tab:focus,.navbar-item.is-tab:hover{background-color:transparent;border-bottom-color:#3273dc}.navbar-item.is-tab.is-active{background-color:transparent;border-bottom-color:#3273dc;border-bottom-style:solid;border-bottom-width:3px;color:#3273dc;padding-bottom:calc(.5rem - 3px)}.navbar-content{flex-grow:1;flex-shrink:1}.navbar-link:not(.is-arrowless){padding-right:2.5em}.navbar-link:not(.is-arrowless)::after{border-color:#3273dc;margin-top:-.375em;right:1.125em}.navbar-dropdown{font-size:.875rem;padding-bottom:.5rem;padding-top:.5rem}.navbar-dropdown .navbar-item{padding-left:1.5rem;padding-right:1.5rem}.navbar-divider{background-color:#f5f5f5;border:none;display:none;height:2px;margin:.5rem 0}@media screen and (max-width:1023px){.navbar>.container{display:block}.navbar-brand .navbar-item,.navbar-tabs .navbar-item{align-items:center;display:flex}.navbar-link::after{display:none}.navbar-menu{background-color:#fff;box-shadow:0 8px 16px rgba(10,10,10,.1);padding:.5rem 0}.navbar-menu.is-active{display:block}.navbar.is-fixed-bottom-touch,.navbar.is-fixed-top-touch{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-touch{bottom:0}.navbar.is-fixed-bottom-touch.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-touch{top:0}.navbar.is-fixed-top .navbar-menu,.navbar.is-fixed-top-touch .navbar-menu{-webkit-overflow-scrolling:touch;max-height:calc(100vh - 3.25rem);overflow:auto}body.has-navbar-fixed-top-touch,html.has-navbar-fixed-top-touch{padding-top:3.25rem}body.has-navbar-fixed-bottom-touch,html.has-navbar-fixed-bottom-touch{padding-bottom:3.25rem}}@media screen and (min-width:1024px){.navbar,.navbar-end,.navbar-menu,.navbar-start{align-items:stretch;display:flex}.navbar{min-height:3.25rem}.navbar.is-spaced{padding:1rem 2rem}.navbar.is-spaced .navbar-end,.navbar.is-spaced .navbar-start{align-items:center}.navbar.is-spaced .navbar-link,.navbar.is-spaced a.navbar-item{border-radius:4px}.navbar.is-transparent .navbar-link.is-active,.navbar.is-transparent .navbar-link:focus,.navbar.is-transparent .navbar-link:hover,.navbar.is-transparent a.navbar-item.is-active,.navbar.is-transparent a.navbar-item:focus,.navbar.is-transparent a.navbar-item:hover{background-color:transparent!important}.navbar.is-transparent .navbar-item.has-dropdown.is-active .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:focus-within .navbar-link,.navbar.is-transparent .navbar-item.has-dropdown.is-hoverable:hover .navbar-link{background-color:transparent!important}.navbar.is-transparent .navbar-dropdown a.navbar-item:focus,.navbar.is-transparent .navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar.is-transparent .navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-burger{display:none}.navbar-item,.navbar-link{align-items:center;display:flex}.navbar-item{display:flex}.navbar-item.has-dropdown{align-items:stretch}.navbar-item.has-dropdown-up .navbar-link::after{transform:rotate(135deg) translate(.25em,-.25em)}.navbar-item.has-dropdown-up .navbar-dropdown{border-bottom:2px solid #dbdbdb;border-radius:6px 6px 0 0;border-top:none;bottom:100%;box-shadow:0 -8px 8px rgba(10,10,10,.1);top:auto}.navbar-item.is-active .navbar-dropdown,.navbar-item.is-hoverable:focus .navbar-dropdown,.navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar-item.is-hoverable:hover .navbar-dropdown{display:block}.navbar-item.is-active .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:focus-within .navbar-dropdown.is-boxed,.navbar-item.is-hoverable:hover .navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-item.is-active .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:focus-within .navbar-dropdown,.navbar.is-spaced .navbar-item.is-hoverable:hover .navbar-dropdown{opacity:1;pointer-events:auto;transform:translateY(0)}.navbar-menu{flex-grow:1;flex-shrink:0}.navbar-start{justify-content:flex-start;margin-right:auto}.navbar-end{justify-content:flex-end;margin-left:auto}.navbar-dropdown{background-color:#fff;border-bottom-left-radius:6px;border-bottom-right-radius:6px;border-top:2px solid #dbdbdb;box-shadow:0 8px 8px rgba(10,10,10,.1);display:none;font-size:.875rem;left:0;min-width:100%;position:absolute;top:100%;z-index:20}.navbar-dropdown .navbar-item{padding:.375rem 1rem;white-space:nowrap}.navbar-dropdown a.navbar-item{padding-right:3rem}.navbar-dropdown a.navbar-item:focus,.navbar-dropdown a.navbar-item:hover{background-color:#f5f5f5;color:#0a0a0a}.navbar-dropdown a.navbar-item.is-active{background-color:#f5f5f5;color:#3273dc}.navbar-dropdown.is-boxed,.navbar.is-spaced .navbar-dropdown{border-radius:6px;border-top:none;box-shadow:0 8px 8px rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.1);display:block;opacity:0;pointer-events:none;top:calc(100% + (-4px));transform:translateY(-5px);transition-duration:86ms;transition-property:opacity,transform}.navbar-dropdown.is-right{left:auto;right:0}.navbar-divider{display:block}.container>.navbar .navbar-brand,.navbar>.container .navbar-brand{margin-left:-.75rem}.container>.navbar .navbar-menu,.navbar>.container .navbar-menu{margin-right:-.75rem}.navbar.is-fixed-bottom-desktop,.navbar.is-fixed-top-desktop{left:0;position:fixed;right:0;z-index:30}.navbar.is-fixed-bottom-desktop{bottom:0}.navbar.is-fixed-bottom-desktop.has-shadow{box-shadow:0 -2px 3px rgba(10,10,10,.1)}.navbar.is-fixed-top-desktop{top:0}body.has-navbar-fixed-top-desktop,html.has-navbar-fixed-top-desktop{padding-top:3.25rem}body.has-navbar-fixed-bottom-desktop,html.has-navbar-fixed-bottom-desktop{padding-bottom:3.25rem}body.has-spaced-navbar-fixed-top,html.has-spaced-navbar-fixed-top{padding-top:5.25rem}body.has-spaced-navbar-fixed-bottom,html.has-spaced-navbar-fixed-bottom{padding-bottom:5.25rem}.navbar-link.is-active,a.navbar-item.is-active{color:#0a0a0a}.navbar-link.is-active:not(:focus):not(:hover),a.navbar-item.is-active:not(:focus):not(:hover){background-color:transparent}.navbar-item.has-dropdown.is-active .navbar-link,.navbar-item.has-dropdown:focus .navbar-link,.navbar-item.has-dropdown:hover .navbar-link{background-color:#fafafa}}.hero.is-fullheight-with-navbar{min-height:calc(100vh - 3.25rem)}.pagination{font-size:1rem;margin:-.25rem}.pagination.is-small{font-size:.75rem}.pagination.is-medium{font-size:1.25rem}.pagination.is-large{font-size:1.5rem}.pagination.is-rounded .pagination-next,.pagination.is-rounded .pagination-previous{padding-left:1em;padding-right:1em;border-radius:290486px}.pagination.is-rounded .pagination-link{border-radius:290486px}.pagination,.pagination-list{align-items:center;display:flex;justify-content:center;text-align:center}.pagination-ellipsis,.pagination-link,.pagination-next,.pagination-previous{font-size:1em;justify-content:center;margin:.25rem;padding-left:.5em;padding-right:.5em;text-align:center}.pagination-link,.pagination-next,.pagination-previous{border-color:#dbdbdb;color:#363636;min-width:2.5em}.pagination-link:hover,.pagination-next:hover,.pagination-previous:hover{border-color:#b5b5b5;color:#363636}.pagination-link:focus,.pagination-next:focus,.pagination-previous:focus{border-color:#3273dc}.pagination-link:active,.pagination-next:active,.pagination-previous:active{box-shadow:inset 0 1px 2px rgba(10,10,10,.2)}.pagination-link[disabled],.pagination-next[disabled],.pagination-previous[disabled]{background-color:#dbdbdb;border-color:#dbdbdb;box-shadow:none;color:#7a7a7a;opacity:.5}.pagination-next,.pagination-previous{padding-left:.75em;padding-right:.75em;white-space:nowrap}.pagination-link.is-current{background-color:#3273dc;border-color:#3273dc;color:#fff}.pagination-ellipsis{color:#b5b5b5;pointer-events:none}.pagination-list{flex-wrap:wrap}@media screen and (max-width:768px){.pagination{flex-wrap:wrap}.pagination-next,.pagination-previous{flex-grow:1;flex-shrink:1}.pagination-list li{flex-grow:1;flex-shrink:1}}@media screen and (min-width:769px),print{.pagination-list{flex-grow:1;flex-shrink:1;justify-content:flex-start;order:1}.pagination-previous{order:2}.pagination-next{order:3}.pagination{justify-content:space-between}.pagination.is-centered .pagination-previous{order:1}.pagination.is-centered .pagination-list{justify-content:center;order:2}.pagination.is-centered .pagination-next{order:3}.pagination.is-right .pagination-previous{order:1}.pagination.is-right .pagination-next{order:2}.pagination.is-right .pagination-list{justify-content:flex-end;order:3}}.panel{border-radius:6px;box-shadow:0 .5em 1em -.125em rgba(10,10,10,.1),0 0 0 1px rgba(10,10,10,.02);font-size:1rem}.panel:not(:last-child){margin-bottom:1.5rem}.panel.is-white .panel-heading{background-color:#fff;color:#0a0a0a}.panel.is-white .panel-tabs a.is-active{border-bottom-color:#fff}.panel.is-white .panel-block.is-active .panel-icon{color:#fff}.panel.is-black .panel-heading{background-color:#0a0a0a;color:#fff}.panel.is-black .panel-tabs a.is-active{border-bottom-color:#0a0a0a}.panel.is-black .panel-block.is-active .panel-icon{color:#0a0a0a}.panel.is-light .panel-heading{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.panel.is-light .panel-tabs a.is-active{border-bottom-color:#f5f5f5}.panel.is-light .panel-block.is-active .panel-icon{color:#f5f5f5}.panel.is-dark .panel-heading{background-color:#363636;color:#fff}.panel.is-dark .panel-tabs a.is-active{border-bottom-color:#363636}.panel.is-dark .panel-block.is-active .panel-icon{color:#363636}.panel.is-primary .panel-heading{background-color:#00d1b2;color:#fff}.panel.is-primary .panel-tabs a.is-active{border-bottom-color:#00d1b2}.panel.is-primary .panel-block.is-active .panel-icon{color:#00d1b2}.panel.is-link .panel-heading{background-color:#3273dc;color:#fff}.panel.is-link .panel-tabs a.is-active{border-bottom-color:#3273dc}.panel.is-link .panel-block.is-active .panel-icon{color:#3273dc}.panel.is-info .panel-heading{background-color:#3298dc;color:#fff}.panel.is-info .panel-tabs a.is-active{border-bottom-color:#3298dc}.panel.is-info .panel-block.is-active .panel-icon{color:#3298dc}.panel.is-success .panel-heading{background-color:#48c774;color:#fff}.panel.is-success .panel-tabs a.is-active{border-bottom-color:#48c774}.panel.is-success .panel-block.is-active .panel-icon{color:#48c774}.panel.is-warning .panel-heading{background-color:#ffdd57;color:rgba(0,0,0,.7)}.panel.is-warning .panel-tabs a.is-active{border-bottom-color:#ffdd57}.panel.is-warning .panel-block.is-active .panel-icon{color:#ffdd57}.panel.is-danger .panel-heading{background-color:#f14668;color:#fff}.panel.is-danger .panel-tabs a.is-active{border-bottom-color:#f14668}.panel.is-danger .panel-block.is-active .panel-icon{color:#f14668}.panel-block:not(:last-child),.panel-tabs:not(:last-child){border-bottom:1px solid #ededed}.panel-heading{background-color:#ededed;border-radius:6px 6px 0 0;color:#363636;font-size:1.25em;font-weight:700;line-height:1.25;padding:.75em 1em}.panel-tabs{align-items:flex-end;display:flex;font-size:.875em;justify-content:center}.panel-tabs a{border-bottom:1px solid #dbdbdb;margin-bottom:-1px;padding:.5em}.panel-tabs a.is-active{border-bottom-color:#4a4a4a;color:#363636}.panel-list a{color:#4a4a4a}.panel-list a:hover{color:#3273dc}.panel-block{align-items:center;color:#363636;display:flex;justify-content:flex-start;padding:.5em .75em}.panel-block input[type=checkbox]{margin-right:.75em}.panel-block>.control{flex-grow:1;flex-shrink:1;width:100%}.panel-block.is-wrapped{flex-wrap:wrap}.panel-block.is-active{border-left-color:#3273dc;color:#363636}.panel-block.is-active .panel-icon{color:#3273dc}.panel-block:last-child{border-bottom-left-radius:6px;border-bottom-right-radius:6px}a.panel-block,label.panel-block{cursor:pointer}a.panel-block:hover,label.panel-block:hover{background-color:#f5f5f5}.panel-icon{display:inline-block;font-size:14px;height:1em;line-height:1em;text-align:center;vertical-align:top;width:1em;color:#7a7a7a;margin-right:.75em}.panel-icon .fa{font-size:inherit;line-height:inherit}.tabs{-webkit-overflow-scrolling:touch;align-items:stretch;display:flex;font-size:1rem;justify-content:space-between;overflow:hidden;overflow-x:auto;white-space:nowrap}.tabs a{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;color:#4a4a4a;display:flex;justify-content:center;margin-bottom:-1px;padding:.5em 1em;vertical-align:top}.tabs a:hover{border-bottom-color:#363636;color:#363636}.tabs li{display:block}.tabs li.is-active a{border-bottom-color:#3273dc;color:#3273dc}.tabs ul{align-items:center;border-bottom-color:#dbdbdb;border-bottom-style:solid;border-bottom-width:1px;display:flex;flex-grow:1;flex-shrink:0;justify-content:flex-start}.tabs ul.is-left{padding-right:.75em}.tabs ul.is-center{flex:none;justify-content:center;padding-left:.75em;padding-right:.75em}.tabs ul.is-right{justify-content:flex-end;padding-left:.75em}.tabs .icon:first-child{margin-right:.5em}.tabs .icon:last-child{margin-left:.5em}.tabs.is-centered ul{justify-content:center}.tabs.is-right ul{justify-content:flex-end}.tabs.is-boxed a{border:1px solid transparent;border-radius:4px 4px 0 0}.tabs.is-boxed a:hover{background-color:#f5f5f5;border-bottom-color:#dbdbdb}.tabs.is-boxed li.is-active a{background-color:#fff;border-color:#dbdbdb;border-bottom-color:transparent!important}.tabs.is-fullwidth li{flex-grow:1;flex-shrink:0}.tabs.is-toggle a{border-color:#dbdbdb;border-style:solid;border-width:1px;margin-bottom:0;position:relative}.tabs.is-toggle a:hover{background-color:#f5f5f5;border-color:#b5b5b5;z-index:2}.tabs.is-toggle li+li{margin-left:-1px}.tabs.is-toggle li:first-child a{border-radius:4px 0 0 4px}.tabs.is-toggle li:last-child a{border-radius:0 4px 4px 0}.tabs.is-toggle li.is-active a{background-color:#3273dc;border-color:#3273dc;color:#fff;z-index:1}.tabs.is-toggle ul{border-bottom:none}.tabs.is-toggle.is-toggle-rounded li:first-child a{border-bottom-left-radius:290486px;border-top-left-radius:290486px;padding-left:1.25em}.tabs.is-toggle.is-toggle-rounded li:last-child a{border-bottom-right-radius:290486px;border-top-right-radius:290486px;padding-right:1.25em}.tabs.is-small{font-size:.75rem}.tabs.is-medium{font-size:1.25rem}.tabs.is-large{font-size:1.5rem}.column{display:block;flex-basis:0;flex-grow:1;flex-shrink:1;padding:.75rem}.columns.is-mobile>.column.is-narrow{flex:none}.columns.is-mobile>.column.is-full{flex:none;width:100%}.columns.is-mobile>.column.is-three-quarters{flex:none;width:75%}.columns.is-mobile>.column.is-two-thirds{flex:none;width:66.6666%}.columns.is-mobile>.column.is-half{flex:none;width:50%}.columns.is-mobile>.column.is-one-third{flex:none;width:33.3333%}.columns.is-mobile>.column.is-one-quarter{flex:none;width:25%}.columns.is-mobile>.column.is-one-fifth{flex:none;width:20%}.columns.is-mobile>.column.is-two-fifths{flex:none;width:40%}.columns.is-mobile>.column.is-three-fifths{flex:none;width:60%}.columns.is-mobile>.column.is-four-fifths{flex:none;width:80%}.columns.is-mobile>.column.is-offset-three-quarters{margin-left:75%}.columns.is-mobile>.column.is-offset-two-thirds{margin-left:66.6666%}.columns.is-mobile>.column.is-offset-half{margin-left:50%}.columns.is-mobile>.column.is-offset-one-third{margin-left:33.3333%}.columns.is-mobile>.column.is-offset-one-quarter{margin-left:25%}.columns.is-mobile>.column.is-offset-one-fifth{margin-left:20%}.columns.is-mobile>.column.is-offset-two-fifths{margin-left:40%}.columns.is-mobile>.column.is-offset-three-fifths{margin-left:60%}.columns.is-mobile>.column.is-offset-four-fifths{margin-left:80%}.columns.is-mobile>.column.is-0{flex:none;width:0%}.columns.is-mobile>.column.is-offset-0{margin-left:0}.columns.is-mobile>.column.is-1{flex:none;width:8.33333%}.columns.is-mobile>.column.is-offset-1{margin-left:8.33333%}.columns.is-mobile>.column.is-2{flex:none;width:16.66667%}.columns.is-mobile>.column.is-offset-2{margin-left:16.66667%}.columns.is-mobile>.column.is-3{flex:none;width:25%}.columns.is-mobile>.column.is-offset-3{margin-left:25%}.columns.is-mobile>.column.is-4{flex:none;width:33.33333%}.columns.is-mobile>.column.is-offset-4{margin-left:33.33333%}.columns.is-mobile>.column.is-5{flex:none;width:41.66667%}.columns.is-mobile>.column.is-offset-5{margin-left:41.66667%}.columns.is-mobile>.column.is-6{flex:none;width:50%}.columns.is-mobile>.column.is-offset-6{margin-left:50%}.columns.is-mobile>.column.is-7{flex:none;width:58.33333%}.columns.is-mobile>.column.is-offset-7{margin-left:58.33333%}.columns.is-mobile>.column.is-8{flex:none;width:66.66667%}.columns.is-mobile>.column.is-offset-8{margin-left:66.66667%}.columns.is-mobile>.column.is-9{flex:none;width:75%}.columns.is-mobile>.column.is-offset-9{margin-left:75%}.columns.is-mobile>.column.is-10{flex:none;width:83.33333%}.columns.is-mobile>.column.is-offset-10{margin-left:83.33333%}.columns.is-mobile>.column.is-11{flex:none;width:91.66667%}.columns.is-mobile>.column.is-offset-11{margin-left:91.66667%}.columns.is-mobile>.column.is-12{flex:none;width:100%}.columns.is-mobile>.column.is-offset-12{margin-left:100%}@media screen and (max-width:768px){.column.is-narrow-mobile{flex:none}.column.is-full-mobile{flex:none;width:100%}.column.is-three-quarters-mobile{flex:none;width:75%}.column.is-two-thirds-mobile{flex:none;width:66.6666%}.column.is-half-mobile{flex:none;width:50%}.column.is-one-third-mobile{flex:none;width:33.3333%}.column.is-one-quarter-mobile{flex:none;width:25%}.column.is-one-fifth-mobile{flex:none;width:20%}.column.is-two-fifths-mobile{flex:none;width:40%}.column.is-three-fifths-mobile{flex:none;width:60%}.column.is-four-fifths-mobile{flex:none;width:80%}.column.is-offset-three-quarters-mobile{margin-left:75%}.column.is-offset-two-thirds-mobile{margin-left:66.6666%}.column.is-offset-half-mobile{margin-left:50%}.column.is-offset-one-third-mobile{margin-left:33.3333%}.column.is-offset-one-quarter-mobile{margin-left:25%}.column.is-offset-one-fifth-mobile{margin-left:20%}.column.is-offset-two-fifths-mobile{margin-left:40%}.column.is-offset-three-fifths-mobile{margin-left:60%}.column.is-offset-four-fifths-mobile{margin-left:80%}.column.is-0-mobile{flex:none;width:0%}.column.is-offset-0-mobile{margin-left:0}.column.is-1-mobile{flex:none;width:8.33333%}.column.is-offset-1-mobile{margin-left:8.33333%}.column.is-2-mobile{flex:none;width:16.66667%}.column.is-offset-2-mobile{margin-left:16.66667%}.column.is-3-mobile{flex:none;width:25%}.column.is-offset-3-mobile{margin-left:25%}.column.is-4-mobile{flex:none;width:33.33333%}.column.is-offset-4-mobile{margin-left:33.33333%}.column.is-5-mobile{flex:none;width:41.66667%}.column.is-offset-5-mobile{margin-left:41.66667%}.column.is-6-mobile{flex:none;width:50%}.column.is-offset-6-mobile{margin-left:50%}.column.is-7-mobile{flex:none;width:58.33333%}.column.is-offset-7-mobile{margin-left:58.33333%}.column.is-8-mobile{flex:none;width:66.66667%}.column.is-offset-8-mobile{margin-left:66.66667%}.column.is-9-mobile{flex:none;width:75%}.column.is-offset-9-mobile{margin-left:75%}.column.is-10-mobile{flex:none;width:83.33333%}.column.is-offset-10-mobile{margin-left:83.33333%}.column.is-11-mobile{flex:none;width:91.66667%}.column.is-offset-11-mobile{margin-left:91.66667%}.column.is-12-mobile{flex:none;width:100%}.column.is-offset-12-mobile{margin-left:100%}}@media screen and (min-width:769px),print{.column.is-narrow,.column.is-narrow-tablet{flex:none}.column.is-full,.column.is-full-tablet{flex:none;width:100%}.column.is-three-quarters,.column.is-three-quarters-tablet{flex:none;width:75%}.column.is-two-thirds,.column.is-two-thirds-tablet{flex:none;width:66.6666%}.column.is-half,.column.is-half-tablet{flex:none;width:50%}.column.is-one-third,.column.is-one-third-tablet{flex:none;width:33.3333%}.column.is-one-quarter,.column.is-one-quarter-tablet{flex:none;width:25%}.column.is-one-fifth,.column.is-one-fifth-tablet{flex:none;width:20%}.column.is-two-fifths,.column.is-two-fifths-tablet{flex:none;width:40%}.column.is-three-fifths,.column.is-three-fifths-tablet{flex:none;width:60%}.column.is-four-fifths,.column.is-four-fifths-tablet{flex:none;width:80%}.column.is-offset-three-quarters,.column.is-offset-three-quarters-tablet{margin-left:75%}.column.is-offset-two-thirds,.column.is-offset-two-thirds-tablet{margin-left:66.6666%}.column.is-offset-half,.column.is-offset-half-tablet{margin-left:50%}.column.is-offset-one-third,.column.is-offset-one-third-tablet{margin-left:33.3333%}.column.is-offset-one-quarter,.column.is-offset-one-quarter-tablet{margin-left:25%}.column.is-offset-one-fifth,.column.is-offset-one-fifth-tablet{margin-left:20%}.column.is-offset-two-fifths,.column.is-offset-two-fifths-tablet{margin-left:40%}.column.is-offset-three-fifths,.column.is-offset-three-fifths-tablet{margin-left:60%}.column.is-offset-four-fifths,.column.is-offset-four-fifths-tablet{margin-left:80%}.column.is-0,.column.is-0-tablet{flex:none;width:0%}.column.is-offset-0,.column.is-offset-0-tablet{margin-left:0}.column.is-1,.column.is-1-tablet{flex:none;width:8.33333%}.column.is-offset-1,.column.is-offset-1-tablet{margin-left:8.33333%}.column.is-2,.column.is-2-tablet{flex:none;width:16.66667%}.column.is-offset-2,.column.is-offset-2-tablet{margin-left:16.66667%}.column.is-3,.column.is-3-tablet{flex:none;width:25%}.column.is-offset-3,.column.is-offset-3-tablet{margin-left:25%}.column.is-4,.column.is-4-tablet{flex:none;width:33.33333%}.column.is-offset-4,.column.is-offset-4-tablet{margin-left:33.33333%}.column.is-5,.column.is-5-tablet{flex:none;width:41.66667%}.column.is-offset-5,.column.is-offset-5-tablet{margin-left:41.66667%}.column.is-6,.column.is-6-tablet{flex:none;width:50%}.column.is-offset-6,.column.is-offset-6-tablet{margin-left:50%}.column.is-7,.column.is-7-tablet{flex:none;width:58.33333%}.column.is-offset-7,.column.is-offset-7-tablet{margin-left:58.33333%}.column.is-8,.column.is-8-tablet{flex:none;width:66.66667%}.column.is-offset-8,.column.is-offset-8-tablet{margin-left:66.66667%}.column.is-9,.column.is-9-tablet{flex:none;width:75%}.column.is-offset-9,.column.is-offset-9-tablet{margin-left:75%}.column.is-10,.column.is-10-tablet{flex:none;width:83.33333%}.column.is-offset-10,.column.is-offset-10-tablet{margin-left:83.33333%}.column.is-11,.column.is-11-tablet{flex:none;width:91.66667%}.column.is-offset-11,.column.is-offset-11-tablet{margin-left:91.66667%}.column.is-12,.column.is-12-tablet{flex:none;width:100%}.column.is-offset-12,.column.is-offset-12-tablet{margin-left:100%}}@media screen and (max-width:1023px){.column.is-narrow-touch{flex:none}.column.is-full-touch{flex:none;width:100%}.column.is-three-quarters-touch{flex:none;width:75%}.column.is-two-thirds-touch{flex:none;width:66.6666%}.column.is-half-touch{flex:none;width:50%}.column.is-one-third-touch{flex:none;width:33.3333%}.column.is-one-quarter-touch{flex:none;width:25%}.column.is-one-fifth-touch{flex:none;width:20%}.column.is-two-fifths-touch{flex:none;width:40%}.column.is-three-fifths-touch{flex:none;width:60%}.column.is-four-fifths-touch{flex:none;width:80%}.column.is-offset-three-quarters-touch{margin-left:75%}.column.is-offset-two-thirds-touch{margin-left:66.6666%}.column.is-offset-half-touch{margin-left:50%}.column.is-offset-one-third-touch{margin-left:33.3333%}.column.is-offset-one-quarter-touch{margin-left:25%}.column.is-offset-one-fifth-touch{margin-left:20%}.column.is-offset-two-fifths-touch{margin-left:40%}.column.is-offset-three-fifths-touch{margin-left:60%}.column.is-offset-four-fifths-touch{margin-left:80%}.column.is-0-touch{flex:none;width:0%}.column.is-offset-0-touch{margin-left:0}.column.is-1-touch{flex:none;width:8.33333%}.column.is-offset-1-touch{margin-left:8.33333%}.column.is-2-touch{flex:none;width:16.66667%}.column.is-offset-2-touch{margin-left:16.66667%}.column.is-3-touch{flex:none;width:25%}.column.is-offset-3-touch{margin-left:25%}.column.is-4-touch{flex:none;width:33.33333%}.column.is-offset-4-touch{margin-left:33.33333%}.column.is-5-touch{flex:none;width:41.66667%}.column.is-offset-5-touch{margin-left:41.66667%}.column.is-6-touch{flex:none;width:50%}.column.is-offset-6-touch{margin-left:50%}.column.is-7-touch{flex:none;width:58.33333%}.column.is-offset-7-touch{margin-left:58.33333%}.column.is-8-touch{flex:none;width:66.66667%}.column.is-offset-8-touch{margin-left:66.66667%}.column.is-9-touch{flex:none;width:75%}.column.is-offset-9-touch{margin-left:75%}.column.is-10-touch{flex:none;width:83.33333%}.column.is-offset-10-touch{margin-left:83.33333%}.column.is-11-touch{flex:none;width:91.66667%}.column.is-offset-11-touch{margin-left:91.66667%}.column.is-12-touch{flex:none;width:100%}.column.is-offset-12-touch{margin-left:100%}}@media screen and (min-width:1024px){.column.is-narrow-desktop{flex:none}.column.is-full-desktop{flex:none;width:100%}.column.is-three-quarters-desktop{flex:none;width:75%}.column.is-two-thirds-desktop{flex:none;width:66.6666%}.column.is-half-desktop{flex:none;width:50%}.column.is-one-third-desktop{flex:none;width:33.3333%}.column.is-one-quarter-desktop{flex:none;width:25%}.column.is-one-fifth-desktop{flex:none;width:20%}.column.is-two-fifths-desktop{flex:none;width:40%}.column.is-three-fifths-desktop{flex:none;width:60%}.column.is-four-fifths-desktop{flex:none;width:80%}.column.is-offset-three-quarters-desktop{margin-left:75%}.column.is-offset-two-thirds-desktop{margin-left:66.6666%}.column.is-offset-half-desktop{margin-left:50%}.column.is-offset-one-third-desktop{margin-left:33.3333%}.column.is-offset-one-quarter-desktop{margin-left:25%}.column.is-offset-one-fifth-desktop{margin-left:20%}.column.is-offset-two-fifths-desktop{margin-left:40%}.column.is-offset-three-fifths-desktop{margin-left:60%}.column.is-offset-four-fifths-desktop{margin-left:80%}.column.is-0-desktop{flex:none;width:0%}.column.is-offset-0-desktop{margin-left:0}.column.is-1-desktop{flex:none;width:8.33333%}.column.is-offset-1-desktop{margin-left:8.33333%}.column.is-2-desktop{flex:none;width:16.66667%}.column.is-offset-2-desktop{margin-left:16.66667%}.column.is-3-desktop{flex:none;width:25%}.column.is-offset-3-desktop{margin-left:25%}.column.is-4-desktop{flex:none;width:33.33333%}.column.is-offset-4-desktop{margin-left:33.33333%}.column.is-5-desktop{flex:none;width:41.66667%}.column.is-offset-5-desktop{margin-left:41.66667%}.column.is-6-desktop{flex:none;width:50%}.column.is-offset-6-desktop{margin-left:50%}.column.is-7-desktop{flex:none;width:58.33333%}.column.is-offset-7-desktop{margin-left:58.33333%}.column.is-8-desktop{flex:none;width:66.66667%}.column.is-offset-8-desktop{margin-left:66.66667%}.column.is-9-desktop{flex:none;width:75%}.column.is-offset-9-desktop{margin-left:75%}.column.is-10-desktop{flex:none;width:83.33333%}.column.is-offset-10-desktop{margin-left:83.33333%}.column.is-11-desktop{flex:none;width:91.66667%}.column.is-offset-11-desktop{margin-left:91.66667%}.column.is-12-desktop{flex:none;width:100%}.column.is-offset-12-desktop{margin-left:100%}}@media screen and (min-width:1216px){.column.is-narrow-widescreen{flex:none}.column.is-full-widescreen{flex:none;width:100%}.column.is-three-quarters-widescreen{flex:none;width:75%}.column.is-two-thirds-widescreen{flex:none;width:66.6666%}.column.is-half-widescreen{flex:none;width:50%}.column.is-one-third-widescreen{flex:none;width:33.3333%}.column.is-one-quarter-widescreen{flex:none;width:25%}.column.is-one-fifth-widescreen{flex:none;width:20%}.column.is-two-fifths-widescreen{flex:none;width:40%}.column.is-three-fifths-widescreen{flex:none;width:60%}.column.is-four-fifths-widescreen{flex:none;width:80%}.column.is-offset-three-quarters-widescreen{margin-left:75%}.column.is-offset-two-thirds-widescreen{margin-left:66.6666%}.column.is-offset-half-widescreen{margin-left:50%}.column.is-offset-one-third-widescreen{margin-left:33.3333%}.column.is-offset-one-quarter-widescreen{margin-left:25%}.column.is-offset-one-fifth-widescreen{margin-left:20%}.column.is-offset-two-fifths-widescreen{margin-left:40%}.column.is-offset-three-fifths-widescreen{margin-left:60%}.column.is-offset-four-fifths-widescreen{margin-left:80%}.column.is-0-widescreen{flex:none;width:0%}.column.is-offset-0-widescreen{margin-left:0}.column.is-1-widescreen{flex:none;width:8.33333%}.column.is-offset-1-widescreen{margin-left:8.33333%}.column.is-2-widescreen{flex:none;width:16.66667%}.column.is-offset-2-widescreen{margin-left:16.66667%}.column.is-3-widescreen{flex:none;width:25%}.column.is-offset-3-widescreen{margin-left:25%}.column.is-4-widescreen{flex:none;width:33.33333%}.column.is-offset-4-widescreen{margin-left:33.33333%}.column.is-5-widescreen{flex:none;width:41.66667%}.column.is-offset-5-widescreen{margin-left:41.66667%}.column.is-6-widescreen{flex:none;width:50%}.column.is-offset-6-widescreen{margin-left:50%}.column.is-7-widescreen{flex:none;width:58.33333%}.column.is-offset-7-widescreen{margin-left:58.33333%}.column.is-8-widescreen{flex:none;width:66.66667%}.column.is-offset-8-widescreen{margin-left:66.66667%}.column.is-9-widescreen{flex:none;width:75%}.column.is-offset-9-widescreen{margin-left:75%}.column.is-10-widescreen{flex:none;width:83.33333%}.column.is-offset-10-widescreen{margin-left:83.33333%}.column.is-11-widescreen{flex:none;width:91.66667%}.column.is-offset-11-widescreen{margin-left:91.66667%}.column.is-12-widescreen{flex:none;width:100%}.column.is-offset-12-widescreen{margin-left:100%}}@media screen and (min-width:1408px){.column.is-narrow-fullhd{flex:none}.column.is-full-fullhd{flex:none;width:100%}.column.is-three-quarters-fullhd{flex:none;width:75%}.column.is-two-thirds-fullhd{flex:none;width:66.6666%}.column.is-half-fullhd{flex:none;width:50%}.column.is-one-third-fullhd{flex:none;width:33.3333%}.column.is-one-quarter-fullhd{flex:none;width:25%}.column.is-one-fifth-fullhd{flex:none;width:20%}.column.is-two-fifths-fullhd{flex:none;width:40%}.column.is-three-fifths-fullhd{flex:none;width:60%}.column.is-four-fifths-fullhd{flex:none;width:80%}.column.is-offset-three-quarters-fullhd{margin-left:75%}.column.is-offset-two-thirds-fullhd{margin-left:66.6666%}.column.is-offset-half-fullhd{margin-left:50%}.column.is-offset-one-third-fullhd{margin-left:33.3333%}.column.is-offset-one-quarter-fullhd{margin-left:25%}.column.is-offset-one-fifth-fullhd{margin-left:20%}.column.is-offset-two-fifths-fullhd{margin-left:40%}.column.is-offset-three-fifths-fullhd{margin-left:60%}.column.is-offset-four-fifths-fullhd{margin-left:80%}.column.is-0-fullhd{flex:none;width:0%}.column.is-offset-0-fullhd{margin-left:0}.column.is-1-fullhd{flex:none;width:8.33333%}.column.is-offset-1-fullhd{margin-left:8.33333%}.column.is-2-fullhd{flex:none;width:16.66667%}.column.is-offset-2-fullhd{margin-left:16.66667%}.column.is-3-fullhd{flex:none;width:25%}.column.is-offset-3-fullhd{margin-left:25%}.column.is-4-fullhd{flex:none;width:33.33333%}.column.is-offset-4-fullhd{margin-left:33.33333%}.column.is-5-fullhd{flex:none;width:41.66667%}.column.is-offset-5-fullhd{margin-left:41.66667%}.column.is-6-fullhd{flex:none;width:50%}.column.is-offset-6-fullhd{margin-left:50%}.column.is-7-fullhd{flex:none;width:58.33333%}.column.is-offset-7-fullhd{margin-left:58.33333%}.column.is-8-fullhd{flex:none;width:66.66667%}.column.is-offset-8-fullhd{margin-left:66.66667%}.column.is-9-fullhd{flex:none;width:75%}.column.is-offset-9-fullhd{margin-left:75%}.column.is-10-fullhd{flex:none;width:83.33333%}.column.is-offset-10-fullhd{margin-left:83.33333%}.column.is-11-fullhd{flex:none;width:91.66667%}.column.is-offset-11-fullhd{margin-left:91.66667%}.column.is-12-fullhd{flex:none;width:100%}.column.is-offset-12-fullhd{margin-left:100%}}.columns{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.columns:last-child{margin-bottom:-.75rem}.columns:not(:last-child){margin-bottom:calc(1.5rem - .75rem)}.columns.is-centered{justify-content:center}.columns.is-gapless{margin-left:0;margin-right:0;margin-top:0}.columns.is-gapless>.column{margin:0;padding:0!important}.columns.is-gapless:not(:last-child){margin-bottom:1.5rem}.columns.is-gapless:last-child{margin-bottom:0}.columns.is-mobile{display:flex}.columns.is-multiline{flex-wrap:wrap}.columns.is-vcentered{align-items:center}@media screen and (min-width:769px),print{.columns:not(.is-desktop){display:flex}}@media screen and (min-width:1024px){.columns.is-desktop{display:flex}}.columns.is-variable{--columnGap:0.75rem;margin-left:calc(-1 * var(--columnGap));margin-right:calc(-1 * var(--columnGap))}.columns.is-variable .column{padding-left:var(--columnGap);padding-right:var(--columnGap)}.columns.is-variable.is-0{--columnGap:0rem}@media screen and (max-width:768px){.columns.is-variable.is-0-mobile{--columnGap:0rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-0-tablet{--columnGap:0rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-0-tablet-only{--columnGap:0rem}}@media screen and (max-width:1023px){.columns.is-variable.is-0-touch{--columnGap:0rem}}@media screen and (min-width:1024px){.columns.is-variable.is-0-desktop{--columnGap:0rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-0-desktop-only{--columnGap:0rem}}@media screen and (min-width:1216px){.columns.is-variable.is-0-widescreen{--columnGap:0rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-0-widescreen-only{--columnGap:0rem}}@media screen and (min-width:1408px){.columns.is-variable.is-0-fullhd{--columnGap:0rem}}.columns.is-variable.is-1{--columnGap:0.25rem}@media screen and (max-width:768px){.columns.is-variable.is-1-mobile{--columnGap:0.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-1-tablet{--columnGap:0.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-1-tablet-only{--columnGap:0.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-1-touch{--columnGap:0.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-1-desktop{--columnGap:0.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-1-desktop-only{--columnGap:0.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-1-widescreen{--columnGap:0.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-1-widescreen-only{--columnGap:0.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-1-fullhd{--columnGap:0.25rem}}.columns.is-variable.is-2{--columnGap:0.5rem}@media screen and (max-width:768px){.columns.is-variable.is-2-mobile{--columnGap:0.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-2-tablet{--columnGap:0.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-2-tablet-only{--columnGap:0.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-2-touch{--columnGap:0.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-2-desktop{--columnGap:0.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-2-desktop-only{--columnGap:0.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-2-widescreen{--columnGap:0.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-2-widescreen-only{--columnGap:0.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-2-fullhd{--columnGap:0.5rem}}.columns.is-variable.is-3{--columnGap:0.75rem}@media screen and (max-width:768px){.columns.is-variable.is-3-mobile{--columnGap:0.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-3-tablet{--columnGap:0.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-3-tablet-only{--columnGap:0.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-3-touch{--columnGap:0.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-3-desktop{--columnGap:0.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-3-desktop-only{--columnGap:0.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-3-widescreen{--columnGap:0.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-3-widescreen-only{--columnGap:0.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-3-fullhd{--columnGap:0.75rem}}.columns.is-variable.is-4{--columnGap:1rem}@media screen and (max-width:768px){.columns.is-variable.is-4-mobile{--columnGap:1rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-4-tablet{--columnGap:1rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-4-tablet-only{--columnGap:1rem}}@media screen and (max-width:1023px){.columns.is-variable.is-4-touch{--columnGap:1rem}}@media screen and (min-width:1024px){.columns.is-variable.is-4-desktop{--columnGap:1rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-4-desktop-only{--columnGap:1rem}}@media screen and (min-width:1216px){.columns.is-variable.is-4-widescreen{--columnGap:1rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-4-widescreen-only{--columnGap:1rem}}@media screen and (min-width:1408px){.columns.is-variable.is-4-fullhd{--columnGap:1rem}}.columns.is-variable.is-5{--columnGap:1.25rem}@media screen and (max-width:768px){.columns.is-variable.is-5-mobile{--columnGap:1.25rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-5-tablet{--columnGap:1.25rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-5-tablet-only{--columnGap:1.25rem}}@media screen and (max-width:1023px){.columns.is-variable.is-5-touch{--columnGap:1.25rem}}@media screen and (min-width:1024px){.columns.is-variable.is-5-desktop{--columnGap:1.25rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-5-desktop-only{--columnGap:1.25rem}}@media screen and (min-width:1216px){.columns.is-variable.is-5-widescreen{--columnGap:1.25rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-5-widescreen-only{--columnGap:1.25rem}}@media screen and (min-width:1408px){.columns.is-variable.is-5-fullhd{--columnGap:1.25rem}}.columns.is-variable.is-6{--columnGap:1.5rem}@media screen and (max-width:768px){.columns.is-variable.is-6-mobile{--columnGap:1.5rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-6-tablet{--columnGap:1.5rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-6-tablet-only{--columnGap:1.5rem}}@media screen and (max-width:1023px){.columns.is-variable.is-6-touch{--columnGap:1.5rem}}@media screen and (min-width:1024px){.columns.is-variable.is-6-desktop{--columnGap:1.5rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-6-desktop-only{--columnGap:1.5rem}}@media screen and (min-width:1216px){.columns.is-variable.is-6-widescreen{--columnGap:1.5rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-6-widescreen-only{--columnGap:1.5rem}}@media screen and (min-width:1408px){.columns.is-variable.is-6-fullhd{--columnGap:1.5rem}}.columns.is-variable.is-7{--columnGap:1.75rem}@media screen and (max-width:768px){.columns.is-variable.is-7-mobile{--columnGap:1.75rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-7-tablet{--columnGap:1.75rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-7-tablet-only{--columnGap:1.75rem}}@media screen and (max-width:1023px){.columns.is-variable.is-7-touch{--columnGap:1.75rem}}@media screen and (min-width:1024px){.columns.is-variable.is-7-desktop{--columnGap:1.75rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-7-desktop-only{--columnGap:1.75rem}}@media screen and (min-width:1216px){.columns.is-variable.is-7-widescreen{--columnGap:1.75rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-7-widescreen-only{--columnGap:1.75rem}}@media screen and (min-width:1408px){.columns.is-variable.is-7-fullhd{--columnGap:1.75rem}}.columns.is-variable.is-8{--columnGap:2rem}@media screen and (max-width:768px){.columns.is-variable.is-8-mobile{--columnGap:2rem}}@media screen and (min-width:769px),print{.columns.is-variable.is-8-tablet{--columnGap:2rem}}@media screen and (min-width:769px) and (max-width:1023px){.columns.is-variable.is-8-tablet-only{--columnGap:2rem}}@media screen and (max-width:1023px){.columns.is-variable.is-8-touch{--columnGap:2rem}}@media screen and (min-width:1024px){.columns.is-variable.is-8-desktop{--columnGap:2rem}}@media screen and (min-width:1024px) and (max-width:1215px){.columns.is-variable.is-8-desktop-only{--columnGap:2rem}}@media screen and (min-width:1216px){.columns.is-variable.is-8-widescreen{--columnGap:2rem}}@media screen and (min-width:1216px) and (max-width:1407px){.columns.is-variable.is-8-widescreen-only{--columnGap:2rem}}@media screen and (min-width:1408px){.columns.is-variable.is-8-fullhd{--columnGap:2rem}}.tile{align-items:stretch;display:block;flex-basis:0;flex-grow:1;flex-shrink:1;min-height:-webkit-min-content;min-height:-moz-min-content;min-height:min-content}.tile.is-ancestor{margin-left:-.75rem;margin-right:-.75rem;margin-top:-.75rem}.tile.is-ancestor:last-child{margin-bottom:-.75rem}.tile.is-ancestor:not(:last-child){margin-bottom:.75rem}.tile.is-child{margin:0!important}.tile.is-parent{padding:.75rem}.tile.is-vertical{flex-direction:column}.tile.is-vertical>.tile.is-child:not(:last-child){margin-bottom:1.5rem!important}@media screen and (min-width:769px),print{.tile:not(.is-child){display:flex}.tile.is-1{flex:none;width:8.33333%}.tile.is-2{flex:none;width:16.66667%}.tile.is-3{flex:none;width:25%}.tile.is-4{flex:none;width:33.33333%}.tile.is-5{flex:none;width:41.66667%}.tile.is-6{flex:none;width:50%}.tile.is-7{flex:none;width:58.33333%}.tile.is-8{flex:none;width:66.66667%}.tile.is-9{flex:none;width:75%}.tile.is-10{flex:none;width:83.33333%}.tile.is-11{flex:none;width:91.66667%}.tile.is-12{flex:none;width:100%}}.hero{align-items:stretch;display:flex;flex-direction:column;justify-content:space-between}.hero .navbar{background:0 0}.hero .tabs ul{border-bottom:none}.hero.is-white{background-color:#fff;color:#0a0a0a}.hero.is-white a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-white strong{color:inherit}.hero.is-white .title{color:#0a0a0a}.hero.is-white .subtitle{color:rgba(10,10,10,.9)}.hero.is-white .subtitle a:not(.button),.hero.is-white .subtitle strong{color:#0a0a0a}@media screen and (max-width:1023px){.hero.is-white .navbar-menu{background-color:#fff}}.hero.is-white .navbar-item,.hero.is-white .navbar-link{color:rgba(10,10,10,.7)}.hero.is-white .navbar-link.is-active,.hero.is-white .navbar-link:hover,.hero.is-white a.navbar-item.is-active,.hero.is-white a.navbar-item:hover{background-color:#f2f2f2;color:#0a0a0a}.hero.is-white .tabs a{color:#0a0a0a;opacity:.9}.hero.is-white .tabs a:hover{opacity:1}.hero.is-white .tabs li.is-active a{opacity:1}.hero.is-white .tabs.is-boxed a,.hero.is-white .tabs.is-toggle a{color:#0a0a0a}.hero.is-white .tabs.is-boxed a:hover,.hero.is-white .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-white .tabs.is-boxed li.is-active a,.hero.is-white .tabs.is-boxed li.is-active a:hover,.hero.is-white .tabs.is-toggle li.is-active a,.hero.is-white .tabs.is-toggle li.is-active a:hover{background-color:#0a0a0a;border-color:#0a0a0a;color:#fff}.hero.is-white.is-bold{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-white.is-bold .navbar-menu{background-image:linear-gradient(141deg,#e6e6e6 0,#fff 71%,#fff 100%)}}.hero.is-black{background-color:#0a0a0a;color:#fff}.hero.is-black a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-black strong{color:inherit}.hero.is-black .title{color:#fff}.hero.is-black .subtitle{color:rgba(255,255,255,.9)}.hero.is-black .subtitle a:not(.button),.hero.is-black .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-black .navbar-menu{background-color:#0a0a0a}}.hero.is-black .navbar-item,.hero.is-black .navbar-link{color:rgba(255,255,255,.7)}.hero.is-black .navbar-link.is-active,.hero.is-black .navbar-link:hover,.hero.is-black a.navbar-item.is-active,.hero.is-black a.navbar-item:hover{background-color:#000;color:#fff}.hero.is-black .tabs a{color:#fff;opacity:.9}.hero.is-black .tabs a:hover{opacity:1}.hero.is-black .tabs li.is-active a{opacity:1}.hero.is-black .tabs.is-boxed a,.hero.is-black .tabs.is-toggle a{color:#fff}.hero.is-black .tabs.is-boxed a:hover,.hero.is-black .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-black .tabs.is-boxed li.is-active a,.hero.is-black .tabs.is-boxed li.is-active a:hover,.hero.is-black .tabs.is-toggle li.is-active a,.hero.is-black .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#0a0a0a}.hero.is-black.is-bold{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}@media screen and (max-width:768px){.hero.is-black.is-bold .navbar-menu{background-image:linear-gradient(141deg,#000 0,#0a0a0a 71%,#181616 100%)}}.hero.is-light{background-color:#f5f5f5;color:rgba(0,0,0,.7)}.hero.is-light a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-light strong{color:inherit}.hero.is-light .title{color:rgba(0,0,0,.7)}.hero.is-light .subtitle{color:rgba(0,0,0,.9)}.hero.is-light .subtitle a:not(.button),.hero.is-light .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-light .navbar-menu{background-color:#f5f5f5}}.hero.is-light .navbar-item,.hero.is-light .navbar-link{color:rgba(0,0,0,.7)}.hero.is-light .navbar-link.is-active,.hero.is-light .navbar-link:hover,.hero.is-light a.navbar-item.is-active,.hero.is-light a.navbar-item:hover{background-color:#e8e8e8;color:rgba(0,0,0,.7)}.hero.is-light .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-light .tabs a:hover{opacity:1}.hero.is-light .tabs li.is-active a{opacity:1}.hero.is-light .tabs.is-boxed a,.hero.is-light .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-light .tabs.is-boxed a:hover,.hero.is-light .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-light .tabs.is-boxed li.is-active a,.hero.is-light .tabs.is-boxed li.is-active a:hover,.hero.is-light .tabs.is-toggle li.is-active a,.hero.is-light .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#f5f5f5}.hero.is-light.is-bold{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}@media screen and (max-width:768px){.hero.is-light.is-bold .navbar-menu{background-image:linear-gradient(141deg,#dfd8d9 0,#f5f5f5 71%,#fff 100%)}}.hero.is-dark{background-color:#363636;color:#fff}.hero.is-dark a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-dark strong{color:inherit}.hero.is-dark .title{color:#fff}.hero.is-dark .subtitle{color:rgba(255,255,255,.9)}.hero.is-dark .subtitle a:not(.button),.hero.is-dark .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-dark .navbar-menu{background-color:#363636}}.hero.is-dark .navbar-item,.hero.is-dark .navbar-link{color:rgba(255,255,255,.7)}.hero.is-dark .navbar-link.is-active,.hero.is-dark .navbar-link:hover,.hero.is-dark a.navbar-item.is-active,.hero.is-dark a.navbar-item:hover{background-color:#292929;color:#fff}.hero.is-dark .tabs a{color:#fff;opacity:.9}.hero.is-dark .tabs a:hover{opacity:1}.hero.is-dark .tabs li.is-active a{opacity:1}.hero.is-dark .tabs.is-boxed a,.hero.is-dark .tabs.is-toggle a{color:#fff}.hero.is-dark .tabs.is-boxed a:hover,.hero.is-dark .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-dark .tabs.is-boxed li.is-active a,.hero.is-dark .tabs.is-boxed li.is-active a:hover,.hero.is-dark .tabs.is-toggle li.is-active a,.hero.is-dark .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#363636}.hero.is-dark.is-bold{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}@media screen and (max-width:768px){.hero.is-dark.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1f191a 0,#363636 71%,#46403f 100%)}}.hero.is-primary{background-color:#00d1b2;color:#fff}.hero.is-primary a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-primary strong{color:inherit}.hero.is-primary .title{color:#fff}.hero.is-primary .subtitle{color:rgba(255,255,255,.9)}.hero.is-primary .subtitle a:not(.button),.hero.is-primary .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-primary .navbar-menu{background-color:#00d1b2}}.hero.is-primary .navbar-item,.hero.is-primary .navbar-link{color:rgba(255,255,255,.7)}.hero.is-primary .navbar-link.is-active,.hero.is-primary .navbar-link:hover,.hero.is-primary a.navbar-item.is-active,.hero.is-primary a.navbar-item:hover{background-color:#00b89c;color:#fff}.hero.is-primary .tabs a{color:#fff;opacity:.9}.hero.is-primary .tabs a:hover{opacity:1}.hero.is-primary .tabs li.is-active a{opacity:1}.hero.is-primary .tabs.is-boxed a,.hero.is-primary .tabs.is-toggle a{color:#fff}.hero.is-primary .tabs.is-boxed a:hover,.hero.is-primary .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-primary .tabs.is-boxed li.is-active a,.hero.is-primary .tabs.is-boxed li.is-active a:hover,.hero.is-primary .tabs.is-toggle li.is-active a,.hero.is-primary .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#00d1b2}.hero.is-primary.is-bold{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}@media screen and (max-width:768px){.hero.is-primary.is-bold .navbar-menu{background-image:linear-gradient(141deg,#009e6c 0,#00d1b2 71%,#00e7eb 100%)}}.hero.is-link{background-color:#3273dc;color:#fff}.hero.is-link a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-link strong{color:inherit}.hero.is-link .title{color:#fff}.hero.is-link .subtitle{color:rgba(255,255,255,.9)}.hero.is-link .subtitle a:not(.button),.hero.is-link .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-link .navbar-menu{background-color:#3273dc}}.hero.is-link .navbar-item,.hero.is-link .navbar-link{color:rgba(255,255,255,.7)}.hero.is-link .navbar-link.is-active,.hero.is-link .navbar-link:hover,.hero.is-link a.navbar-item.is-active,.hero.is-link a.navbar-item:hover{background-color:#2366d1;color:#fff}.hero.is-link .tabs a{color:#fff;opacity:.9}.hero.is-link .tabs a:hover{opacity:1}.hero.is-link .tabs li.is-active a{opacity:1}.hero.is-link .tabs.is-boxed a,.hero.is-link .tabs.is-toggle a{color:#fff}.hero.is-link .tabs.is-boxed a:hover,.hero.is-link .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-link .tabs.is-boxed li.is-active a,.hero.is-link .tabs.is-boxed li.is-active a:hover,.hero.is-link .tabs.is-toggle li.is-active a,.hero.is-link .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3273dc}.hero.is-link.is-bold{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}@media screen and (max-width:768px){.hero.is-link.is-bold .navbar-menu{background-image:linear-gradient(141deg,#1577c6 0,#3273dc 71%,#4366e5 100%)}}.hero.is-info{background-color:#3298dc;color:#fff}.hero.is-info a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-info strong{color:inherit}.hero.is-info .title{color:#fff}.hero.is-info .subtitle{color:rgba(255,255,255,.9)}.hero.is-info .subtitle a:not(.button),.hero.is-info .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-info .navbar-menu{background-color:#3298dc}}.hero.is-info .navbar-item,.hero.is-info .navbar-link{color:rgba(255,255,255,.7)}.hero.is-info .navbar-link.is-active,.hero.is-info .navbar-link:hover,.hero.is-info a.navbar-item.is-active,.hero.is-info a.navbar-item:hover{background-color:#238cd1;color:#fff}.hero.is-info .tabs a{color:#fff;opacity:.9}.hero.is-info .tabs a:hover{opacity:1}.hero.is-info .tabs li.is-active a{opacity:1}.hero.is-info .tabs.is-boxed a,.hero.is-info .tabs.is-toggle a{color:#fff}.hero.is-info .tabs.is-boxed a:hover,.hero.is-info .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-info .tabs.is-boxed li.is-active a,.hero.is-info .tabs.is-boxed li.is-active a:hover,.hero.is-info .tabs.is-toggle li.is-active a,.hero.is-info .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#3298dc}.hero.is-info.is-bold{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}@media screen and (max-width:768px){.hero.is-info.is-bold .navbar-menu{background-image:linear-gradient(141deg,#159dc6 0,#3298dc 71%,#4389e5 100%)}}.hero.is-success{background-color:#48c774;color:#fff}.hero.is-success a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-success strong{color:inherit}.hero.is-success .title{color:#fff}.hero.is-success .subtitle{color:rgba(255,255,255,.9)}.hero.is-success .subtitle a:not(.button),.hero.is-success .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-success .navbar-menu{background-color:#48c774}}.hero.is-success .navbar-item,.hero.is-success .navbar-link{color:rgba(255,255,255,.7)}.hero.is-success .navbar-link.is-active,.hero.is-success .navbar-link:hover,.hero.is-success a.navbar-item.is-active,.hero.is-success a.navbar-item:hover{background-color:#3abb67;color:#fff}.hero.is-success .tabs a{color:#fff;opacity:.9}.hero.is-success .tabs a:hover{opacity:1}.hero.is-success .tabs li.is-active a{opacity:1}.hero.is-success .tabs.is-boxed a,.hero.is-success .tabs.is-toggle a{color:#fff}.hero.is-success .tabs.is-boxed a:hover,.hero.is-success .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-success .tabs.is-boxed li.is-active a,.hero.is-success .tabs.is-boxed li.is-active a:hover,.hero.is-success .tabs.is-toggle li.is-active a,.hero.is-success .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#48c774}.hero.is-success.is-bold{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}@media screen and (max-width:768px){.hero.is-success.is-bold .navbar-menu{background-image:linear-gradient(141deg,#29b342 0,#48c774 71%,#56d296 100%)}}.hero.is-warning{background-color:#ffdd57;color:rgba(0,0,0,.7)}.hero.is-warning a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-warning strong{color:inherit}.hero.is-warning .title{color:rgba(0,0,0,.7)}.hero.is-warning .subtitle{color:rgba(0,0,0,.9)}.hero.is-warning .subtitle a:not(.button),.hero.is-warning .subtitle strong{color:rgba(0,0,0,.7)}@media screen and (max-width:1023px){.hero.is-warning .navbar-menu{background-color:#ffdd57}}.hero.is-warning .navbar-item,.hero.is-warning .navbar-link{color:rgba(0,0,0,.7)}.hero.is-warning .navbar-link.is-active,.hero.is-warning .navbar-link:hover,.hero.is-warning a.navbar-item.is-active,.hero.is-warning a.navbar-item:hover{background-color:#ffd83d;color:rgba(0,0,0,.7)}.hero.is-warning .tabs a{color:rgba(0,0,0,.7);opacity:.9}.hero.is-warning .tabs a:hover{opacity:1}.hero.is-warning .tabs li.is-active a{opacity:1}.hero.is-warning .tabs.is-boxed a,.hero.is-warning .tabs.is-toggle a{color:rgba(0,0,0,.7)}.hero.is-warning .tabs.is-boxed a:hover,.hero.is-warning .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-warning .tabs.is-boxed li.is-active a,.hero.is-warning .tabs.is-boxed li.is-active a:hover,.hero.is-warning .tabs.is-toggle li.is-active a,.hero.is-warning .tabs.is-toggle li.is-active a:hover{background-color:rgba(0,0,0,.7);border-color:rgba(0,0,0,.7);color:#ffdd57}.hero.is-warning.is-bold{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}@media screen and (max-width:768px){.hero.is-warning.is-bold .navbar-menu{background-image:linear-gradient(141deg,#ffaf24 0,#ffdd57 71%,#fffa70 100%)}}.hero.is-danger{background-color:#f14668;color:#fff}.hero.is-danger a:not(.button):not(.dropdown-item):not(.tag):not(.pagination-link.is-current),.hero.is-danger strong{color:inherit}.hero.is-danger .title{color:#fff}.hero.is-danger .subtitle{color:rgba(255,255,255,.9)}.hero.is-danger .subtitle a:not(.button),.hero.is-danger .subtitle strong{color:#fff}@media screen and (max-width:1023px){.hero.is-danger .navbar-menu{background-color:#f14668}}.hero.is-danger .navbar-item,.hero.is-danger .navbar-link{color:rgba(255,255,255,.7)}.hero.is-danger .navbar-link.is-active,.hero.is-danger .navbar-link:hover,.hero.is-danger a.navbar-item.is-active,.hero.is-danger a.navbar-item:hover{background-color:#ef2e55;color:#fff}.hero.is-danger .tabs a{color:#fff;opacity:.9}.hero.is-danger .tabs a:hover{opacity:1}.hero.is-danger .tabs li.is-active a{opacity:1}.hero.is-danger .tabs.is-boxed a,.hero.is-danger .tabs.is-toggle a{color:#fff}.hero.is-danger .tabs.is-boxed a:hover,.hero.is-danger .tabs.is-toggle a:hover{background-color:rgba(10,10,10,.1)}.hero.is-danger .tabs.is-boxed li.is-active a,.hero.is-danger .tabs.is-boxed li.is-active a:hover,.hero.is-danger .tabs.is-toggle li.is-active a,.hero.is-danger .tabs.is-toggle li.is-active a:hover{background-color:#fff;border-color:#fff;color:#f14668}.hero.is-danger.is-bold{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}@media screen and (max-width:768px){.hero.is-danger.is-bold .navbar-menu{background-image:linear-gradient(141deg,#fa0a62 0,#f14668 71%,#f7595f 100%)}}.hero.is-small .hero-body{padding-bottom:1.5rem;padding-top:1.5rem}@media screen and (min-width:769px),print{.hero.is-medium .hero-body{padding-bottom:9rem;padding-top:9rem}}@media screen and (min-width:769px),print{.hero.is-large .hero-body{padding-bottom:18rem;padding-top:18rem}}.hero.is-fullheight .hero-body,.hero.is-fullheight-with-navbar .hero-body,.hero.is-halfheight .hero-body{align-items:center;display:flex}.hero.is-fullheight .hero-body>.container,.hero.is-fullheight-with-navbar .hero-body>.container,.hero.is-halfheight .hero-body>.container{flex-grow:1;flex-shrink:1}.hero.is-halfheight{min-height:50vh}.hero.is-fullheight{min-height:100vh}.hero-video{overflow:hidden}.hero-video video{left:50%;min-height:100%;min-width:100%;position:absolute;top:50%;transform:translate3d(-50%,-50%,0)}.hero-video.is-transparent{opacity:.3}@media screen and (max-width:768px){.hero-video{display:none}}.hero-buttons{margin-top:1.5rem}@media screen and (max-width:768px){.hero-buttons .button{display:flex}.hero-buttons .button:not(:last-child){margin-bottom:.75rem}}@media screen and (min-width:769px),print{.hero-buttons{display:flex;justify-content:center}.hero-buttons .button:not(:last-child){margin-right:1.5rem}}.hero-foot,.hero-head{flex-grow:0;flex-shrink:0}.hero-body{flex-grow:1;flex-shrink:0;padding:3rem 1.5rem}.section{padding:3rem 1.5rem}@media screen and (min-width:1024px){.section.is-medium{padding:9rem 1.5rem}.section.is-large{padding:18rem 1.5rem}}.footer{background-color:#fafafa;padding:3rem 1.5rem 6rem} \ No newline at end of file diff --git a/meilisearch-http/public/interface.html b/meilisearch-http/public/interface.html deleted file mode 100644 index bcdf5f176..000000000 --- a/meilisearch-http/public/interface.html +++ /dev/null @@ -1,364 +0,0 @@ - - - - - - - MeiliSearch - - - - -
- -
-
-
-

- Welcome to MeiliSearch -

-

- This dashboard will help you check the search results with ease. -

-
-
- -
-
-
- -
-
- - - -
-
- -
-
-
-
-
-
-

Documents

-

0

-
-
-

Time Spent

-

N/A

-
-
-
-
-
-
-
- -
-
-
    - -
-
-
- - - - diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index c9106496b..11347175b 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -1,12 +1,9 @@ use std::hash::{Hash, Hasher}; -use std::{error, thread}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use log::error; +use log::debug; use serde::Serialize; -use serde_qs as qs; use siphasher::sip::SipHasher; -use walkdir::WalkDir; use crate::Data; use crate::Opt; @@ -21,31 +18,21 @@ struct EventProperties { } impl EventProperties { - fn from(data: Data) -> Result> { - let mut index_list = Vec::new(); + async fn from(data: Data) -> anyhow::Result { + let stats = data.index_controller.get_all_stats().await?; - let reader = data.db.main_read_txn()?; - - for index_uid in data.db.indexes_uids() { - if let Some(index) = data.db.open_index(&index_uid) { - let number_of_documents = index.main.number_of_documents(&reader)?; - index_list.push(number_of_documents); - } - } - - let database_size = WalkDir::new(&data.db_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()); - - let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); + let database_size = stats.database_size; + let last_update_timestamp = stats.last_update.map(|u| u.timestamp()); + let number_of_documents = stats + .indexes + .values() + .map(|index| index.number_of_documents) + .collect(); Ok(EventProperties { database_size, last_update_timestamp, - number_of_documents: index_list, + number_of_documents, }) } } @@ -72,10 +59,10 @@ struct Event<'a> { #[derive(Debug, Serialize)] struct AmplitudeRequest<'a> { api_key: &'a str, - event: &'a str, + events: Vec>, } -pub fn analytics_sender(data: Data, opt: Opt) { +pub async fn analytics_sender(data: Data, opt: Opt) { let username = whoami::username(); let hostname = whoami::hostname(); let platform = whoami::platform(); @@ -97,7 +84,7 @@ pub fn analytics_sender(data: Data, opt: Opt) { let time = n.as_secs(); let event_type = "runtime_tick"; let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day - let event_properties = EventProperties::from(data.clone()).ok(); + let event_properties = EventProperties::from(data.clone()).await.ok(); let app_version = env!("CARGO_PKG_VERSION").to_string(); let app_version = app_version.as_str(); let user_email = std::env::var("MEILI_USER_EMAIL").ok(); @@ -116,27 +103,24 @@ pub fn analytics_sender(data: Data, opt: Opt) { time, app_version, user_properties, - event_properties + event_properties, }; - let event = serde_json::to_string(&event).unwrap(); let request = AmplitudeRequest { api_key: AMPLITUDE_API_KEY, - event: &event, + events: vec![event], }; - let body = qs::to_string(&request).unwrap(); - let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body); - match response { - Err(ureq::Error::Status(_ , response)) => { - error!("Unsuccessful call to Amplitude: {}", response.into_string().unwrap_or_default()); - } - Err(e) => { - error!("Unsuccessful call to Amplitude: {}", e); - } - _ => (), + let response = reqwest::Client::new() + .post("https://api2.amplitude.com/2/httpapi") + .timeout(Duration::from_secs(60)) // 1 minute max + .json(&request) + .send() + .await; + if let Err(e) = response { + debug!("Unsuccessful call to Amplitude: {}", e); } - thread::sleep(Duration::from_secs(3600)) // one hour + tokio::time::sleep(Duration::from_secs(3600)).await; } } diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs deleted file mode 100644 index 2deeab693..000000000 --- a/meilisearch-http/src/data.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::error::Error; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; - -use meilisearch_core::{Database, DatabaseOptions, Index}; -use sha2::Digest; - -use crate::error::{Error as MSError, ResponseError}; -use crate::index_update_callback; -use crate::option::Opt; -use crate::dump::DumpInfo; - -#[derive(Clone)] -pub struct Data { - inner: Arc, -} - -impl Deref for Data { - type Target = DataInner; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -#[derive(Clone)] -pub struct DataInner { - pub db: Arc, - pub db_path: String, - pub dumps_dir: PathBuf, - pub dump_batch_size: usize, - pub api_keys: ApiKeys, - pub server_pid: u32, - pub http_payload_size_limit: usize, - pub current_dump: Arc>>, -} - -#[derive(Clone)] -pub struct ApiKeys { - pub public: Option, - pub private: Option, - pub master: Option, -} - -impl ApiKeys { - pub fn generate_missing_api_keys(&mut self) { - if let Some(master_key) = &self.master { - if self.private.is_none() { - let key = format!("{}-private", master_key); - let sha = sha2::Sha256::digest(key.as_bytes()); - self.private = Some(format!("{:x}", sha)); - } - if self.public.is_none() { - let key = format!("{}-public", master_key); - let sha = sha2::Sha256::digest(key.as_bytes()); - self.public = Some(format!("{:x}", sha)); - } - } - } -} - -impl Data { - pub fn new(opt: Opt) -> Result> { - let db_path = opt.db_path.clone(); - let dumps_dir = opt.dumps_dir.clone(); - let dump_batch_size = opt.dump_batch_size; - let server_pid = std::process::id(); - - let db_opt = DatabaseOptions { - main_map_size: opt.max_mdb_size, - update_map_size: opt.max_udb_size, - }; - - let http_payload_size_limit = opt.http_payload_size_limit; - - let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?); - - let mut api_keys = ApiKeys { - master: opt.master_key, - private: None, - public: None, - }; - - api_keys.generate_missing_api_keys(); - - let current_dump = Arc::new(Mutex::new(None)); - - let inner_data = DataInner { - db: db.clone(), - db_path, - dumps_dir, - dump_batch_size, - api_keys, - server_pid, - http_payload_size_limit, - current_dump, - }; - - let data = Data { - inner: Arc::new(inner_data), - }; - - let callback_context = data.clone(); - db.set_update_callback(Box::new(move |index_uid, status| { - index_update_callback(&index_uid, &callback_context, status); - })); - - Ok(data) - } - - fn create_index(&self, uid: &str) -> Result { - if !uid - .chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') - { - return Err(MSError::InvalidIndexUid.into()); - } - - let created_index = self.db.create_index(&uid).map_err(|e| match e { - meilisearch_core::Error::IndexAlreadyExists => e.into(), - _ => ResponseError::from(MSError::create_index(e)), - })?; - - self.db.main_write::<_, _, ResponseError>(|mut writer| { - created_index.main.put_name(&mut writer, uid)?; - - created_index - .main - .created_at(&writer)? - .ok_or(MSError::internal("Impossible to read created at"))?; - - created_index - .main - .updated_at(&writer)? - .ok_or(MSError::internal("Impossible to read updated at"))?; - Ok(()) - })?; - - Ok(created_index) - } - - pub fn get_current_dump_info(&self) -> Option { - self.current_dump.lock().unwrap().clone() - } - - pub fn set_current_dump_info(&self, dump_info: DumpInfo) { - self.current_dump.lock().unwrap().replace(dump_info); - } - - pub fn get_or_create_index(&self, uid: &str, f: F) -> Result - where - F: FnOnce(&Index) -> Result, - { - let mut index_has_been_created = false; - - let index = match self.db.open_index(&uid) { - Some(index) => index, - None => { - index_has_been_created = true; - self.create_index(&uid)? - } - }; - - match f(&index) { - Ok(r) => Ok(r), - Err(err) => { - if index_has_been_created { - let _ = self.db.delete_index(&uid); - } - Err(err) - } - } - } -} diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs new file mode 100644 index 000000000..48dfcfa06 --- /dev/null +++ b/meilisearch-http/src/data/mod.rs @@ -0,0 +1,133 @@ +use std::ops::Deref; +use std::sync::Arc; + +use sha2::Digest; + +use crate::index::{Checked, Settings}; +use crate::index_controller::{ + error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats, +}; +use crate::option::Opt; + +pub mod search; +mod updates; + +#[derive(Clone)] +pub struct Data { + inner: Arc, +} + +impl Deref for Data { + type Target = DataInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +pub struct DataInner { + pub index_controller: IndexController, + pub api_keys: ApiKeys, + options: Opt, +} + +#[derive(Clone)] +pub struct ApiKeys { + pub public: Option, + pub private: Option, + pub master: Option, +} + +impl ApiKeys { + pub fn generate_missing_api_keys(&mut self) { + if let Some(master_key) = &self.master { + if self.private.is_none() { + let key = format!("{}-private", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.private = Some(format!("{:x}", sha)); + } + if self.public.is_none() { + let key = format!("{}-public", master_key); + let sha = sha2::Sha256::digest(key.as_bytes()); + self.public = Some(format!("{:x}", sha)); + } + } + } +} + +impl Data { + pub fn new(options: Opt) -> anyhow::Result { + let path = options.db_path.clone(); + + let index_controller = IndexController::new(&path, &options)?; + + let mut api_keys = ApiKeys { + master: options.clone().master_key, + private: None, + public: None, + }; + + api_keys.generate_missing_api_keys(); + + let inner = DataInner { + index_controller, + api_keys, + options, + }; + let inner = Arc::new(inner); + + Ok(Data { inner }) + } + + pub async fn settings(&self, uid: String) -> Result> { + self.index_controller.settings(uid).await + } + + pub async fn list_indexes(&self) -> Result> { + self.index_controller.list_indexes().await + } + + pub async fn index(&self, uid: String) -> Result { + self.index_controller.get_index(uid).await + } + + pub async fn create_index( + &self, + uid: String, + primary_key: Option, + ) -> Result { + let settings = IndexSettings { + uid: Some(uid), + primary_key, + }; + + let meta = self.index_controller.create_index(settings).await?; + Ok(meta) + } + + pub async fn get_index_stats(&self, uid: String) -> Result { + Ok(self.index_controller.get_index_stats(uid).await?) + } + + pub async fn get_all_stats(&self) -> Result { + Ok(self.index_controller.get_all_stats().await?) + } + + pub async fn create_dump(&self) -> Result { + Ok(self.index_controller.create_dump().await?) + } + + pub async fn dump_status(&self, uid: String) -> Result { + Ok(self.index_controller.dump_info(uid).await?) + } + + #[inline] + pub fn http_payload_size_limit(&self) -> usize { + self.options.http_payload_size_limit.get_bytes() as usize + } + + #[inline] + pub fn api_keys(&self) -> &ApiKeys { + &self.api_keys + } +} diff --git a/meilisearch-http/src/data/search.rs b/meilisearch-http/src/data/search.rs new file mode 100644 index 000000000..5ad8d4a07 --- /dev/null +++ b/meilisearch-http/src/data/search.rs @@ -0,0 +1,34 @@ +use serde_json::{Map, Value}; + +use super::Data; +use crate::index::{SearchQuery, SearchResult}; +use crate::index_controller::error::Result; + +impl Data { + pub async fn search(&self, index: String, search_query: SearchQuery) -> Result { + self.index_controller.search(index, search_query).await + } + + pub async fn retrieve_documents( + &self, + index: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result>> { + self.index_controller + .documents(index, offset, limit, attributes_to_retrieve) + .await + } + + pub async fn retrieve_document( + &self, + index: String, + document_id: String, + attributes_to_retrieve: Option>, + ) -> Result> { + self.index_controller + .document(index, document_id, attributes_to_retrieve) + .await + } +} diff --git a/meilisearch-http/src/data/updates.rs b/meilisearch-http/src/data/updates.rs new file mode 100644 index 000000000..4e38294e9 --- /dev/null +++ b/meilisearch-http/src/data/updates.rs @@ -0,0 +1,80 @@ +use milli::update::{IndexDocumentsMethod, UpdateFormat}; + +use crate::extractors::payload::Payload; +use crate::index::{Checked, Settings}; +use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus}; +use crate::Data; + +impl Data { + pub async fn add_documents( + &self, + index: String, + method: IndexDocumentsMethod, + format: UpdateFormat, + stream: Payload, + primary_key: Option, + ) -> Result { + let update_status = self + .index_controller + .add_documents(index, method, format, stream, primary_key) + .await?; + Ok(update_status) + } + + pub async fn update_settings( + &self, + index: String, + settings: Settings, + create: bool, + ) -> Result { + let update = self + .index_controller + .update_settings(index, settings, create) + .await?; + Ok(update) + } + + pub async fn clear_documents(&self, index: String) -> Result { + let update = self.index_controller.clear_documents(index).await?; + Ok(update) + } + + pub async fn delete_documents( + &self, + index: String, + document_ids: Vec, + ) -> Result { + let update = self + .index_controller + .delete_documents(index, document_ids) + .await?; + Ok(update) + } + + pub async fn delete_index(&self, index: String) -> Result<()> { + self.index_controller.delete_index(index).await?; + Ok(()) + } + + pub async fn get_update_status(&self, index: String, uid: u64) -> Result { + self.index_controller.update_status(index, uid).await + } + + pub async fn get_updates_status(&self, index: String) -> Result> { + self.index_controller.all_update_status(index).await + } + + pub async fn update_index( + &self, + uid: String, + primary_key: Option, + new_uid: Option, + ) -> Result { + let settings = IndexSettings { + uid: new_uid, + primary_key, + }; + + self.index_controller.update_index(uid, settings).await + } +} diff --git a/meilisearch-http/src/dump.rs b/meilisearch-http/src/dump.rs deleted file mode 100644 index bf5752830..000000000 --- a/meilisearch-http/src/dump.rs +++ /dev/null @@ -1,412 +0,0 @@ -use std::fs::{create_dir_all, File}; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use std::thread; - -use actix_web::web; -use chrono::offset::Utc; -use indexmap::IndexMap; -use log::{error, info}; -use meilisearch_core::{MainWriter, MainReader, UpdateReader}; -use meilisearch_core::settings::Settings; -use meilisearch_core::update::{apply_settings_update, apply_documents_addition}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tempfile::TempDir; - -use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::compression; -use crate::routes::index; -use crate::routes::index::IndexResponse; - -#[derive(Debug, Serialize, Deserialize, Copy, Clone)] -enum DumpVersion { - V1, -} - -impl DumpVersion { - const CURRENT: Self = Self::V1; -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DumpMetadata { - indexes: Vec, - db_version: String, - dump_version: DumpVersion, -} - -impl DumpMetadata { - /// Create a DumpMetadata with the current dump version of meilisearch. - pub fn new(indexes: Vec, db_version: String) -> Self { - DumpMetadata { - indexes, - db_version, - dump_version: DumpVersion::CURRENT, - } - } - - /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` - fn from_path(dir_path: &Path) -> Result { - let path = dir_path.join("metadata.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) - } - - /// Write DumpMetadata in `metadata.json` file at provided `dir_path` - fn to_path(&self, dir_path: &Path) -> Result<(), Error> { - let path = dir_path.join("metadata.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, &self)?; - - Ok(()) - } -} - -/// Extract Settings from `settings.json` file present at provided `dir_path` -fn settings_from_path(dir_path: &Path) -> Result { - let path = dir_path.join("settings.json"); - let file = File::open(path)?; - let reader = std::io::BufReader::new(file); - let metadata = serde_json::from_reader(reader)?; - - Ok(metadata) -} - -/// Write Settings in `settings.json` file at provided `dir_path` -fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> { - let path = dir_path.join("settings.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, settings)?; - - Ok(()) -} - -/// Import settings and documents of a dump with version `DumpVersion::V1` in specified index. -fn import_index_v1( - data: &Data, - dumps_dir: &Path, - index_uid: &str, - document_batch_size: usize, - write_txn: &mut MainWriter, -) -> Result<(), Error> { - - // open index - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - // index dir path in dump dir - let index_path = &dumps_dir.join(index_uid); - - // extract `settings.json` file and import content - let settings = settings_from_path(&index_path)?; - let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?; - apply_settings_update(write_txn, &index, settings)?; - - // create iterator over documents in `documents.jsonl` to make batch importation - // create iterator over documents in `documents.jsonl` to make batch importation - let documents = { - let file = File::open(&index_path.join("documents.jsonl"))?; - let reader = std::io::BufReader::new(file); - let deserializer = serde_json::Deserializer::from_reader(reader); - deserializer.into_iter::>() - }; - - // batch import document every `document_batch_size`: - // create a Vec to bufferize documents - let mut values = Vec::with_capacity(document_batch_size); - // iterate over documents - for document in documents { - // push document in buffer - values.push(document?); - // if buffer is full, create and apply a batch, and clean buffer - if values.len() == document_batch_size { - let batch = std::mem::replace(&mut values, Vec::with_capacity(document_batch_size)); - apply_documents_addition(write_txn, &index, batch, None)?; - } - } - - // apply documents remaining in the buffer - if !values.is_empty() { - apply_documents_addition(write_txn, &index, values, None)?; - } - - // sync index information: stats, updated_at, last_update - if let Err(e) = crate::index_update_callback_txn(index, index_uid, data, write_txn) { - return Err(Error::Internal(e)); - } - - Ok(()) -} - -/// Import dump from `dump_path` in database. -pub fn import_dump( - data: &Data, - dump_path: &Path, - document_batch_size: usize, -) -> Result<(), Error> { - info!("Importing dump from {:?}...", dump_path); - - // create a temporary directory - let tmp_dir = TempDir::new()?; - let tmp_dir_path = tmp_dir.path(); - - // extract dump in temporary directory - compression::from_tar_gz(dump_path, tmp_dir_path)?; - - // read dump metadata - let metadata = DumpMetadata::from_path(&tmp_dir_path)?; - - // choose importation function from DumpVersion of metadata - let import_index = match metadata.dump_version { - DumpVersion::V1 => import_index_v1, - }; - - // remove indexes which have same `uid` than indexes to import and create empty indexes - let existing_index_uids = data.db.indexes_uids(); - for index in metadata.indexes.iter() { - if existing_index_uids.contains(&index.uid) { - data.db.delete_index(index.uid.clone())?; - } - index::create_index_sync(&data.db, index.uid.clone(), index.name.clone(), index.primary_key.clone())?; - } - - // import each indexes content - data.db.main_write::<_, _, Error>(|mut writer| { - for index in metadata.indexes { - import_index(&data, tmp_dir_path, &index.uid, document_batch_size, &mut writer)?; - } - Ok(()) - })?; - - info!("Dump importation from {:?} succeed", dump_path); - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] -#[serde(rename_all = "snake_case")] -pub enum DumpStatus { - Done, - InProgress, - Failed, -} - -#[derive(Debug, Serialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct DumpInfo { - pub uid: String, - pub status: DumpStatus, - #[serde(skip_serializing_if = "Option::is_none", flatten)] - pub error: Option, - -} - -impl DumpInfo { - pub fn new(uid: String, status: DumpStatus) -> Self { - Self { uid, status, error: None } - } - - pub fn with_error(mut self, error: ResponseError) -> Self { - self.status = DumpStatus::Failed; - self.error = Some(json!(error)); - - self - } - - pub fn dump_already_in_progress(&self) -> bool { - self.status == DumpStatus::InProgress - } -} - -/// Generate uid from creation date -fn generate_uid() -> String { - Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() -} - -/// Infer dumps_dir from dump_uid -pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf { - dumps_dir.join(format!("{}.dump", dump_uid)) -} - -/// Write metadata in dump -fn dump_metadata(data: &web::Data, dir_path: &Path, indexes: Vec) -> Result<(), Error> { - let (db_major, db_minor, db_patch) = data.db.version(); - let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch)); - - metadata.to_path(dir_path) -} - -/// Export settings of provided index in dump -fn dump_index_settings(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?; - - settings_to_path(&settings, dir_path) -} - -/// Export updates of provided index in dump -fn dump_index_updates(data: &web::Data, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let updates_path = dir_path.join("updates.jsonl"); - let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?; - - let file = File::create(updates_path)?; - - for update in updates { - serde_json::to_writer(&file, &update)?; - writeln!(&file)?; - } - - Ok(()) -} - -/// Export documents of provided index in dump -fn dump_index_documents(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { - let documents_path = dir_path.join("documents.jsonl"); - let file = File::create(documents_path)?; - let dump_batch_size = data.dump_batch_size; - - let mut offset = 0; - loop { - let documents = crate::routes::document::get_all_documents_sync(data, reader, index_uid, offset, dump_batch_size, None)?; - if documents.is_empty() { break; } else { offset += dump_batch_size; } - - for document in documents { - serde_json::to_writer(&file, &document)?; - writeln!(&file)?; - } - } - - Ok(()) -} - -/// Write error with a context. -fn fail_dump_process(data: &web::Data, dump_info: DumpInfo, context: &str, error: E) { - let error_message = format!("{}; {}", context, error); - error!("Something went wrong during dump process: {}", &error_message); - data.set_current_dump_info(dump_info.with_error(Error::dump_failed(error_message).into())) -} - -/// Main function of dump. -fn dump_process(data: web::Data, dumps_dir: PathBuf, dump_info: DumpInfo) { - // open read transaction on Update - let update_reader = match data.db.update_read_txn() { - Ok(r) => r, - Err(e) => { - fail_dump_process(&data, dump_info, "creating RO transaction on updates", e); - return ; - } - }; - - // open read transaction on Main - let main_reader = match data.db.main_read_txn() { - Ok(r) => r, - Err(e) => { - fail_dump_process(&data, dump_info, "creating RO transaction on main", e); - return ; - } - }; - - // create a temporary directory - let tmp_dir = match TempDir::new() { - Ok(tmp_dir) => tmp_dir, - Err(e) => { - fail_dump_process(&data, dump_info, "creating temporary directory", e); - return ; - } - }; - let tmp_dir_path = tmp_dir.path(); - - // fetch indexes - let indexes = match crate::routes::index::list_indexes_sync(&data, &main_reader) { - Ok(indexes) => indexes, - Err(e) => { - fail_dump_process(&data, dump_info, "listing indexes", e); - return ; - } - }; - - // create metadata - if let Err(e) = dump_metadata(&data, &tmp_dir_path, indexes.clone()) { - fail_dump_process(&data, dump_info, "generating metadata", e); - return ; - } - - // export settings, updates and documents for each indexes - for index in indexes { - let index_path = tmp_dir_path.join(&index.uid); - - // create index sub-dircetory - if let Err(e) = create_dir_all(&index_path) { - fail_dump_process(&data, dump_info, &format!("creating directory for index {}", &index.uid), e); - return ; - } - - // export settings - if let Err(e) = dump_index_settings(&data, &main_reader, &index_path, &index.uid) { - fail_dump_process(&data, dump_info, &format!("generating settings for index {}", &index.uid), e); - return ; - } - - // export documents - if let Err(e) = dump_index_documents(&data, &main_reader, &index_path, &index.uid) { - fail_dump_process(&data, dump_info, &format!("generating documents for index {}", &index.uid), e); - return ; - } - - // export updates - if let Err(e) = dump_index_updates(&data, &update_reader, &index_path, &index.uid) { - fail_dump_process(&data, dump_info, &format!("generating updates for index {}", &index.uid), e); - return ; - } - } - - // compress dump in a file named `{dump_uid}.dump` in `dumps_dir` - if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) { - fail_dump_process(&data, dump_info, "compressing dump", e); - return ; - } - - // update dump info to `done` - let resume = DumpInfo::new( - dump_info.uid, - DumpStatus::Done - ); - - data.set_current_dump_info(resume); -} - -pub fn init_dump_process(data: &web::Data, dumps_dir: &Path) -> Result { - create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; - - // check if a dump is already in progress - if let Some(resume) = data.get_current_dump_info() { - if resume.dump_already_in_progress() { - return Err(Error::dump_conflict()) - } - } - - // generate a new dump info - let info = DumpInfo::new( - generate_uid(), - DumpStatus::InProgress - ); - - data.set_current_dump_info(info.clone()); - - let data = data.clone(); - let dumps_dir = dumps_dir.to_path_buf(); - let info_cloned = info.clone(); - // run dump process in a new thread - thread::spawn(move || - dump_process(data, dumps_dir, info_cloned) - ); - - Ok(info) -} diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index e779c5708..4f47abd66 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,307 +1,167 @@ -use std::error; +use std::error::Error; use std::fmt; -use actix_http::ResponseBuilder; use actix_web as aweb; -use actix_web::error::{JsonPayloadError, QueryPayloadError}; +use actix_web::body::Body; +use actix_web::dev::BaseHttpResponseBuilder; use actix_web::http::StatusCode; -use serde::ser::{Serialize, Serializer, SerializeStruct}; +use aweb::error::{JsonPayloadError, QueryPayloadError}; +use meilisearch_error::{Code, ErrorCode}; +use milli::UserError; +use serde::{Deserialize, Serialize}; -use meilisearch_error::{ErrorCode, Code}; - -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct ResponseError { - inner: Box, -} - -impl error::Error for ResponseError {} - -impl ErrorCode for ResponseError { - fn error_code(&self) -> Code { - self.inner.error_code() - } + #[serde(skip)] + code: StatusCode, + message: String, + error_code: String, + error_type: String, + error_link: String, } impl fmt::Display for ResponseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.fmt(f) + self.message.fmt(f) } } -impl From for ResponseError { - fn from(error: Error) -> ResponseError { - ResponseError { inner: Box::new(error) } - } -} - -impl From for ResponseError { - fn from(err: meilisearch_core::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - -impl From for ResponseError { - fn from(err: meilisearch_schema::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - -impl From for ResponseError { - fn from(err: FacetCountError) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - -impl Serialize for ResponseError { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let struct_name = "ResponseError"; - let field_count = 4; - - let mut state = serializer.serialize_struct(struct_name, field_count)?; - state.serialize_field("message", &self.to_string())?; - state.serialize_field("errorCode", &self.error_name())?; - state.serialize_field("errorType", &self.error_type())?; - state.serialize_field("errorLink", &self.error_url())?; - state.end() +impl From for ResponseError +where + T: ErrorCode, +{ + fn from(other: T) -> Self { + Self { + code: other.http_status(), + message: other.to_string(), + error_code: other.error_name(), + error_type: other.error_type(), + error_link: other.error_url(), + } } } impl aweb::error::ResponseError for ResponseError { - fn error_response(&self) -> aweb::HttpResponse { - ResponseBuilder::new(self.status_code()).json(&self) + fn error_response(&self) -> aweb::BaseHttpResponse { + let json = serde_json::to_vec(self).unwrap(); + BaseHttpResponseBuilder::new(self.status_code()) + .content_type("application/json") + .body(json) } fn status_code(&self) -> StatusCode { - self.http_status() + self.code + } +} + +macro_rules! internal_error { + ($target:ty : $($other:path), *) => { + $( + impl From<$other> for $target { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* } } #[derive(Debug)] -pub enum Error { - BadParameter(String, String), - BadRequest(String), - CreateIndex(String), - DocumentNotFound(String), - IndexNotFound(String), - IndexAlreadyExists(String), - Internal(String), - InvalidIndexUid, - InvalidToken(String), - MissingAuthorizationHeader, - NotFound(String), - OpenIndex(String), - RetrieveDocument(u32, String), - SearchDocuments(String), - PayloadTooLarge, - UnsupportedMediaType, - DumpAlreadyInProgress, - DumpProcessFailed(String), +pub struct MilliError<'a>(pub &'a milli::Error); + +impl Error for MilliError<'_> {} + +impl fmt::Display for MilliError<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } } -impl error::Error for Error {} - -impl ErrorCode for Error { +impl ErrorCode for MilliError<'_> { fn error_code(&self) -> Code { - use Error::*; + match self.0 { + milli::Error::InternalError(_) => Code::Internal, + milli::Error::IoError(_) => Code::Internal, + milli::Error::UserError(ref error) => { + match error { + // TODO: wait for spec for new error codes. + UserError::Csv(_) + | UserError::SerdeJson(_) + | UserError::MaxDatabaseSizeReached + | UserError::InvalidCriterionName { .. } + | UserError::InvalidDocumentId { .. } + | UserError::InvalidStoreFile + | UserError::NoSpaceLeftOnDevice + | UserError::DocumentLimitReached => Code::Internal, + UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded, + UserError::InvalidFilter(_) => Code::Filter, + UserError::InvalidFilterAttribute(_) => Code::Filter, + UserError::MissingDocumentId { .. } => Code::MissingDocumentId, + UserError::MissingPrimaryKey => Code::MissingPrimaryKey, + UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent, + UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent, + UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound, + UserError::InvalidFacetsDistribution { .. } => Code::BadRequest, + } + } + } + } +} + +impl fmt::Display for PayloadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - BadParameter(_, _) => Code::BadParameter, - BadRequest(_) => Code::BadRequest, - CreateIndex(_) => Code::CreateIndex, - DocumentNotFound(_) => Code::DocumentNotFound, - IndexNotFound(_) => Code::IndexNotFound, - IndexAlreadyExists(_) => Code::IndexAlreadyExists, - Internal(_) => Code::Internal, - InvalidIndexUid => Code::InvalidIndexUid, - InvalidToken(_) => Code::InvalidToken, - MissingAuthorizationHeader => Code::MissingAuthorizationHeader, - NotFound(_) => Code::NotFound, - OpenIndex(_) => Code::OpenIndex, - RetrieveDocument(_, _) => Code::RetrieveDocument, - SearchDocuments(_) => Code::SearchDocuments, - PayloadTooLarge => Code::PayloadTooLarge, - UnsupportedMediaType => Code::UnsupportedMediaType, - DumpAlreadyInProgress => Code::DumpAlreadyInProgress, - DumpProcessFailed(_) => Code::DumpProcessFailed, + PayloadError::Json(e) => e.fmt(f), + PayloadError::Query(e) => e.fmt(f), } } } #[derive(Debug)] -pub enum FacetCountError { - AttributeNotSet(String), - SyntaxError(String), - UnexpectedToken { found: String, expected: &'static [&'static str] }, - NoFacetSet, +pub enum PayloadError { + Json(JsonPayloadError), + Query(QueryPayloadError), } -impl error::Error for FacetCountError {} +impl Error for PayloadError {} -impl ErrorCode for FacetCountError { +impl ErrorCode for PayloadError { fn error_code(&self) -> Code { - Code::BadRequest - } -} - -impl FacetCountError { - pub fn unexpected_token(found: impl ToString, expected: &'static [&'static str]) -> FacetCountError { - let found = found.to_string(); - FacetCountError::UnexpectedToken { expected, found } - } -} - -impl From for FacetCountError { - fn from(other: serde_json::error::Error) -> FacetCountError { - FacetCountError::SyntaxError(other.to_string()) - } -} - -impl fmt::Display for FacetCountError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use FacetCountError::*; - match self { - AttributeNotSet(attr) => write!(f, "Attribute {} is not set as facet", attr), - SyntaxError(msg) => write!(f, "Syntax error: {}", msg), - UnexpectedToken { expected, found } => write!(f, "Unexpected {} found, expected {:?}", found, expected), - NoFacetSet => write!(f, "Can't perform facet count, as no facet is set"), + PayloadError::Json(err) => match err { + JsonPayloadError::Overflow => Code::PayloadTooLarge, + JsonPayloadError::ContentType => Code::UnsupportedMediaType, + JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => { + Code::PayloadTooLarge + } + JsonPayloadError::Deserialize(_) | JsonPayloadError::Payload(_) => Code::BadRequest, + JsonPayloadError::Serialize(_) => Code::Internal, + _ => Code::Internal, + }, + PayloadError::Query(err) => match err { + QueryPayloadError::Deserialize(_) => Code::BadRequest, + _ => Code::Internal, + }, } } } -impl Error { - pub fn internal(err: impl fmt::Display) -> Error { - Error::Internal(err.to_string()) - } - - pub fn bad_request(err: impl fmt::Display) -> Error { - Error::BadRequest(err.to_string()) - } - - pub fn missing_authorization_header() -> Error { - Error::MissingAuthorizationHeader - } - - pub fn invalid_token(err: impl fmt::Display) -> Error { - Error::InvalidToken(err.to_string()) - } - - pub fn not_found(err: impl fmt::Display) -> Error { - Error::NotFound(err.to_string()) - } - - pub fn index_not_found(err: impl fmt::Display) -> Error { - Error::IndexNotFound(err.to_string()) - } - - pub fn document_not_found(err: impl fmt::Display) -> Error { - Error::DocumentNotFound(err.to_string()) - } - - pub fn bad_parameter(param: impl fmt::Display, err: impl fmt::Display) -> Error { - Error::BadParameter(param.to_string(), err.to_string()) - } - - pub fn open_index(err: impl fmt::Display) -> Error { - Error::OpenIndex(err.to_string()) - } - - pub fn create_index(err: impl fmt::Display) -> Error { - Error::CreateIndex(err.to_string()) - } - - pub fn invalid_index_uid() -> Error { - Error::InvalidIndexUid - } - - pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error { - Error::RetrieveDocument(doc_id, err.to_string()) - } - - pub fn search_documents(err: impl fmt::Display) -> Error { - Error::SearchDocuments(err.to_string()) - } - - pub fn dump_conflict() -> Error { - Error::DumpAlreadyInProgress - } - - pub fn dump_failed(message: String) -> Error { - Error::DumpProcessFailed(message) +impl From for PayloadError { + fn from(other: JsonPayloadError) -> Self { + Self::Json(other) } } -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::BadParameter(param, err) => write!(f, "Url parameter {} error: {}", param, err), - Self::BadRequest(err) => f.write_str(err), - Self::CreateIndex(err) => write!(f, "Impossible to create index; {}", err), - Self::DocumentNotFound(document_id) => write!(f, "Document with id {} not found", document_id), - Self::IndexNotFound(index_uid) => write!(f, "Index {} not found", index_uid), - Self::IndexAlreadyExists(index_uid) => write!(f, "Index {} already exists", index_uid), - Self::Internal(err) => f.write_str(err), - Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."), - Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err), - Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"), - Self::NotFound(err) => write!(f, "{} not found", err), - Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err), - Self::RetrieveDocument(id, err) => write!(f, "Impossible to retrieve the document with id: {}; {}", id, err), - Self::SearchDocuments(err) => write!(f, "Impossible to search documents; {}", err), - Self::PayloadTooLarge => f.write_str("Payload too large"), - Self::UnsupportedMediaType => f.write_str("Unsupported media type"), - Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"), - Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message), - } +impl From for PayloadError { + fn from(other: QueryPayloadError) -> Self { + Self::Query(other) } } -impl From for Error { - fn from(err: std::io::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: actix_http::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: meilisearch_core::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: serde_json::error::Error) -> Error { - Error::Internal(err.to_string()) - } -} - -impl From for Error { - fn from(err: JsonPayloadError) -> Error { - match err { - JsonPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid JSON: {}", err)), - JsonPayloadError::Overflow => Error::PayloadTooLarge, - JsonPayloadError::ContentType => Error::UnsupportedMediaType, - JsonPayloadError::Payload(err) => Error::BadRequest(format!("Problem while decoding the request: {}", err)), - } - } -} - -impl From for Error { - fn from(err: QueryPayloadError) -> Error { - match err { - QueryPayloadError::Deserialize(err) => Error::BadRequest(format!("Invalid query parameters: {}", err)), - } - } -} - -pub fn payload_error_handler>(err: E) -> ResponseError { - let error: Error = err.into(); - error.into() +pub fn payload_error_handler(err: E) -> ResponseError +where + E: Into, +{ + err.into().into() } diff --git a/meilisearch-http/src/extractors/authentication/error.rs b/meilisearch-http/src/extractors/authentication/error.rs new file mode 100644 index 000000000..902634045 --- /dev/null +++ b/meilisearch-http/src/extractors/authentication/error.rs @@ -0,0 +1,25 @@ +use meilisearch_error::{Code, ErrorCode}; + +#[derive(Debug, thiserror::Error)] +pub enum AuthenticationError { + #[error("You must have an authorization token")] + MissingAuthorizationHeader, + #[error("Invalid API key")] + InvalidToken(String), + // Triggered on configuration error. + #[error("Irretrievable state")] + IrretrievableState, + #[error("Unknown authentication policy")] + UnknownPolicy, +} + +impl ErrorCode for AuthenticationError { + fn error_code(&self) -> Code { + match self { + AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, + AuthenticationError::InvalidToken(_) => Code::InvalidToken, + AuthenticationError::IrretrievableState => Code::Internal, + AuthenticationError::UnknownPolicy => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs new file mode 100644 index 000000000..13ced2248 --- /dev/null +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -0,0 +1,182 @@ +mod error; + +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use std::marker::PhantomData; +use std::ops::Deref; + +use actix_web::FromRequest; +use futures::future::err; +use futures::future::{ok, Ready}; + +use crate::error::ResponseError; +use error::AuthenticationError; + +macro_rules! create_policies { + ($($name:ident), *) => { + pub mod policies { + use std::collections::HashSet; + use crate::extractors::authentication::Policy; + + $( + #[derive(Debug, Default)] + pub struct $name { + inner: HashSet> + } + + impl $name { + pub fn new() -> Self { + Self { inner: HashSet::new() } + } + + pub fn add(&mut self, token: Vec) { + self.inner.insert(token); + } + } + + impl Policy for $name { + fn authenticate(&self, token: &[u8]) -> bool { + self.inner.contains(token) + } + } + )* + } + }; +} + +create_policies!(Public, Private, Admin); + +/// Instanciate a `Policies`, filled with the given policies. +macro_rules! init_policies { + ($($name:ident), *) => { + { + let mut policies = crate::extractors::authentication::Policies::new(); + $( + let policy = $name::new(); + policies.insert(policy); + )* + policies + } + }; +} + +/// Adds user to all specified policies. +macro_rules! create_users { + ($policies:ident, $($user:expr => { $($policy:ty), * }), *) => { + { + $( + $( + $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned())); + )* + )* + } + }; +} + +pub struct GuardedData { + data: D, + _marker: PhantomData, +} + +impl Deref for GuardedData { + type Target = D; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +pub trait Policy { + fn authenticate(&self, token: &[u8]) -> bool; +} + +#[derive(Debug)] +pub struct Policies { + inner: HashMap>, +} + +impl Policies { + pub fn new() -> Self { + Self { + inner: HashMap::new(), + } + } + + pub fn insert(&mut self, policy: S) { + self.inner.insert(TypeId::of::(), Box::new(policy)); + } + + pub fn get(&self) -> Option<&S> { + self.inner + .get(&TypeId::of::()) + .and_then(|p| p.downcast_ref::()) + } + + pub fn get_mut(&mut self) -> Option<&mut S> { + self.inner + .get_mut(&TypeId::of::()) + .and_then(|p| p.downcast_mut::()) + } +} + +impl Default for Policies { + fn default() -> Self { + Self::new() + } +} + +pub enum AuthConfig { + NoAuth, + Auth(Policies), +} + +impl Default for AuthConfig { + fn default() -> Self { + Self::NoAuth + } +} + +impl FromRequest for GuardedData { + type Config = AuthConfig; + + type Error = ResponseError; + + type Future = Ready>; + + fn from_request( + req: &actix_web::HttpRequest, + _payload: &mut actix_http::Payload, + ) -> Self::Future { + match req.app_data::() { + Some(config) => match config { + AuthConfig::NoAuth => match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => err(AuthenticationError::IrretrievableState.into()), + }, + AuthConfig::Auth(policies) => match policies.get::

() { + Some(policy) => match req.headers().get("x-meili-api-key") { + Some(token) => { + if policy.authenticate(token.as_bytes()) { + match req.app_data::().cloned() { + Some(data) => ok(Self { + data, + _marker: PhantomData, + }), + None => err(AuthenticationError::IrretrievableState.into()), + } + } else { + err(AuthenticationError::InvalidToken(String::from("hello")).into()) + } + } + None => err(AuthenticationError::MissingAuthorizationHeader.into()), + }, + None => err(AuthenticationError::UnknownPolicy.into()), + }, + }, + None => err(AuthenticationError::IrretrievableState.into()), + } + } +} diff --git a/meilisearch-http/src/extractors/mod.rs b/meilisearch-http/src/extractors/mod.rs new file mode 100644 index 000000000..09a56e4a0 --- /dev/null +++ b/meilisearch-http/src/extractors/mod.rs @@ -0,0 +1,3 @@ +pub mod payload; +#[macro_use] +pub mod authentication; diff --git a/meilisearch-http/src/extractors/payload.rs b/meilisearch-http/src/extractors/payload.rs new file mode 100644 index 000000000..260561e40 --- /dev/null +++ b/meilisearch-http/src/extractors/payload.rs @@ -0,0 +1,69 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use actix_http::error::PayloadError; +use actix_web::{dev, web, FromRequest, HttpRequest}; +use futures::future::{ready, Ready}; +use futures::Stream; + +pub struct Payload { + payload: dev::Payload, + limit: usize, +} + +pub struct PayloadConfig { + limit: usize, +} + +impl PayloadConfig { + pub fn new(limit: usize) -> Self { + Self { limit } + } +} + +impl Default for PayloadConfig { + fn default() -> Self { + Self { limit: 256 * 1024 } + } +} + +impl FromRequest for Payload { + type Config = PayloadConfig; + + type Error = PayloadError; + + type Future = Ready>; + + #[inline] + fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future { + let limit = req + .app_data::() + .map(|c| c.limit) + .unwrap_or(Self::Config::default().limit); + ready(Ok(Payload { + payload: payload.take(), + limit, + })) + } +} + +impl Stream for Payload { + type Item = Result; + + #[inline] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match Pin::new(&mut self.payload).poll_next(cx) { + Poll::Ready(Some(result)) => match result { + Ok(bytes) => match self.limit.checked_sub(bytes.len()) { + Some(new_limit) => { + self.limit = new_limit; + Poll::Ready(Some(Ok(bytes))) + } + None => Poll::Ready(Some(Err(PayloadError::Overflow))), + }, + x => Poll::Ready(Some(x)), + }, + otherwise => otherwise, + } + } +} diff --git a/meilisearch-http/src/helpers/authentication.rs b/meilisearch-http/src/helpers/authentication.rs deleted file mode 100644 index c33d6cde3..000000000 --- a/meilisearch-http/src/helpers/authentication.rs +++ /dev/null @@ -1,107 +0,0 @@ -use std::cell::RefCell; -use std::pin::Pin; -use std::rc::Rc; -use std::task::{Context, Poll}; - -use actix_service::{Service, Transform}; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web}; -use futures::future::{err, ok, Future, Ready}; -use actix_web::error::ResponseError as _; -use actix_web::dev::Body; - -use crate::error::{Error, ResponseError}; -use crate::Data; - -#[derive(Clone)] -pub enum Authentication { - Public, - Private, - Admin, -} - -impl Transform for Authentication -where - S: Service, Error = actix_web::Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = actix_web::Error; - type InitError = (); - type Transform = LoggingMiddleware; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ok(LoggingMiddleware { - acl: self.clone(), - service: Rc::new(RefCell::new(service)), - }) - } -} - -pub struct LoggingMiddleware { - acl: Authentication, - service: Rc>, -} - -#[allow(clippy::type_complexity)] -impl Service for LoggingMiddleware -where - S: Service, Error = actix_web::Error> + 'static, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = actix_web::Error; - type Future = Pin>>>; - - fn poll_ready(&mut self, cx: &mut Context) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, req: ServiceRequest) -> Self::Future { - let mut svc = self.service.clone(); - // This unwrap is left because this error should never appear. If that's the case, then - // it means that actix-web has an issue or someone changes the type `Data`. - let data = req.app_data::>().unwrap(); - - if data.api_keys.master.is_none() { - return Box::pin(svc.call(req)); - } - - let auth_header = match req.headers().get("X-Meili-API-Key") { - Some(auth) => match auth.to_str() { - Ok(auth) => auth, - Err(_) => { - let error = ResponseError::from(Error::MissingAuthorizationHeader).error_response(); - let (request, _) = req.into_parts(); - return Box::pin(ok(ServiceResponse::new(request, error))) - } - }, - None => { - return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())); - } - }; - - let authenticated = match self.acl { - Authentication::Admin => data.api_keys.master.as_deref() == Some(auth_header), - Authentication::Private => { - data.api_keys.master.as_deref() == Some(auth_header) - || data.api_keys.private.as_deref() == Some(auth_header) - } - Authentication::Public => { - data.api_keys.master.as_deref() == Some(auth_header) - || data.api_keys.private.as_deref() == Some(auth_header) - || data.api_keys.public.as_deref() == Some(auth_header) - } - }; - - if authenticated { - Box::pin(svc.call(req)) - } else { - let error = ResponseError::from(Error::InvalidToken(auth_header.to_string())).error_response(); - let (request, _) = req.into_parts(); - Box::pin(ok(ServiceResponse::new(request, error))) - } - } -} diff --git a/meilisearch-http/src/helpers/compression.rs b/meilisearch-http/src/helpers/compression.rs index 93f5c6a08..c4747cb21 100644 --- a/meilisearch-http/src/helpers/compression.rs +++ b/meilisearch-http/src/helpers/compression.rs @@ -1,35 +1,26 @@ -use flate2::Compression; -use flate2::read::GzDecoder; -use flate2::write::GzEncoder; -use std::fs::{create_dir_all, rename, File}; +use std::fs::{create_dir_all, File}; +use std::io::Write; use std::path::Path; -use tar::{Builder, Archive}; -use uuid::Uuid; -use crate::error::Error; +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use tar::{Archive, Builder}; -pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { - let file_name = format!(".{}", Uuid::new_v4().to_urn()); - let p = dest.with_file_name(file_name); - let tmp_dest = p.as_path(); - - let f = File::create(tmp_dest)?; - let gz_encoder = GzEncoder::new(f, Compression::default()); +pub fn to_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { + let mut f = File::create(dest)?; + let gz_encoder = GzEncoder::new(&mut f, Compression::default()); let mut tar_encoder = Builder::new(gz_encoder); tar_encoder.append_dir_all(".", src)?; let gz_encoder = tar_encoder.into_inner()?; gz_encoder.finish()?; - - rename(tmp_dest, dest)?; - + f.flush()?; Ok(()) } -pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> { - let f = File::open(src)?; +pub fn from_tar_gz(src: impl AsRef, dest: impl AsRef) -> anyhow::Result<()> { + let f = File::open(&src)?; let gz = GzDecoder::new(f); let mut ar = Archive::new(gz); - create_dir_all(dest)?; - ar.unpack(dest)?; + create_dir_all(&dest)?; + ar.unpack(&dest)?; Ok(()) } diff --git a/meilisearch-http/src/helpers/env.rs b/meilisearch-http/src/helpers/env.rs new file mode 100644 index 000000000..9bc81bc69 --- /dev/null +++ b/meilisearch-http/src/helpers/env.rs @@ -0,0 +1,16 @@ +use walkdir::WalkDir; + +pub trait EnvSizer { + fn size(&self) -> u64; +} + +impl EnvSizer for heed::Env { + fn size(&self) -> u64 { + WalkDir::new(self.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()) + } +} diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs deleted file mode 100644 index 1cf25e315..000000000 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ /dev/null @@ -1,649 +0,0 @@ -use std::cmp::Ordering; -use std::collections::{HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::time::Instant; - -use indexmap::IndexMap; -use log::error; -use meilisearch_core::{Filter, MainReader}; -use meilisearch_core::facets::FacetFilter; -use meilisearch_core::criterion::*; -use meilisearch_core::settings::RankingRule; -use meilisearch_core::{Highlight, Index, RankedMap}; -use meilisearch_schema::{FieldId, Schema}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use siphasher::sip::SipHasher; -use slice_group_by::GroupBy; - -use crate::error::{Error, ResponseError}; - -pub trait IndexSearchExt { - fn new_search(&self, query: Option) -> SearchBuilder; -} - -impl IndexSearchExt for Index { - fn new_search(&self, query: Option) -> SearchBuilder { - SearchBuilder { - index: self, - query, - offset: 0, - limit: 20, - attributes_to_crop: None, - attributes_to_retrieve: None, - attributes_to_highlight: None, - filters: None, - matches: false, - facet_filters: None, - facets: None, - } - } -} - -pub struct SearchBuilder<'a> { - index: &'a Index, - query: Option, - offset: usize, - limit: usize, - attributes_to_crop: Option>, - attributes_to_retrieve: Option>, - attributes_to_highlight: Option>, - filters: Option, - matches: bool, - facet_filters: Option, - facets: Option> -} - -impl<'a> SearchBuilder<'a> { - pub fn offset(&mut self, value: usize) -> &SearchBuilder { - self.offset = value; - self - } - - pub fn limit(&mut self, value: usize) -> &SearchBuilder { - self.limit = value; - self - } - - pub fn attributes_to_crop(&mut self, value: HashMap) -> &SearchBuilder { - self.attributes_to_crop = Some(value); - self - } - - pub fn attributes_to_retrieve(&mut self, value: HashSet) -> &SearchBuilder { - self.attributes_to_retrieve = Some(value); - self - } - - pub fn add_retrievable_field(&mut self, value: String) -> &SearchBuilder { - let attributes_to_retrieve = self.attributes_to_retrieve.get_or_insert(HashSet::new()); - attributes_to_retrieve.insert(value); - self - } - - pub fn attributes_to_highlight(&mut self, value: HashSet) -> &SearchBuilder { - self.attributes_to_highlight = Some(value); - self - } - - pub fn add_facet_filters(&mut self, filters: FacetFilter) -> &SearchBuilder { - self.facet_filters = Some(filters); - self - } - - pub fn filters(&mut self, value: String) -> &SearchBuilder { - self.filters = Some(value); - self - } - - pub fn get_matches(&mut self) -> &SearchBuilder { - self.matches = true; - self - } - - pub fn add_facets(&mut self, facets: Vec<(FieldId, String)>) -> &SearchBuilder { - self.facets = Some(facets); - self - } - - pub fn search(self, reader: &MainReader) -> Result { - let schema = self - .index - .main - .schema(reader)? - .ok_or(Error::internal("missing schema"))?; - - let ranked_map = self.index.main.ranked_map(reader)?.unwrap_or_default(); - - // Change criteria - let mut query_builder = match self.get_criteria(reader, &ranked_map, &schema)? { - Some(criteria) => self.index.query_builder_with_criteria(criteria), - None => self.index.query_builder(), - }; - - if let Some(filter_expression) = &self.filters { - let filter = Filter::parse(filter_expression, &schema)?; - let index = &self.index; - query_builder.with_filter(move |id| { - let reader = &reader; - let filter = &filter; - match filter.test(reader, index, id) { - Ok(res) => res, - Err(e) => { - log::warn!("unexpected error during filtering: {}", e); - false - } - } - }); - } - - if let Some(field) = self.index.main.distinct_attribute(reader)? { - let index = &self.index; - query_builder.with_distinct(1, move |id| { - match index.document_attribute_bytes(reader, id, field) { - Ok(Some(bytes)) => { - let mut s = SipHasher::new(); - bytes.hash(&mut s); - Some(s.finish()) - } - _ => None, - } - }); - } - - query_builder.set_facet_filter(self.facet_filters); - query_builder.set_facets(self.facets); - - let start = Instant::now(); - let result = query_builder.query(reader, self.query.as_deref(), self.offset..(self.offset + self.limit)); - let search_result = result.map_err(Error::search_documents)?; - let time_ms = start.elapsed().as_millis() as usize; - - let mut all_attributes: HashSet<&str> = HashSet::new(); - let mut all_formatted: HashSet<&str> = HashSet::new(); - - match &self.attributes_to_retrieve { - Some(to_retrieve) => { - all_attributes.extend(to_retrieve.iter().map(String::as_str)); - - if let Some(to_highlight) = &self.attributes_to_highlight { - all_formatted.extend(to_highlight.iter().map(String::as_str)); - } - - if let Some(to_crop) = &self.attributes_to_crop { - all_formatted.extend(to_crop.keys().map(String::as_str)); - } - - all_attributes.extend(&all_formatted); - }, - None => { - all_attributes.extend(schema.displayed_names()); - // If we specified at least one attribute to highlight or crop then - // all available attributes will be returned in the _formatted field. - if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() { - all_formatted.extend(all_attributes.iter().cloned()); - } - }, - } - - let mut hits = Vec::with_capacity(self.limit); - for doc in search_result.documents { - let mut document: IndexMap = self - .index - .document(reader, Some(&all_attributes), doc.id) - .map_err(|e| Error::retrieve_document(doc.id.0, e))? - .unwrap_or_default(); - - let mut formatted = document.iter() - .filter(|(key, _)| all_formatted.contains(key.as_str())) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut matches = doc.highlights.clone(); - - // Crops fields if needed - if let Some(fields) = &self.attributes_to_crop { - crop_document(&mut formatted, &mut matches, &schema, fields); - } - - // Transform to readable matches - if let Some(attributes_to_highlight) = &self.attributes_to_highlight { - let matches = calculate_matches( - &matches, - self.attributes_to_highlight.clone(), - &schema, - ); - formatted = calculate_highlights(&formatted, &matches, attributes_to_highlight); - } - - let matches_info = if self.matches { - Some(calculate_matches(&matches, self.attributes_to_retrieve.clone(), &schema)) - } else { - None - }; - - if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve { - document.retain(|key, _| attributes_to_retrieve.contains(&key.to_string())) - } - - let hit = SearchHit { - document, - formatted, - matches_info, - }; - - hits.push(hit); - } - - let results = SearchResult { - hits, - offset: self.offset, - limit: self.limit, - nb_hits: search_result.nb_hits, - exhaustive_nb_hits: search_result.exhaustive_nb_hit, - processing_time_ms: time_ms, - query: self.query.unwrap_or_default(), - facets_distribution: search_result.facets, - exhaustive_facets_count: search_result.exhaustive_facets_count, - }; - - Ok(results) - } - - pub fn get_criteria( - &self, - reader: &MainReader, - ranked_map: &'a RankedMap, - schema: &Schema, - ) -> Result>, ResponseError> { - let ranking_rules = self.index.main.ranking_rules(reader)?; - - if let Some(ranking_rules) = ranking_rules { - let mut builder = CriteriaBuilder::with_capacity(7 + ranking_rules.len()); - for rule in ranking_rules { - match rule { - RankingRule::Typo => builder.push(Typo), - RankingRule::Words => builder.push(Words), - RankingRule::Proximity => builder.push(Proximity), - RankingRule::Attribute => builder.push(Attribute), - RankingRule::WordsPosition => builder.push(WordsPosition), - RankingRule::Exactness => builder.push(Exactness), - RankingRule::Asc(field) => { - match SortByAttr::lower_is_better(&ranked_map, &schema, &field) { - Ok(rule) => builder.push(rule), - Err(err) => error!("Error during criteria builder; {:?}", err), - } - } - RankingRule::Desc(field) => { - match SortByAttr::higher_is_better(&ranked_map, &schema, &field) { - Ok(rule) => builder.push(rule), - Err(err) => error!("Error during criteria builder; {:?}", err), - } - } - } - } - builder.push(DocumentId); - return Ok(Some(builder.build())); - } - - Ok(None) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct MatchPosition { - pub start: usize, - pub length: usize, -} - -impl PartialOrd for MatchPosition { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for MatchPosition { - fn cmp(&self, other: &Self) -> Ordering { - match self.start.cmp(&other.start) { - Ordering::Equal => self.length.cmp(&other.length), - _ => self.start.cmp(&other.start), - } - } -} - -pub type HighlightInfos = HashMap; -pub type MatchesInfos = HashMap>; -// pub type RankingInfos = HashMap; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SearchHit { - #[serde(flatten)] - pub document: IndexMap, - #[serde(rename = "_formatted", skip_serializing_if = "IndexMap::is_empty")] - pub formatted: IndexMap, - #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] - pub matches_info: Option, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchResult { - pub hits: Vec, - pub offset: usize, - pub limit: usize, - pub nb_hits: usize, - pub exhaustive_nb_hits: bool, - pub processing_time_ms: usize, - pub query: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub facets_distribution: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub exhaustive_facets_count: Option, -} - -/// returns the start index and the length on the crop. -fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) { - let is_word_component = |c: &char| c.is_alphanumeric() && !super::is_cjk(*c); - - let word_end_index = |mut index| { - if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) { - index += text.chars().skip(index).take_while(is_word_component).count(); - } - index - }; - - if context == 0 { - // count need to be at least 1 for cjk queries to return something - return (match_index, 1 + text.chars().skip(match_index).take_while(is_word_component).count()); - } - let start = match match_index.saturating_sub(context) { - 0 => 0, - n => { - let word_end_index = word_end_index(n); - // skip whitespaces if any - word_end_index + text.chars().skip(word_end_index).take_while(char::is_ascii_whitespace).count() - } - }; - let end = word_end_index(match_index + context); - - (start, end - start) -} - -fn crop_text( - text: &str, - matches: impl IntoIterator, - context: usize, -) -> (String, Vec) { - let mut matches = matches.into_iter().peekable(); - - let char_index = matches.peek().map(|m| m.char_index as usize).unwrap_or(0); - let (start, count) = aligned_crop(text, char_index, context); - - // TODO do something about double allocation - let text = text - .chars() - .skip(start) - .take(count) - .collect::() - .trim() - .to_string(); - - // update matches index to match the new cropped text - let matches = matches - .take_while(|m| (m.char_index as usize) + (m.char_length as usize) <= start + count) - .map(|m| Highlight { - char_index: m.char_index - start as u16, - ..m - }) - .collect(); - - (text, matches) -} - -fn crop_document( - document: &mut IndexMap, - matches: &mut Vec, - schema: &Schema, - fields: &HashMap, -) { - matches.sort_unstable_by_key(|m| (m.char_index, m.char_length)); - - for (field, length) in fields { - let attribute = match schema.id(field) { - Some(attribute) => attribute, - None => continue, - }; - - let selected_matches = matches - .iter() - .filter(|m| FieldId::new(m.attribute) == attribute) - .cloned(); - - if let Some(Value::String(ref mut original_text)) = document.get_mut(field) { - let (cropped_text, cropped_matches) = - crop_text(original_text, selected_matches, *length); - - *original_text = cropped_text; - - matches.retain(|m| FieldId::new(m.attribute) != attribute); - matches.extend_from_slice(&cropped_matches); - } - } -} - -fn calculate_matches( - matches: &[Highlight], - attributes_to_retrieve: Option>, - schema: &Schema, -) -> MatchesInfos { - let mut matches_result: HashMap> = HashMap::new(); - for m in matches.iter() { - if let Some(attribute) = schema.name(FieldId::new(m.attribute)) { - if let Some(ref attributes_to_retrieve) = attributes_to_retrieve { - if !attributes_to_retrieve.contains(attribute) { - continue; - } - } - if !schema.displayed_names().contains(&attribute) { - continue; - } - if let Some(pos) = matches_result.get_mut(attribute) { - pos.push(MatchPosition { - start: m.char_index as usize, - length: m.char_length as usize, - }); - } else { - let mut positions = Vec::new(); - positions.push(MatchPosition { - start: m.char_index as usize, - length: m.char_length as usize, - }); - matches_result.insert(attribute.to_string(), positions); - } - } - } - for (_, val) in matches_result.iter_mut() { - val.sort_unstable(); - val.dedup(); - } - matches_result -} - -fn calculate_highlights( - document: &IndexMap, - matches: &MatchesInfos, - attributes_to_highlight: &HashSet, -) -> IndexMap { - let mut highlight_result = document.clone(); - - for (attribute, matches) in matches.iter() { - if attributes_to_highlight.contains(attribute) { - if let Some(Value::String(value)) = document.get(attribute) { - let value = value; - let mut highlighted_value = String::new(); - let mut index = 0; - - let longest_matches = matches - .linear_group_by_key(|m| m.start) - .map(|group| group.last().unwrap()) - .filter(move |m| m.start >= index); - - for m in longest_matches { - let before = value.get(index..m.start); - let highlighted = value.get(m.start..(m.start + m.length)); - if let (Some(before), Some(highlighted)) = (before, highlighted) { - highlighted_value.push_str(before); - highlighted_value.push_str(""); - highlighted_value.push_str(highlighted); - highlighted_value.push_str(""); - index = m.start + m.length; - } else { - error!("value: {:?}; index: {:?}, match: {:?}", value, index, m); - } - } - highlighted_value.push_str(&value[index..]); - highlight_result.insert(attribute.to_string(), Value::String(highlighted_value)); - }; - } - } - highlight_result -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn aligned_crops() { - let text = r#"En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation."#; - - // simple test - let (start, length) = aligned_crop(&text, 6, 2); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("début", cropped); - - // first word test - let (start, length) = aligned_crop(&text, 0, 1); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("En", cropped); - // last word test - let (start, length) = aligned_crop(&text, 510, 2); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("Fondation", cropped); - - // CJK tests - let text = "this isのス foo myタイリ test"; - - // mixed charset - let (start, length) = aligned_crop(&text, 5, 3); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("isの", cropped); - - // split regular word / CJK word, no space - let (start, length) = aligned_crop(&text, 7, 1); - let cropped = text.chars().skip(start).take(length).collect::().trim().to_string(); - assert_eq!("の", cropped); - } - - #[test] - fn calculate_matches() { - let mut matches = Vec::new(); - matches.push(Highlight { attribute: 0, char_index: 0, char_length: 3}); - matches.push(Highlight { attribute: 0, char_index: 0, char_length: 2}); - - let mut attributes_to_retrieve: HashSet = HashSet::new(); - attributes_to_retrieve.insert("title".to_string()); - - let schema = Schema::with_primary_key("title"); - - let matches_result = super::calculate_matches(&matches, Some(attributes_to_retrieve), &schema); - - let mut matches_result_expected: HashMap> = HashMap::new(); - - let mut positions = Vec::new(); - positions.push(MatchPosition { - start: 0, - length: 2, - }); - positions.push(MatchPosition { - start: 0, - length: 3, - }); - matches_result_expected.insert("title".to_string(), positions); - - assert_eq!(matches_result, matches_result_expected); - } - - #[test] - fn calculate_highlights() { - let data = r#"{ - "title": "Fondation (Isaac ASIMOV)", - "description": "En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation." - }"#; - - let document: IndexMap = serde_json::from_str(data).unwrap(); - let mut attributes_to_highlight = HashSet::new(); - attributes_to_highlight.insert("title".to_string()); - attributes_to_highlight.insert("description".to_string()); - - let mut matches = HashMap::new(); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 0, - length: 9, - }); - matches.insert("title".to_string(), m); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 529, - length: 9, - }); - matches.insert("description".to_string(), m); - let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); - - let mut result_expected = IndexMap::new(); - result_expected.insert( - "title".to_string(), - Value::String("Fondation (Isaac ASIMOV)".to_string()), - ); - result_expected.insert("description".to_string(), Value::String("En ce début de trentième millénaire, l'Empire n'a jamais été aussi puissant, aussi étendu à travers toute la galaxie. C'est dans sa capitale, Trantor, que l'éminent savant Hari Seldon invente la psychohistoire, une science toute nouvelle, à base de psychologie et de mathématiques, qui lui permet de prédire l'avenir... C'est-à-dire l'effondrement de l'Empire d'ici cinq siècles et au-delà, trente mille années de chaos et de ténèbres. Pour empêcher cette catastrophe et sauver la civilisation, Seldon crée la Fondation.".to_string())); - - assert_eq!(result, result_expected); - } - - #[test] - fn highlight_longest_match() { - let data = r#"{ - "title": "Ice" - }"#; - - let document: IndexMap = serde_json::from_str(data).unwrap(); - let mut attributes_to_highlight = HashSet::new(); - attributes_to_highlight.insert("title".to_string()); - - let mut matches = HashMap::new(); - - let mut m = Vec::new(); - m.push(MatchPosition { - start: 0, - length: 2, - }); - m.push(MatchPosition { - start: 0, - length: 3, - }); - matches.insert("title".to_string(), m); - - let result = super::calculate_highlights(&document, &matches, &attributes_to_highlight); - - let mut result_expected = IndexMap::new(); - result_expected.insert( - "title".to_string(), - Value::String("Ice".to_string()), - ); - - assert_eq!(result, result_expected); - } -} diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index 9a78e6b71..c664f15aa 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,26 +1,4 @@ -pub mod authentication; -pub mod meilisearch; -pub mod normalize_path; pub mod compression; +mod env; -pub use authentication::Authentication; -pub use normalize_path::NormalizePath; - -pub fn is_cjk(c: char) -> bool { - ('\u{1100}'..'\u{11ff}').contains(&c) // Hangul Jamo - || ('\u{2e80}'..'\u{2eff}').contains(&c) // CJK Radicals Supplement - || ('\u{2f00}'..'\u{2fdf}').contains(&c) // Kangxi radical - || ('\u{3000}'..'\u{303f}').contains(&c) // Japanese-style punctuation - || ('\u{3040}'..'\u{309f}').contains(&c) // Japanese Hiragana - || ('\u{30a0}'..'\u{30ff}').contains(&c) // Japanese Katakana - || ('\u{3100}'..'\u{312f}').contains(&c) - || ('\u{3130}'..'\u{318F}').contains(&c) // Hangul Compatibility Jamo - || ('\u{3200}'..'\u{32ff}').contains(&c) // Enclosed CJK Letters and Months - || ('\u{3400}'..'\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A - || ('\u{4e00}'..'\u{9fff}').contains(&c) // CJK Unified Ideographs - || ('\u{a960}'..'\u{a97f}').contains(&c) // Hangul Jamo Extended-A - || ('\u{ac00}'..'\u{d7a3}').contains(&c) // Hangul Syllables - || ('\u{d7b0}'..'\u{d7ff}').contains(&c) // Hangul Jamo Extended-B - || ('\u{f900}'..'\u{faff}').contains(&c) // CJK Compatibility Ideographs - || ('\u{ff00}'..'\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana -} +pub use env::EnvSizer; diff --git a/meilisearch-http/src/helpers/normalize_path.rs b/meilisearch-http/src/helpers/normalize_path.rs deleted file mode 100644 index e669b9d94..000000000 --- a/meilisearch-http/src/helpers/normalize_path.rs +++ /dev/null @@ -1,86 +0,0 @@ -/// From https://docs.rs/actix-web/3.0.0-alpha.2/src/actix_web/middleware/normalize.rs.html#34 -use actix_http::Error; -use actix_service::{Service, Transform}; -use actix_web::{ - dev::ServiceRequest, - dev::ServiceResponse, - http::uri::{PathAndQuery, Uri}, -}; -use futures::future::{ok, Ready}; -use regex::Regex; -use std::task::{Context, Poll}; -pub struct NormalizePath; - -impl Transform for NormalizePath -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = NormalizePathNormalization; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - ok(NormalizePathNormalization { - service, - merge_slash: Regex::new("//+").unwrap(), - }) - } -} - -pub struct NormalizePathNormalization { - service: S, - merge_slash: Regex, -} - -impl Service for NormalizePathNormalization -where - S: Service, Error = Error>, - S::Future: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = S::Future; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, mut req: ServiceRequest) -> Self::Future { - let head = req.head_mut(); - - // always add trailing slash, might be an extra one - let path = head.uri.path().to_string() + "/"; - - if self.merge_slash.find(&path).is_some() { - // normalize multiple /'s to one / - let path = self.merge_slash.replace_all(&path, "/"); - - let path = if path.len() > 1 { - path.trim_end_matches('/') - } else { - &path - }; - - let mut parts = head.uri.clone().into_parts(); - let pq = parts.path_and_query.as_ref().unwrap(); - - let path = if let Some(q) = pq.query() { - bytes::Bytes::from(format!("{}?{}", path, q)) - } else { - bytes::Bytes::copy_from_slice(path.as_bytes()) - }; - parts.path_and_query = Some(PathAndQuery::from_maybe_shared(path).unwrap()); - - let uri = Uri::from_parts(parts).unwrap(); - req.match_info_mut().get_mut().update(&uri); - req.head_mut().uri = uri; - } - - self.service.call(req) - } -} diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs new file mode 100644 index 000000000..263e3bd52 --- /dev/null +++ b/meilisearch-http/src/index/dump.rs @@ -0,0 +1,134 @@ +use std::fs::{create_dir_all, File}; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::sync::Arc; + +use anyhow::{bail, Context}; +use heed::RoTxn; +use indexmap::IndexMap; +use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; +use serde::{Deserialize, Serialize}; + +use crate::option::IndexerOpts; + +use super::error::Result; +use super::{update_handler::UpdateHandler, Index, Settings, Unchecked}; + +#[derive(Serialize, Deserialize)] +struct DumpMeta { + settings: Settings, + primary_key: Option, +} + +const META_FILE_NAME: &str = "meta.json"; +const DATA_FILE_NAME: &str = "documents.jsonl"; + +impl Index { + pub fn dump(&self, path: impl AsRef) -> Result<()> { + // acquire write txn make sure any ongoing write is finished before we start. + let txn = self.env.write_txn()?; + + self.dump_documents(&txn, &path)?; + self.dump_meta(&txn, &path)?; + + Ok(()) + } + + fn dump_documents(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { + let document_file_path = path.as_ref().join(DATA_FILE_NAME); + let mut document_file = File::create(&document_file_path)?; + + let documents = self.all_documents(txn)?; + let fields_ids_map = self.fields_ids_map(txn)?; + + // dump documents + let mut json_map = IndexMap::new(); + for document in documents { + let (_, reader) = document?; + + for (fid, bytes) in reader.iter() { + if let Some(name) = fields_ids_map.name(fid) { + json_map.insert(name, serde_json::from_slice::(bytes)?); + } + } + + serde_json::to_writer(&mut document_file, &json_map)?; + document_file.write_all(b"\n")?; + + json_map.clear(); + } + + Ok(()) + } + + fn dump_meta(&self, txn: &RoTxn, path: impl AsRef) -> Result<()> { + let meta_file_path = path.as_ref().join(META_FILE_NAME); + let mut meta_file = File::create(&meta_file_path)?; + + let settings = self.settings_txn(txn)?.into_unchecked(); + let primary_key = self.primary_key(txn)?.map(String::from); + let meta = DumpMeta { + settings, + primary_key, + }; + + serde_json::to_writer(&mut meta_file, &meta)?; + + Ok(()) + } + + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + size: usize, + indexing_options: &IndexerOpts, + ) -> anyhow::Result<()> { + let dir_name = src + .as_ref() + .file_name() + .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?; + + let dst_dir_path = dst.as_ref().join("indexes").join(dir_name); + create_dir_all(&dst_dir_path)?; + + let meta_path = src.as_ref().join(META_FILE_NAME); + let mut meta_file = File::open(meta_path)?; + let DumpMeta { + settings, + primary_key, + } = serde_json::from_reader(&mut meta_file)?; + let settings = settings.check(); + let index = Self::open(&dst_dir_path, size)?; + let mut txn = index.write_txn()?; + + let handler = UpdateHandler::new(&indexing_options)?; + + index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?; + + let document_file_path = src.as_ref().join(DATA_FILE_NAME); + let reader = File::open(&document_file_path)?; + let mut reader = BufReader::new(reader); + reader.fill_buf()?; + // If the document file is empty, we don't perform the document addition, to prevent + // a primary key error to be thrown. + if !reader.buffer().is_empty() { + index.update_documents_txn( + &mut txn, + JsonStream, + IndexDocumentsMethod::UpdateDocuments, + Some(reader), + handler.update_builder(0), + primary_key.as_deref(), + )?; + } + + txn.commit()?; + + match Arc::try_unwrap(index.0) { + Ok(inner) => inner.prepare_for_closing().wait(), + Err(_) => bail!("Could not close index properly."), + } + + Ok(()) + } +} diff --git a/meilisearch-http/src/index/error.rs b/meilisearch-http/src/index/error.rs new file mode 100644 index 000000000..cfae11a1f --- /dev/null +++ b/meilisearch-http/src/index/error.rs @@ -0,0 +1,52 @@ +use std::error::Error; + +use meilisearch_error::{Code, ErrorCode}; +use serde_json::Value; + +use crate::error::MilliError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IndexError { + #[error("Internal error: {0}")] + Internal(Box), + #[error("Document with id {0} not found.")] + DocumentNotFound(String), + #[error("{0}")] + Facet(#[from] FacetError), + #[error("{0}")] + Milli(#[from] milli::Error), +} + +internal_error!( + IndexError: std::io::Error, + heed::Error, + fst::Error, + serde_json::Error +); + +impl ErrorCode for IndexError { + fn error_code(&self) -> Code { + match self { + IndexError::Internal(_) => Code::Internal, + IndexError::DocumentNotFound(_) => Code::DocumentNotFound, + IndexError::Facet(e) => e.error_code(), + IndexError::Milli(e) => MilliError(e).error_code(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FacetError { + #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))] + InvalidExpression(&'static [&'static str], Value), +} + +impl ErrorCode for FacetError { + fn error_code(&self) -> Code { + match self { + FacetError::InvalidExpression(_, _) => Code::Facet, + } + } +} diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs new file mode 100644 index 000000000..7227f8d35 --- /dev/null +++ b/meilisearch-http/src/index/mod.rs @@ -0,0 +1,194 @@ +use std::collections::{BTreeSet, HashSet}; +use std::fs::create_dir_all; +use std::marker::PhantomData; +use std::ops::Deref; +use std::path::Path; +use std::sync::Arc; + +use heed::{EnvOpenOptions, RoTxn}; +use milli::obkv_to_json; +use serde::{de::Deserializer, Deserialize}; +use serde_json::{Map, Value}; + +use crate::helpers::EnvSizer; +use error::Result; + +pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; +pub use updates::{Checked, Facets, Settings, Unchecked}; + +use self::error::IndexError; + +pub mod error; +pub mod update_handler; + +mod dump; +mod search; +mod updates; + +pub type Document = Map; + +#[derive(Clone)] +pub struct Index(pub Arc); + +impl Deref for Index { + type Target = milli::Index; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +pub fn deserialize_some<'de, T, D>(deserializer: D) -> std::result::Result, D::Error> +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + Deserialize::deserialize(deserializer).map(Some) +} + +impl Index { + pub fn open(path: impl AsRef, size: usize) -> Result { + create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, &path)?; + Ok(Index(Arc::new(index))) + } + + pub fn settings(&self) -> Result> { + let txn = self.read_txn()?; + self.settings_txn(&txn) + } + + pub fn settings_txn(&self, txn: &RoTxn) -> Result> { + let displayed_attributes = self + .displayed_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()); + + let searchable_attributes = self + .searchable_fields(&txn)? + .map(|fields| fields.into_iter().map(String::from).collect()); + + let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect(); + + let criteria = self + .criteria(&txn)? + .into_iter() + .map(|c| c.to_string()) + .collect(); + + let stop_words = self + .stop_words(&txn)? + .map(|stop_words| -> Result> { + Ok(stop_words.stream().into_strs()?.into_iter().collect()) + }) + .transpose()? + .unwrap_or_else(BTreeSet::new); + let distinct_field = self.distinct_field(&txn)?.map(String::from); + + // in milli each word in the synonyms map were split on their separator. Since we lost + // this information we are going to put space between words. + let synonyms = self + .synonyms(&txn)? + .iter() + .map(|(key, values)| { + ( + key.join(" "), + values.iter().map(|value| value.join(" ")).collect(), + ) + }) + .collect(); + + Ok(Settings { + displayed_attributes: Some(displayed_attributes), + searchable_attributes: Some(searchable_attributes), + filterable_attributes: Some(Some(faceted_attributes)), + ranking_rules: Some(Some(criteria)), + stop_words: Some(Some(stop_words)), + distinct_attribute: Some(distinct_field), + synonyms: Some(Some(synonyms)), + _kind: PhantomData, + }) + } + + pub fn retrieve_documents>( + &self, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result>> { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + let fields_to_display = + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + + let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); + + let mut documents = Vec::new(); + + for entry in iter { + let (_id, obkv) = entry?; + let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; + documents.push(object); + } + + Ok(documents) + } + + pub fn retrieve_document>( + &self, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result> { + let txn = self.read_txn()?; + + let fields_ids_map = self.fields_ids_map(&txn)?; + + let fields_to_display = + self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + + let internal_id = self + .external_documents_ids(&txn)? + .get(doc_id.as_bytes()) + .ok_or_else(|| IndexError::DocumentNotFound(doc_id.clone()))?; + + let document = self + .documents(&txn, std::iter::once(internal_id))? + .into_iter() + .next() + .map(|(_, d)| d) + .ok_or(IndexError::DocumentNotFound(doc_id))?; + + let document = obkv_to_json(&fields_to_display, &fields_ids_map, document)?; + + Ok(document) + } + + pub fn size(&self) -> u64 { + self.env.size() + } + + fn fields_to_display>( + &self, + txn: &heed::RoTxn, + attributes_to_retrieve: &Option>, + fields_ids_map: &milli::FieldsIdsMap, + ) -> Result> { + let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + Some(ids) => ids.into_iter().collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let attributes_to_retrieve_ids = match attributes_to_retrieve { + Some(attrs) => attrs + .iter() + .filter_map(|f| fields_ids_map.id(f.as_ref())) + .collect::>(), + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); + Ok(displayed_fields_ids) + } +} diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs new file mode 100644 index 000000000..2d8095559 --- /dev/null +++ b/meilisearch-http/src/index/search.rs @@ -0,0 +1,1182 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::time::Instant; + +use either::Either; +use heed::RoTxn; +use indexmap::IndexMap; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token}; +use milli::{FieldId, FieldsIdsMap, FilterCondition, MatchingWords}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::index::error::FacetError; + +use super::error::Result; +use super::Index; + +pub type Document = IndexMap; +type MatchesInfo = BTreeMap>; + +#[derive(Serialize, Debug, Clone)] +pub struct MatchInfo { + start: usize, + length: usize, +} + +pub const DEFAULT_SEARCH_LIMIT: usize = 20; +const fn default_search_limit() -> usize { + DEFAULT_SEARCH_LIMIT +} + +pub const DEFAULT_CROP_LENGTH: usize = 200; +pub const fn default_crop_length() -> usize { + DEFAULT_CROP_LENGTH +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchQuery { + pub q: Option, + pub offset: Option, + #[serde(default = "default_search_limit")] + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + #[serde(default = "default_crop_length")] + pub crop_length: usize, + pub attributes_to_highlight: Option>, + // Default to false + #[serde(default = "Default::default")] + pub matches: bool, + pub filter: Option, + pub facets_distribution: Option>, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SearchHit { + #[serde(flatten)] + pub document: Document, + #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] + pub formatted: Document, + #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] + pub matches_info: Option, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { + pub hits: Vec, + pub nb_hits: u64, + pub exhaustive_nb_hits: bool, + pub query: String, + pub limit: usize, + pub offset: usize, + pub processing_time_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub facets_distribution: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exhaustive_facets_count: Option, +} + +#[derive(Copy, Clone)] +struct FormatOptions { + highlight: bool, + crop: Option, +} + +impl Index { + pub fn perform_search(&self, query: SearchQuery) -> Result { + let before_search = Instant::now(); + let rtxn = self.read_txn()?; + + let mut search = self.search(&rtxn); + + if let Some(ref query) = query.q { + search.query(query); + } + + search.limit(query.limit); + search.offset(query.offset.unwrap_or_default()); + + if let Some(ref filter) = query.filter { + if let Some(facets) = parse_filter(filter, self, &rtxn)? { + search.filter(facets); + } + } + + let milli::SearchResult { + documents_ids, + matching_words, + candidates, + .. + } = search.execute()?; + + let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); + + let displayed_ids = self + .displayed_fields_ids(&rtxn)? + .map(|fields| fields.into_iter().collect::>()) + .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); + + let fids = |attrs: &BTreeSet| { + let mut ids = BTreeSet::new(); + for attr in attrs { + if attr == "*" { + ids = displayed_ids.clone(); + break; + } + + if let Some(id) = fields_ids_map.id(attr) { + ids.insert(id); + } + } + ids + }; + + // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default), + // but these attributes must be also be present + // - in the fields_ids_map + // - in the the displayed attributes + let to_retrieve_ids: BTreeSet<_> = query + .attributes_to_retrieve + .as_ref() + .map(fids) + .unwrap_or_else(|| displayed_ids.clone()) + .intersection(&displayed_ids) + .cloned() + .collect(); + + let attr_to_highlight = query.attributes_to_highlight.unwrap_or_default(); + + let attr_to_crop = query.attributes_to_crop.unwrap_or_default(); + + // Attributes in `formatted_options` correspond to the attributes that will be in `_formatted` + // These attributes are: + // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`) + // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped + // But these attributes must be also present in displayed attributes + let formatted_options = compute_formatted_options( + &attr_to_highlight, + &attr_to_crop, + query.crop_length, + &to_retrieve_ids, + &fields_ids_map, + &displayed_ids, + ); + + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut documents = Vec::new(); + + let documents_iter = self.documents(&rtxn, documents_ids)?; + + for (_id, obkv) in documents_iter { + let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?; + + let matches_info = query + .matches + .then(|| compute_matches(&matching_words, &document, &analyzer)); + + let formatted = format_fields( + &fields_ids_map, + obkv, + &formatter, + &matching_words, + &formatted_options, + )?; + + let hit = SearchHit { + document, + formatted, + matches_info, + }; + documents.push(hit); + } + + let nb_hits = candidates.len(); + + let facets_distribution = match query.facets_distribution { + Some(ref fields) => { + let mut facets_distribution = self.facets_distribution(&rtxn); + if fields.iter().all(|f| f != "*") { + facets_distribution.facets(fields); + } + let distribution = facets_distribution.candidates(candidates).execute()?; + + Some(distribution) + } + None => None, + }; + + let exhaustive_facets_count = facets_distribution.as_ref().map(|_| false); // not implemented yet + + let result = SearchResult { + exhaustive_nb_hits: false, // not implemented yet + hits: documents, + nb_hits, + query: query.q.clone().unwrap_or_default(), + limit: query.limit, + offset: query.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + facets_distribution, + exhaustive_facets_count, + }; + Ok(result) + } +} + +fn compute_matches>( + matcher: &impl Matcher, + document: &Document, + analyzer: &Analyzer, +) -> MatchesInfo { + let mut matches = BTreeMap::new(); + + for (key, value) in document { + let mut infos = Vec::new(); + compute_value_matches(&mut infos, value, matcher, &analyzer); + if !infos.is_empty() { + matches.insert(key.clone(), infos); + } + } + matches +} + +fn compute_value_matches<'a, A: AsRef<[u8]>>( + infos: &mut Vec, + value: &Value, + matcher: &impl Matcher, + analyzer: &Analyzer<'a, A>, +) { + match value { + Value::String(s) => { + let analyzed = analyzer.analyze(s); + let mut start = 0; + for (word, token) in analyzed.reconstruct() { + if token.is_word() { + if let Some(length) = matcher.matches(token.text()) { + infos.push(MatchInfo { start, length }); + } + } + + start += word.len(); + } + } + Value::Array(vals) => vals + .iter() + .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), + Value::Object(vals) => vals + .values() + .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), + _ => (), + } +} + +fn compute_formatted_options( + attr_to_highlight: &HashSet, + attr_to_crop: &[String], + query_crop_length: usize, + to_retrieve_ids: &BTreeSet, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &BTreeSet, +) -> BTreeMap { + let mut formatted_options = BTreeMap::new(); + + add_highlight_to_formatted_options( + &mut formatted_options, + attr_to_highlight, + fields_ids_map, + displayed_ids, + ); + + add_crop_to_formatted_options( + &mut formatted_options, + attr_to_crop, + query_crop_length, + fields_ids_map, + displayed_ids, + ); + + // Should not return `_formatted` if no valid attributes to highlight/crop + if !formatted_options.is_empty() { + add_non_formatted_ids_to_formatted_options(&mut formatted_options, to_retrieve_ids); + } + + formatted_options +} + +fn add_highlight_to_formatted_options( + formatted_options: &mut BTreeMap, + attr_to_highlight: &HashSet, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &BTreeSet, +) { + for attr in attr_to_highlight { + let new_format = FormatOptions { + highlight: true, + crop: None, + }; + + if attr == "*" { + for id in displayed_ids { + formatted_options.insert(*id, new_format); + } + break; + } + + if let Some(id) = fields_ids_map.id(&attr) { + if displayed_ids.contains(&id) { + formatted_options.insert(id, new_format); + } + } + } +} + +fn add_crop_to_formatted_options( + formatted_options: &mut BTreeMap, + attr_to_crop: &[String], + crop_length: usize, + fields_ids_map: &FieldsIdsMap, + displayed_ids: &BTreeSet, +) { + for attr in attr_to_crop { + let mut split = attr.rsplitn(2, ':'); + let (attr_name, attr_len) = match split.next().zip(split.next()) { + Some((len, name)) => { + let crop_len = len.parse::().unwrap_or(crop_length); + (name, crop_len) + } + None => (attr.as_str(), crop_length), + }; + + if attr_name == "*" { + for id in displayed_ids { + formatted_options + .entry(*id) + .and_modify(|f| f.crop = Some(attr_len)) + .or_insert(FormatOptions { + highlight: false, + crop: Some(attr_len), + }); + } + } + + if let Some(id) = fields_ids_map.id(&attr_name) { + if displayed_ids.contains(&id) { + formatted_options + .entry(id) + .and_modify(|f| f.crop = Some(attr_len)) + .or_insert(FormatOptions { + highlight: false, + crop: Some(attr_len), + }); + } + } + } +} + +fn add_non_formatted_ids_to_formatted_options( + formatted_options: &mut BTreeMap, + to_retrieve_ids: &BTreeSet, +) { + for id in to_retrieve_ids { + formatted_options.entry(*id).or_insert(FormatOptions { + highlight: false, + crop: None, + }); + } +} + +fn make_document( + attributes_to_retrieve: &BTreeSet, + field_ids_map: &FieldsIdsMap, + obkv: obkv::KvReader, +) -> Result { + let mut document = Document::new(); + + for attr in attributes_to_retrieve { + if let Some(value) = obkv.get(*attr) { + let value = serde_json::from_slice(value)?; + + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*attr) + .expect("Missing field name") + .to_string(); + + document.insert(key, value); + } + } + Ok(document) +} + +fn format_fields>( + field_ids_map: &FieldsIdsMap, + obkv: obkv::KvReader, + formatter: &Formatter, + matching_words: &impl Matcher, + formatted_options: &BTreeMap, +) -> Result { + let mut document = Document::new(); + + for (id, format) in formatted_options { + if let Some(value) = obkv.get(*id) { + let mut value: Value = serde_json::from_slice(value)?; + + value = formatter.format_value(value, matching_words, *format); + + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + let key = field_ids_map + .name(*id) + .expect("Missing field name") + .to_string(); + + document.insert(key, value); + } + } + + Ok(document) +} + +/// trait to allow unit testing of `format_fields` +trait Matcher { + fn matches(&self, w: &str) -> Option; +} + +#[cfg(test)] +impl Matcher for BTreeMap<&str, Option> { + fn matches(&self, w: &str) -> Option { + self.get(w).cloned().flatten() + } +} + +impl Matcher for MatchingWords { + fn matches(&self, w: &str) -> Option { + self.matching_bytes(w) + } +} + +struct Formatter<'a, A> { + analyzer: &'a Analyzer<'a, A>, + marks: (String, String), +} + +impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { + pub fn new(analyzer: &'a Analyzer<'a, A>, marks: (String, String)) -> Self { + Self { analyzer, marks } + } + + fn format_value( + &self, + value: Value, + matcher: &impl Matcher, + format_options: FormatOptions, + ) -> Value { + match value { + Value::String(old_string) => { + let value = self.format_string(old_string, matcher, format_options); + Value::String(value) + } + Value::Array(values) => Value::Array( + values + .into_iter() + .map(|v| { + self.format_value( + v, + matcher, + FormatOptions { + highlight: format_options.highlight, + crop: None, + }, + ) + }) + .collect(), + ), + Value::Object(object) => Value::Object( + object + .into_iter() + .map(|(k, v)| { + ( + k, + self.format_value( + v, + matcher, + FormatOptions { + highlight: format_options.highlight, + crop: None, + }, + ), + ) + }) + .collect(), + ), + value => value, + } + } + + fn format_string( + &self, + s: String, + matcher: &impl Matcher, + format_options: FormatOptions, + ) -> String { + let analyzed = self.analyzer.analyze(&s); + + let tokens: Box> = match format_options.crop { + Some(crop_len) => { + let mut buffer = Vec::new(); + let mut tokens = analyzed.reconstruct().peekable(); + + while let Some((word, token)) = + tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) + { + buffer.push((word, token)); + } + + match tokens.next() { + Some(token) => { + let mut total_len: usize = buffer.iter().map(|(word, _)| word.len()).sum(); + let before_iter = buffer.into_iter().skip_while(move |(word, _)| { + total_len -= word.len(); + total_len >= crop_len + }); + + let mut taken_after = 0; + let after_iter = tokens.take_while(move |(word, _)| { + let take = taken_after < crop_len; + taken_after += word.chars().count(); + take + }); + + let iter = before_iter.chain(Some(token)).chain(after_iter); + + Box::new(iter) + } + // If no word matches in the attribute + None => { + let mut count = 0; + let iter = buffer.into_iter().take_while(move |(word, _)| { + let take = count < crop_len; + count += word.len(); + take + }); + + Box::new(iter) + } + } + } + None => Box::new(analyzed.reconstruct()), + }; + + tokens.fold(String::new(), |mut out, (word, token)| { + // Check if we need to do highlighting or computed matches before calling + // Matcher::match since the call is expensive. + if format_options.highlight && token.is_word() { + if let Some(length) = matcher.matches(token.text()) { + if format_options.highlight { + out.push_str(&self.marks.0); + out.push_str(&word[..length]); + out.push_str(&self.marks.1); + out.push_str(&word[length..]); + return out; + } + } + } + out.push_str(word); + out + }) + } +} + +fn parse_filter(facets: &Value, index: &Index, txn: &RoTxn) -> Result> { + match facets { + Value::String(expr) => { + let condition = FilterCondition::from_str(txn, index, expr)?; + Ok(Some(condition)) + } + Value::Array(arr) => parse_filter_array(txn, index, arr), + v => Err(FacetError::InvalidExpression(&["Array"], v.clone()).into()), + } +} + +fn parse_filter_array( + txn: &RoTxn, + index: &Index, + arr: &[Value], +) -> Result> { + let mut ands = Vec::new(); + for value in arr { + match value { + Value::String(s) => ands.push(Either::Right(s.clone())), + Value::Array(arr) => { + let mut ors = Vec::new(); + for value in arr { + match value { + Value::String(s) => ors.push(s.clone()), + v => { + return Err(FacetError::InvalidExpression(&["String"], v.clone()).into()) + } + } + } + ands.push(Either::Left(ors)); + } + v => { + return Err( + FacetError::InvalidExpression(&["String", "[String]"], v.clone()).into(), + ) + } + } + } + + Ok(FilterCondition::from_array(txn, &index.0, ands)?) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn no_ids_no_formatted() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let id = fields.insert("test").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let formatted_options = BTreeMap::new(); + + let matching_words = MatchingWords::default(); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert!(value.is_empty()); + } + + #[test] + fn formatted_with_highlight_in_word() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("The Hobbit".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. R. R. Tolkien".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("hobbit", Some(3)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "The Hobbit"); + assert_eq!(value["author"], "J. R. R. Tolkien"); + } + + #[test] + fn formatted_with_crop_2() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(2), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("potter", Some(6)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry Potter and"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_10() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(10), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("potter", Some(6)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry Potter and the Half"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_0() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(0), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("potter", Some(6)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Potter"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_and_no_match() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: false, + crop: Some(6), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: Some(20), + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("rowling", Some(3)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Harry "); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_and_highlight() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: Some(1), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("and", Some(3)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], " and "); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn formatted_with_crop_and_highlight_in_word() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Harry Potter and the Half-Blood Prince".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. K. Rowling".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: Some(9), + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("blood", Some(3)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "the Half-Blood Prince"); + assert_eq!(value["author"], "J. K. Rowling"); + } + + #[test] + fn test_compute_value_matches() { + let text = "Call me Ishmael. Some years ago—never mind how long precisely—having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world."; + let value = serde_json::json!(text); + + let mut matcher = BTreeMap::new(); + matcher.insert("ishmael", Some(3)); + matcher.insert("little", Some(6)); + matcher.insert("particular", Some(1)); + + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let mut infos = Vec::new(); + + compute_value_matches(&mut infos, &value, &matcher, &analyzer); + + let mut infos = infos.into_iter(); + let crop = |info: MatchInfo| &text[info.start..info.start + info.length]; + + assert_eq!(crop(infos.next().unwrap()), "Ish"); + assert_eq!(crop(infos.next().unwrap()), "little"); + assert_eq!(crop(infos.next().unwrap()), "p"); + assert_eq!(crop(infos.next().unwrap()), "little"); + assert!(infos.next().is_none()); + } + + #[test] + fn test_compute_match() { + let value = serde_json::from_str(r#"{ + "color": "Green", + "name": "Lucas Hess", + "gender": "male", + "address": "412 Losee Terrace, Blairstown, Georgia, 2825", + "about": "Mollit ad in exercitation quis Laboris . Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n" + }"#).unwrap(); + let mut matcher = BTreeMap::new(); + matcher.insert("green", Some(3)); + matcher.insert("mollit", Some(6)); + matcher.insert("laboris", Some(7)); + + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + + let matches = compute_matches(&matcher, &value, &analyzer); + assert_eq!( + format!("{:?}", matches), + r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"## + ); + } +} diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs new file mode 100644 index 000000000..4d860ed7e --- /dev/null +++ b/meilisearch-http/src/index/update_handler.rs @@ -0,0 +1,92 @@ +use std::fs::File; + +use crate::index::Index; +use grenad::CompressionType; +use milli::update::UpdateBuilder; +use rayon::ThreadPool; + +use crate::index_controller::UpdateMeta; +use crate::index_controller::{Failed, Processed, Processing}; +use crate::option::IndexerOpts; + +pub struct UpdateHandler { + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: ThreadPool, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new(opt: &IndexerOpts) -> anyhow::Result { + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(num_cpus::get() / 2)) + .build()?; + Ok(Self { + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + pub fn update_builder(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + pub fn handle_update( + &self, + meta: Processing, + content: Option, + index: Index, + ) -> Result { + use UpdateMeta::*; + + let update_id = meta.id(); + + let update_builder = self.update_builder(update_id); + + let result = match meta.meta() { + DocumentsAddition { + method, + format, + primary_key, + } => index.update_documents( + *format, + *method, + content, + update_builder, + primary_key.as_deref(), + ), + ClearDocuments => index.clear_documents(update_builder), + DeleteDocuments { ids } => index.delete_documents(ids, update_builder), + Settings(settings) => index.update_settings(&settings.clone().check(), update_builder), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.into())), + } + } +} diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs new file mode 100644 index 000000000..09535721f --- /dev/null +++ b/meilisearch-http/src/index/updates.rs @@ -0,0 +1,382 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::io; +use std::marker::PhantomData; +use std::num::NonZeroUsize; + +use flate2::read::GzDecoder; +use log::{debug, info, trace}; +use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::index_controller::UpdateResult; + +use super::error::Result; +use super::{deserialize_some, Index}; + +fn serialize_with_wildcard( + field: &Option>>, + s: S, +) -> std::result::Result +where + S: Serializer, +{ + let wildcard = vec!["*".to_string()]; + s.serialize_some(&field.as_ref().map(|o| o.as_ref().unwrap_or(&wildcard))) +} + +#[derive(Clone, Default, Debug, Serialize)] +pub struct Checked; +#[derive(Clone, Default, Debug, Serialize, Deserialize)] +pub struct Unchecked; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + serialize_with = "serialize_with_wildcard", + skip_serializing_if = "Option::is_none" + )] + pub displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + serialize_with = "serialize_with_wildcard", + skip_serializing_if = "Option::is_none" + )] + pub searchable_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub filterable_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub ranking_rules: Option>>, + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub stop_words: Option>>, + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub synonyms: Option>>>, + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none" + )] + pub distinct_attribute: Option>, + + #[serde(skip)] + pub _kind: PhantomData, +} + +impl Settings { + pub fn cleared() -> Settings { + Settings { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + filterable_attributes: Some(None), + ranking_rules: Some(None), + stop_words: Some(None), + synonyms: Some(None), + distinct_attribute: Some(None), + _kind: PhantomData, + } + } + + pub fn into_unchecked(self) -> Settings { + let Self { + displayed_attributes, + searchable_attributes, + filterable_attributes, + ranking_rules, + stop_words, + synonyms, + distinct_attribute, + .. + } = self; + + Settings { + displayed_attributes, + searchable_attributes, + filterable_attributes, + ranking_rules, + stop_words, + synonyms, + distinct_attribute, + _kind: PhantomData, + } + } +} + +impl Settings { + pub fn check(mut self) -> Settings { + let displayed_attributes = match self.displayed_attributes.take() { + Some(Some(fields)) => { + if fields.iter().any(|f| f == "*") { + Some(None) + } else { + Some(Some(fields)) + } + } + otherwise => otherwise, + }; + + let searchable_attributes = match self.searchable_attributes.take() { + Some(Some(fields)) => { + if fields.iter().any(|f| f == "*") { + Some(None) + } else { + Some(Some(fields)) + } + } + otherwise => otherwise, + }; + + Settings { + displayed_attributes, + searchable_attributes, + filterable_attributes: self.filterable_attributes, + ranking_rules: self.ranking_rules, + stop_words: self.stop_words, + synonyms: self.synonyms, + distinct_attribute: self.distinct_attribute, + _kind: PhantomData, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + pub level_group_size: Option, + pub min_level_size: Option, +} + +impl Index { + pub fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: Option, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + ) -> Result { + let mut txn = self.write_txn()?; + let result = self.update_documents_txn( + &mut txn, + format, + method, + content, + update_builder, + primary_key, + )?; + txn.commit()?; + Ok(result) + } + + pub fn update_documents_txn<'a, 'b>( + &'a self, + txn: &mut heed::RwTxn<'a, 'b>, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: Option, + update_builder: UpdateBuilder, + primary_key: Option<&str>, + ) -> Result { + trace!("performing document addition"); + + // Set the primary key if not set already, ignore if already set. + if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { + let mut builder = UpdateBuilder::new(0).settings(txn, &self); + builder.set_primary_key(primary_key.to_string()); + builder.execute(|_, _| ())?; + } + + let mut builder = update_builder.index_documents(txn, self); + builder.update_format(format); + builder.index_documents_method(method); + + let indexing_callback = + |indexing_step, update_id| debug!("update {}: {:?}", update_id, indexing_step); + + let gzipped = false; + let addition = match content { + Some(content) if gzipped => { + builder.execute(GzDecoder::new(content), indexing_callback)? + } + Some(content) => builder.execute(content, indexing_callback)?, + None => builder.execute(std::io::empty(), indexing_callback)?, + }; + + info!("document addition done: {:?}", addition); + + Ok(UpdateResult::DocumentsAddition(addition)) + } + + pub fn clear_documents(&self, update_builder: UpdateBuilder) -> Result { + // We must use the write transaction of the update here. + let mut wtxn = self.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, self); + + let _count = builder.execute()?; + + wtxn.commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into) + } + + pub fn update_settings_txn<'a, 'b>( + &'a self, + txn: &mut heed::RwTxn<'a, 'b>, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> Result { + // We must use the write transaction of the update here. + let mut builder = update_builder.settings(txn, self); + + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + if let Some(ref facet_types) = settings.filterable_attributes { + let facet_types = facet_types.clone().unwrap_or_else(HashSet::new); + builder.set_filterable_fields(facet_types); + } + + if let Some(ref criteria) = settings.ranking_rules { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + if let Some(ref stop_words) = settings.stop_words { + match stop_words { + Some(stop_words) => builder.set_stop_words(stop_words.clone()), + None => builder.reset_stop_words(), + } + } + + if let Some(ref synonyms) = settings.synonyms { + match synonyms { + Some(synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()), + None => builder.reset_synonyms(), + } + } + + if let Some(ref distinct_attribute) = settings.distinct_attribute { + match distinct_attribute { + Some(attr) => builder.set_distinct_field(attr.clone()), + None => builder.reset_distinct_field(), + } + } + + builder.execute(|indexing_step, update_id| { + debug!("update {}: {:?}", update_id, indexing_step) + })?; + + Ok(UpdateResult::Other) + } + + pub fn update_settings( + &self, + settings: &Settings, + update_builder: UpdateBuilder, + ) -> Result { + let mut txn = self.write_txn()?; + let result = self.update_settings_txn(&mut txn, settings, update_builder)?; + txn.commit()?; + Ok(result) + } + + pub fn delete_documents( + &self, + document_ids: &[String], + update_builder: UpdateBuilder, + ) -> Result { + let mut txn = self.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, self)?; + + // We ignore unexisting document ids + document_ids.iter().for_each(|id| { + builder.delete_external_id(id); + }); + + let deleted = builder.execute()?; + txn.commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_setting_check() { + // test no changes + let settings = Settings { + displayed_attributes: Some(Some(vec![String::from("hello")])), + searchable_attributes: Some(Some(vec![String::from("hello")])), + filterable_attributes: None, + ranking_rules: None, + stop_words: None, + synonyms: None, + distinct_attribute: None, + _kind: PhantomData::, + }; + + let checked = settings.clone().check(); + assert_eq!(settings.displayed_attributes, checked.displayed_attributes); + assert_eq!( + settings.searchable_attributes, + checked.searchable_attributes + ); + + // test wildcard + // test no changes + let settings = Settings { + displayed_attributes: Some(Some(vec![String::from("*")])), + searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])), + filterable_attributes: None, + ranking_rules: None, + stop_words: None, + synonyms: None, + distinct_attribute: None, + _kind: PhantomData::, + }; + + let checked = settings.check(); + assert_eq!(checked.displayed_attributes, Some(None)); + assert_eq!(checked.searchable_attributes, Some(None)); + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs new file mode 100644 index 000000000..eee733c4a --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -0,0 +1,157 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_stream::stream; +use chrono::Utc; +use futures::{lock::Mutex, stream::StreamExt}; +use log::{error, trace}; +use tokio::sync::{mpsc, oneshot, RwLock}; +use update_actor::UpdateActorHandle; +use uuid_resolver::UuidResolverHandle; + +use super::error::{DumpActorError, Result}; +use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask}; +use crate::index_controller::{update_actor, uuid_resolver}; + +pub const CONCURRENT_DUMP_MSG: usize = 10; + +pub struct DumpActor { + inbox: Option>, + uuid_resolver: UuidResolver, + update: Update, + dump_path: PathBuf, + lock: Arc>, + dump_infos: Arc>>, + update_db_size: usize, + index_db_size: usize, +} + +/// Generate uid from creation date +fn generate_uid() -> String { + Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +} + +impl DumpActor +where + UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static, + Update: UpdateActorHandle + Send + Sync + Clone + 'static, +{ + pub fn new( + inbox: mpsc::Receiver, + uuid_resolver: UuidResolver, + update: Update, + dump_path: impl AsRef, + index_db_size: usize, + update_db_size: usize, + ) -> Self { + let dump_infos = Arc::new(RwLock::new(HashMap::new())); + let lock = Arc::new(Mutex::new(())); + Self { + inbox: Some(inbox), + uuid_resolver, + update, + dump_path: dump_path.as_ref().into(), + dump_infos, + lock, + index_db_size, + update_db_size, + } + } + + pub async fn run(mut self) { + trace!("Started dump actor."); + + let mut inbox = self + .inbox + .take() + .expect("Dump Actor must have a inbox at this point."); + + let stream = stream! { + loop { + match inbox.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + stream + .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg)) + .await; + + error!("Dump actor stopped."); + } + + async fn handle_message(&self, msg: DumpMsg) { + use DumpMsg::*; + + match msg { + CreateDump { ret } => { + let _ = self.handle_create_dump(ret).await; + } + DumpInfo { ret, uid } => { + let _ = ret.send(self.handle_dump_info(uid).await); + } + } + } + + async fn handle_create_dump(&self, ret: oneshot::Sender>) { + let uid = generate_uid(); + let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); + + let _lock = match self.lock.try_lock() { + Some(lock) => lock, + None => { + ret.send(Err(DumpActorError::DumpAlreadyRunning)) + .expect("Dump actor is dead"); + return; + } + }; + + self.dump_infos + .write() + .await + .insert(uid.clone(), info.clone()); + + ret.send(Ok(info)).expect("Dump actor is dead"); + + let task = DumpTask { + path: self.dump_path.clone(), + uuid_resolver: self.uuid_resolver.clone(), + update_handle: self.update.clone(), + uid: uid.clone(), + update_db_size: self.update_db_size, + index_db_size: self.index_db_size, + }; + + let task_result = tokio::task::spawn(task.run()).await; + + let mut dump_infos = self.dump_infos.write().await; + let dump_infos = dump_infos + .get_mut(&uid) + .expect("dump entry deleted while lock was acquired"); + + match task_result { + Ok(Ok(())) => { + dump_infos.done(); + trace!("Dump succeed"); + } + Ok(Err(e)) => { + dump_infos.with_error(e.to_string()); + error!("Dump failed: {}", e); + } + Err(_) => { + dump_infos.with_error("Unexpected error while performing dump.".to_string()); + error!("Dump panicked. Dump status set to failed"); + } + }; + } + + async fn handle_dump_info(&self, uid: String) -> Result { + match self.dump_infos.read().await.get(&uid) { + Some(info) => Ok(info.clone()), + _ => Err(DumpActorError::DumpDoesNotExist(uid)), + } + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/error.rs b/meilisearch-http/src/index_controller/dump_actor/error.rs new file mode 100644 index 000000000..b6bddb5ea --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/error.rs @@ -0,0 +1,52 @@ +use meilisearch_error::{Code, ErrorCode}; + +use crate::index_controller::update_actor::error::UpdateActorError; +use crate::index_controller::uuid_resolver::error::UuidResolverError; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum DumpActorError { + #[error("Another dump is already in progress")] + DumpAlreadyRunning, + #[error("Dump `{0}` not found")] + DumpDoesNotExist(String), + #[error("Internal error: {0}")] + Internal(Box), + #[error("{0}")] + UuidResolver(#[from] UuidResolverError), + #[error("{0}")] + UpdateActor(#[from] UpdateActorError), +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for DumpActorError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +internal_error!( + heed::Error, + std::io::Error, + tokio::task::JoinError, + serde_json::error::Error, + tempfile::PersistError +); + +impl ErrorCode for DumpActorError { + fn error_code(&self) -> Code { + match self { + DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress, + DumpActorError::DumpDoesNotExist(_) => Code::NotFound, + DumpActorError::Internal(_) => Code::Internal, + DumpActorError::UuidResolver(e) => e.error_code(), + DumpActorError::UpdateActor(e) => e.error_code(), + } + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs new file mode 100644 index 000000000..db11fb8fc --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/handle_impl.rs @@ -0,0 +1,53 @@ +use std::path::Path; + +use actix_web::web::Bytes; +use tokio::sync::{mpsc, oneshot}; + +use super::error::Result; +use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg}; + +#[derive(Clone)] +pub struct DumpActorHandleImpl { + sender: mpsc::Sender, +} + +#[async_trait::async_trait] +impl DumpActorHandle for DumpActorHandleImpl { + async fn create_dump(&self) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = DumpMsg::CreateDump { ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } + + async fn dump_info(&self, uid: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = DumpMsg::DumpInfo { ret, uid }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } +} + +impl DumpActorHandleImpl { + pub fn new( + path: impl AsRef, + uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl, + update: crate::index_controller::update_actor::UpdateActorHandleImpl, + index_db_size: usize, + update_db_size: usize, + ) -> anyhow::Result { + let (sender, receiver) = mpsc::channel(10); + let actor = DumpActor::new( + receiver, + uuid_resolver, + update, + path, + index_db_size, + update_db_size, + ); + + tokio::task::spawn(actor.run()); + + Ok(Self { sender }) + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs new file mode 100644 index 000000000..ae6adc7cf --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs @@ -0,0 +1,2 @@ +pub mod v1; +pub mod v2; diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs new file mode 100644 index 000000000..a7f1aa8d1 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -0,0 +1,182 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{create_dir_all, File}; +use std::io::BufRead; +use std::marker::PhantomData; +use std::path::Path; +use std::sync::Arc; + +use heed::EnvOpenOptions; +use log::{error, info, warn}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata}; +use crate::{ + index::{deserialize_some, update_handler::UpdateHandler, Index, Unchecked}, + option::IndexerOpts, +}; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct MetadataV1 { + db_version: String, + indexes: Vec, +} + +impl MetadataV1 { + pub fn load_dump( + self, + src: impl AsRef, + dst: impl AsRef, + size: usize, + indexer_options: &IndexerOpts, + ) -> anyhow::Result<()> { + info!( + "Loading dump, dump database version: {}, dump version: V1", + self.db_version + ); + + let uuid_store = HeedUuidStore::new(&dst)?; + for index in self.indexes { + let uuid = Uuid::new_v4(); + uuid_store.insert(index.uid.clone(), uuid)?; + let src = src.as_ref().join(index.uid); + load_index( + &src, + &dst, + uuid, + index.meta.primary_key.as_deref(), + size, + indexer_options, + )?; + } + + Ok(()) + } +} + +// These are the settings used in legacy meilisearch (>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub distinct_attribute: Option>, + #[serde(default, deserialize_with = "deserialize_some")] + pub searchable_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub displayed_attributes: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub stop_words: Option>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub synonyms: Option>>>, + #[serde(default, deserialize_with = "deserialize_some")] + pub filterable_attributes: Option>>, +} + +fn load_index( + src: impl AsRef, + dst: impl AsRef, + uuid: Uuid, + primary_key: Option<&str>, + size: usize, + indexer_options: &IndexerOpts, +) -> anyhow::Result<()> { + let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid)); + + create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(size); + let index = milli::Index::new(options, index_path)?; + let index = Index(Arc::new(index)); + + // extract `settings.json` file and import content + let settings = import_settings(&src)?; + let settings: index_controller::Settings = settings.into(); + + let mut txn = index.write_txn()?; + + let handler = UpdateHandler::new(&indexer_options)?; + + index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?; + + let file = File::open(&src.as_ref().join("documents.jsonl"))?; + let mut reader = std::io::BufReader::new(file); + reader.fill_buf()?; + if !reader.buffer().is_empty() { + index.update_documents_txn( + &mut txn, + UpdateFormat::JsonStream, + IndexDocumentsMethod::ReplaceDocuments, + Some(reader), + handler.update_builder(0), + primary_key, + )?; + } + + txn.commit()?; + + // Finaly, we extract the original milli::Index and close it + Arc::try_unwrap(index.0) + .map_err(|_e| "Couldn't close the index properly") + .unwrap() + .prepare_for_closing() + .wait(); + + // Updates are ignored in dumps V1. + + Ok(()) +} + +/// we need to **always** be able to convert the old settings to the settings currently being used +impl From for index_controller::Settings { + fn from(settings: Settings) -> Self { + Self { + distinct_attribute: settings.distinct_attribute, + // we need to convert the old `Vec` into a `BTreeSet` + displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + searchable_attributes: settings.searchable_attributes, + // we previously had a `Vec` but now we have a `HashMap` + // representing the name of the faceted field + the type of the field. Since the type + // was not known in the V1 of the dump we are just going to assume everything is a + // String + filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + // we need to convert the old `Vec` into a `BTreeSet` + ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + match criterion.as_str() { + "words" | "typo" | "proximity" | "attribute" => Some(criterion), + s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + "wordsPosition" => { + warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); + Some(String::from("words")) + } + "exactness" => { + error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); + None + } + s => { + error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); + None + } + } + }).collect())), + // we need to convert the old `Vec` into a `BTreeSet` + stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())), + // we need to convert the old `Vec` into a `BTreeMap` + synonyms: settings.synonyms.map(|o| o.map(|vec| vec.into_iter().collect())), + _kind: PhantomData, + } + } +} + +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn import_settings(dir_path: impl AsRef) -> anyhow::Result { + let path = dir_path.as_ref().join("settings.json"); + let file = File::open(path)?; + let reader = std::io::BufReader::new(file); + let metadata = serde_json::from_reader(reader)?; + + Ok(metadata) +} diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs new file mode 100644 index 000000000..eddd8a3b7 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -0,0 +1,59 @@ +use std::path::Path; + +use chrono::{DateTime, Utc}; +use log::info; +use serde::{Deserialize, Serialize}; + +use crate::index::Index; +use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}; +use crate::option::IndexerOpts; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct MetadataV2 { + db_version: String, + index_db_size: usize, + update_db_size: usize, + dump_date: DateTime, +} + +impl MetadataV2 { + pub fn new(index_db_size: usize, update_db_size: usize) -> Self { + Self { + db_version: env!("CARGO_PKG_VERSION").to_string(), + index_db_size, + update_db_size, + dump_date: Utc::now(), + } + } + + pub fn load_dump( + self, + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + update_db_size: usize, + indexing_options: &IndexerOpts, + ) -> anyhow::Result<()> { + info!( + "Loading dump from {}, dump database version: {}, dump version: V2", + self.dump_date, self.db_version + ); + + info!("Loading index database."); + HeedUuidStore::load_dump(src.as_ref(), &dst)?; + + info!("Loading updates."); + UpdateStore::load_dump(&src, &dst, update_db_size)?; + + info!("Loading indexes."); + let indexes_path = src.as_ref().join("indexes"); + let indexes = indexes_path.read_dir()?; + for index in indexes { + let index = index?; + Index::load_dump(&index.path(), &dst, index_db_size, indexing_options)?; + } + + Ok(()) + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/message.rs b/meilisearch-http/src/index_controller/dump_actor/message.rs new file mode 100644 index 000000000..6c9dded9f --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/message.rs @@ -0,0 +1,14 @@ +use tokio::sync::oneshot; + +use super::error::Result; +use super::DumpInfo; + +pub enum DumpMsg { + CreateDump { + ret: oneshot::Sender>, + }, + DumpInfo { + uid: String, + ret: oneshot::Sender>, + }, +} diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs new file mode 100644 index 000000000..a73740b02 --- /dev/null +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -0,0 +1,203 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; + +use anyhow::Context; +use chrono::{DateTime, Utc}; +use log::{info, trace, warn}; +#[cfg(test)] +use mockall::automock; +use serde::{Deserialize, Serialize}; +use tokio::fs::create_dir_all; + +use loaders::v1::MetadataV1; +use loaders::v2::MetadataV2; + +pub use actor::DumpActor; +pub use handle_impl::*; +pub use message::DumpMsg; + +use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; +use crate::index_controller::dump_actor::error::DumpActorError; +use crate::{helpers::compression, option::IndexerOpts}; +use error::Result; + +mod actor; +pub mod error; +mod handle_impl; +mod loaders; +mod message; + +const META_FILE_NAME: &str = "metadata.json"; + +#[async_trait::async_trait] +#[cfg_attr(test, automock)] +pub trait DumpActorHandle { + /// Start the creation of a dump + /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump] + async fn create_dump(&self) -> Result; + + /// Return the status of an already created dump + /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status] + async fn dump_info(&self, uid: String) -> Result; +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "dumpVersion")] +pub enum Metadata { + V1(MetadataV1), + V2(MetadataV2), +} + +impl Metadata { + pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self { + let meta = MetadataV2::new(index_db_size, update_db_size); + Self::V2(meta) + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[serde(rename_all = "snake_case")] +pub enum DumpStatus { + Done, + InProgress, + Failed, +} + +#[derive(Debug, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DumpInfo { + pub uid: String, + pub status: DumpStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + started_at: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + finished_at: Option>, +} + +impl DumpInfo { + pub fn new(uid: String, status: DumpStatus) -> Self { + Self { + uid, + status, + error: None, + started_at: Utc::now(), + finished_at: None, + } + } + + pub fn with_error(&mut self, error: String) { + self.status = DumpStatus::Failed; + self.finished_at = Some(Utc::now()); + self.error = Some(error); + } + + pub fn done(&mut self) { + self.finished_at = Some(Utc::now()); + self.status = DumpStatus::Done; + } + + pub fn dump_already_in_progress(&self) -> bool { + self.status == DumpStatus::InProgress + } +} + +pub fn load_dump( + dst_path: impl AsRef, + src_path: impl AsRef, + index_db_size: usize, + update_db_size: usize, + indexer_opts: &IndexerOpts, +) -> anyhow::Result<()> { + let tmp_src = tempfile::tempdir_in(".")?; + let tmp_src_path = tmp_src.path(); + + compression::from_tar_gz(&src_path, tmp_src_path)?; + + let meta_path = tmp_src_path.join(META_FILE_NAME); + let mut meta_file = File::open(&meta_path)?; + let meta: Metadata = serde_json::from_reader(&mut meta_file)?; + + let dst_dir = dst_path + .as_ref() + .parent() + .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?; + + let tmp_dst = tempfile::tempdir_in(dst_dir)?; + + match meta { + Metadata::V1(meta) => { + meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)? + } + Metadata::V2(meta) => meta.load_dump( + &tmp_src_path, + tmp_dst.path(), + index_db_size, + update_db_size, + indexer_opts, + )?, + } + // Persist and atomically rename the db + let persisted_dump = tmp_dst.into_path(); + if dst_path.as_ref().exists() { + warn!("Overwriting database at {}", dst_path.as_ref().display()); + std::fs::remove_dir_all(&dst_path)?; + } + + std::fs::rename(&persisted_dump, &dst_path)?; + + Ok(()) +} + +struct DumpTask { + path: PathBuf, + uuid_resolver: U, + update_handle: P, + uid: String, + update_db_size: usize, + index_db_size: usize, +} + +impl DumpTask +where + U: UuidResolverHandle + Send + Sync + Clone + 'static, + P: UpdateActorHandle + Send + Sync + Clone + 'static, +{ + async fn run(self) -> Result<()> { + trace!("Performing dump."); + + create_dir_all(&self.path).await?; + + let path_clone = self.path.clone(); + let temp_dump_dir = + tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); + let meta_path = temp_dump_path.join(META_FILE_NAME); + let mut meta_file = File::create(&meta_path)?; + serde_json::to_writer(&mut meta_file, &meta)?; + + let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?; + + self.update_handle + .dump(uuids, temp_dump_path.clone()) + .await?; + + let dump_path = tokio::task::spawn_blocking(move || -> Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path()) + .map_err(|e| DumpActorError::Internal(e.into()))?; + + let dump_path = self.path.join(self.uid).with_extension("dump"); + temp_dump_file.persist(&dump_path)?; + + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } +} diff --git a/meilisearch-http/src/index_controller/error.rs b/meilisearch-http/src/index_controller/error.rs new file mode 100644 index 000000000..00f6b8656 --- /dev/null +++ b/meilisearch-http/src/index_controller/error.rs @@ -0,0 +1,40 @@ +use meilisearch_error::Code; +use meilisearch_error::ErrorCode; + +use crate::index::error::IndexError; + +use super::dump_actor::error::DumpActorError; +use super::index_actor::error::IndexActorError; +use super::update_actor::error::UpdateActorError; +use super::uuid_resolver::error::UuidResolverError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IndexControllerError { + #[error("Index creation must have an uid")] + MissingUid, + #[error("{0}")] + Uuid(#[from] UuidResolverError), + #[error("{0}")] + IndexActor(#[from] IndexActorError), + #[error("{0}")] + UpdateActor(#[from] UpdateActorError), + #[error("{0}")] + DumpActor(#[from] DumpActorError), + #[error("{0}")] + IndexError(#[from] IndexError), +} + +impl ErrorCode for IndexControllerError { + fn error_code(&self) -> Code { + match self { + IndexControllerError::MissingUid => Code::BadRequest, + IndexControllerError::Uuid(e) => e.error_code(), + IndexControllerError::IndexActor(e) => e.error_code(), + IndexControllerError::UpdateActor(e) => e.error_code(), + IndexControllerError::DumpActor(e) => e.error_code(), + IndexControllerError::IndexError(e) => e.error_code(), + } + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs new file mode 100644 index 000000000..15d96b7ad --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -0,0 +1,348 @@ +use std::fs::File; +use std::path::PathBuf; +use std::sync::Arc; + +use async_stream::stream; +use futures::stream::StreamExt; +use heed::CompactionOption; +use log::debug; +use milli::update::UpdateBuilder; +use tokio::task::spawn_blocking; +use tokio::{fs, sync::mpsc}; +use uuid::Uuid; + +use crate::index::{ + update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings, +}; +use crate::index_controller::{ + get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing, +}; +use crate::option::IndexerOpts; + +use super::error::{IndexActorError, Result}; +use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore}; + +pub const CONCURRENT_INDEX_MSG: usize = 10; + +pub struct IndexActor { + receiver: Option>, + update_handler: Arc, + store: S, +} + +impl IndexActor { + pub fn new(receiver: mpsc::Receiver, store: S) -> anyhow::Result { + let options = IndexerOpts::default(); + let update_handler = UpdateHandler::new(&options)?; + let update_handler = Arc::new(update_handler); + let receiver = Some(receiver); + Ok(Self { + receiver, + update_handler, + store, + }) + } + + /// `run` poll the write_receiver and read_receiver concurrently, but while messages send + /// through the read channel are processed concurrently, the messages sent through the write + /// channel are processed one at a time. + pub async fn run(mut self) { + let mut receiver = self + .receiver + .take() + .expect("Index Actor must have a inbox at this point."); + + let stream = stream! { + loop { + match receiver.recv().await { + Some(msg) => yield msg, + None => break, + } + } + }; + + stream + .for_each_concurrent(Some(CONCURRENT_INDEX_MSG), |msg| self.handle_message(msg)) + .await; + } + + async fn handle_message(&self, msg: IndexMsg) { + use IndexMsg::*; + match msg { + CreateIndex { + uuid, + primary_key, + ret, + } => { + let _ = ret.send(self.handle_create_index(uuid, primary_key).await); + } + Update { + ret, + meta, + data, + uuid, + } => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + Search { ret, query, uuid } => { + let _ = ret.send(self.handle_search(uuid, query).await); + } + Settings { ret, uuid } => { + let _ = ret.send(self.handle_settings(uuid).await); + } + Documents { + ret, + uuid, + attributes_to_retrieve, + offset, + limit, + } => { + let _ = ret.send( + self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve) + .await, + ); + } + Document { + uuid, + attributes_to_retrieve, + doc_id, + ret, + } => { + let _ = ret.send( + self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve) + .await, + ); + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + } + GetMeta { uuid, ret } => { + let _ = ret.send(self.handle_get_meta(uuid).await); + } + UpdateIndex { + uuid, + index_settings, + ret, + } => { + let _ = ret.send(self.handle_update_index(uuid, index_settings).await); + } + Snapshot { uuid, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuid, path).await); + } + Dump { uuid, path, ret } => { + let _ = ret.send(self.handle_dump(uuid, path).await); + } + GetStats { uuid, ret } => { + let _ = ret.send(self.handle_get_stats(uuid).await); + } + } + } + + async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + let result = spawn_blocking(move || index.perform_search(query)).await??; + Ok(result) + } + + async fn handle_create_index( + &self, + uuid: Uuid, + primary_key: Option, + ) -> Result { + let index = self.store.create(uuid, primary_key).await?; + let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; + Ok(meta) + } + + async fn handle_update( + &self, + uuid: Uuid, + meta: Processing, + data: Option, + ) -> Result> { + debug!("Processing update {}", meta.id()); + let update_handler = self.update_handler.clone(); + let index = match self.store.get(uuid).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, + }; + + Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?) + } + + async fn handle_settings(&self, uuid: Uuid) -> Result> { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + let result = spawn_blocking(move || index.settings()).await??; + Ok(result) + } + + async fn handle_fetch_documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + let result = + spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)) + .await??; + + Ok(result) + } + + async fn handle_fetch_document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + + let result = + spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve)) + .await??; + + Ok(result) + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let index = self.store.delete(uuid).await?; + + if let Some(index) = index { + tokio::task::spawn(async move { + let index = index.0; + let store = get_arc_ownership_blocking(index).await; + spawn_blocking(move || { + store.prepare_for_closing().wait(); + debug!("Index closed"); + }); + }); + } + + Ok(()) + } + + async fn handle_get_meta(&self, uuid: Uuid) -> Result { + match self.store.get(uuid).await? { + Some(index) => { + let meta = spawn_blocking(move || IndexMeta::new(&index)).await??; + Ok(meta) + } + None => Err(IndexActorError::UnexistingIndex), + } + } + + async fn handle_update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + + let result = spawn_blocking(move || match index_settings.primary_key { + Some(primary_key) => { + let mut txn = index.write_txn()?; + if index.primary_key(&txn)?.is_some() { + return Err(IndexActorError::ExistingPrimaryKey); + } + let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); + builder.set_primary_key(primary_key); + builder.execute(|_, _| ())?; + let meta = IndexMeta::new_txn(&index, &txn)?; + txn.commit()?; + Ok(meta) + } + None => { + let meta = IndexMeta::new(&index)?; + Ok(meta) + } + }) + .await??; + + Ok(result) + } + + async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> { + use tokio::fs::create_dir_all; + + path.push("indexes"); + create_dir_all(&path).await?; + + if let Some(index) = self.store.get(uuid).await? { + let mut index_path = path.join(format!("index-{}", uuid)); + + create_dir_all(&index_path).await?; + + index_path.push("data.mdb"); + spawn_blocking(move || -> Result<()> { + // Get write txn to wait for ongoing write transaction before snapshot. + let _txn = index.write_txn()?; + index + .env + .copy_to_path(index_path, CompactionOption::Enabled)?; + Ok(()) + }) + .await??; + } + + Ok(()) + } + + /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the + /// documents and all the settings. + async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + + let path = path.join(format!("indexes/index-{}/", uuid)); + fs::create_dir_all(&path).await?; + + tokio::task::spawn_blocking(move || index.dump(path)).await??; + + Ok(()) + } + + async fn handle_get_stats(&self, uuid: Uuid) -> Result { + let index = self + .store + .get(uuid) + .await? + .ok_or(IndexActorError::UnexistingIndex)?; + + spawn_blocking(move || { + let rtxn = index.read_txn()?; + + Ok(IndexStats { + size: index.size(), + number_of_documents: index.number_of_documents(&rtxn)?, + is_indexing: None, + field_distribution: index.field_distribution(&rtxn)?, + }) + }) + .await? + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/error.rs b/meilisearch-http/src/index_controller/index_actor/error.rs new file mode 100644 index 000000000..12a81796b --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/error.rs @@ -0,0 +1,48 @@ +use meilisearch_error::{Code, ErrorCode}; + +use crate::{error::MilliError, index::error::IndexError}; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum IndexActorError { + #[error("{0}")] + IndexError(#[from] IndexError), + #[error("Index already exists")] + IndexAlreadyExists, + #[error("Index not found")] + UnexistingIndex, + #[error("A primary key is already present. It's impossible to update it")] + ExistingPrimaryKey, + #[error("Internal Error: {0}")] + Internal(Box), + #[error("{0}")] + Milli(#[from] milli::Error), +} + +macro_rules! internal_error { + ($($other:path), *) => { + $( + impl From<$other> for IndexActorError { + fn from(other: $other) -> Self { + Self::Internal(Box::new(other)) + } + } + )* + } +} + +internal_error!(heed::Error, tokio::task::JoinError, std::io::Error); + +impl ErrorCode for IndexActorError { + fn error_code(&self) -> Code { + match self { + IndexActorError::IndexError(e) => e.error_code(), + IndexActorError::IndexAlreadyExists => Code::IndexAlreadyExists, + IndexActorError::UnexistingIndex => Code::IndexNotFound, + IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent, + IndexActorError::Internal(_) => Code::Internal, + IndexActorError::Milli(e) => MilliError(e).error_code(), + } + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs new file mode 100644 index 000000000..231a3a44b --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -0,0 +1,159 @@ +use std::path::{Path, PathBuf}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use crate::{ + index::Checked, + index_controller::{IndexSettings, IndexStats, Processing}, +}; +use crate::{ + index::{Document, SearchQuery, SearchResult, Settings}, + index_controller::{Failed, Processed}, +}; + +use super::error::Result; +use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore}; + +#[derive(Clone)] +pub struct IndexActorHandleImpl { + sender: mpsc::Sender, +} + +#[async_trait::async_trait] +impl IndexActorHandle for IndexActorHandleImpl { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::CreateIndex { + ret, + uuid, + primary_key, + }; + let _ = self.sender.send(msg).await; + receiver.await.expect("IndexActor has been killed") + } + + async fn update( + &self, + uuid: Uuid, + meta: Processing, + data: Option, + ) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Update { + ret, + meta, + data, + uuid, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Search { uuid, query, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn settings(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Settings { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Documents { + uuid, + ret, + offset, + attributes_to_retrieve, + limit, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Document { + uuid, + ret, + doc_id, + attributes_to_retrieve, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Delete { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn get_index_meta(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetMeta { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::UpdateIndex { + uuid, + index_settings, + ret, + }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Snapshot { uuid, path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::Dump { uuid, path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } + + async fn get_index_stats(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = IndexMsg::GetStats { uuid, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver.await.expect("IndexActor has been killed")?) + } +} + +impl IndexActorHandleImpl { + pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + let (sender, receiver) = mpsc::channel(100); + + let store = MapIndexStore::new(path, index_size); + let actor = IndexActor::new(receiver, store)?; + tokio::task::spawn(actor.run()); + Ok(Self { sender }) + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs new file mode 100644 index 000000000..415b90e4b --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -0,0 +1,74 @@ +use std::path::PathBuf; + +use tokio::sync::oneshot; +use uuid::Uuid; + +use super::error::Result as IndexResult; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{Failed, IndexStats, Processed, Processing}; + +use super::{IndexMeta, IndexSettings}; + +#[allow(clippy::large_enum_variant)] +pub enum IndexMsg { + CreateIndex { + uuid: Uuid, + primary_key: Option, + ret: oneshot::Sender>, + }, + Update { + uuid: Uuid, + meta: Processing, + data: Option, + ret: oneshot::Sender>>, + }, + Search { + uuid: Uuid, + query: SearchQuery, + ret: oneshot::Sender>, + }, + Settings { + uuid: Uuid, + ret: oneshot::Sender>>, + }, + Documents { + uuid: Uuid, + attributes_to_retrieve: Option>, + offset: usize, + limit: usize, + ret: oneshot::Sender>>, + }, + Document { + uuid: Uuid, + attributes_to_retrieve: Option>, + doc_id: String, + ret: oneshot::Sender>, + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, + GetMeta { + uuid: Uuid, + ret: oneshot::Sender>, + }, + UpdateIndex { + uuid: Uuid, + index_settings: IndexSettings, + ret: oneshot::Sender>, + }, + Snapshot { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, + Dump { + uuid: Uuid, + path: PathBuf, + ret: oneshot::Sender>, + }, + GetStats { + uuid: Uuid, + ret: oneshot::Sender>, + }, +} diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs new file mode 100644 index 000000000..4085dc0bd --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -0,0 +1,169 @@ +use std::fs::File; +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use mockall::automock; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use actor::IndexActor; +pub use actor::CONCURRENT_INDEX_MSG; +pub use handle_impl::IndexActorHandleImpl; +use message::IndexMsg; +use store::{IndexStore, MapIndexStore}; + +use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings}; +use crate::index_controller::{Failed, IndexStats, Processed, Processing}; +use error::Result; + +use super::IndexSettings; + +mod actor; +pub mod error; +mod handle_impl; +mod message; +mod store; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMeta { + created_at: DateTime, + pub updated_at: DateTime, + pub primary_key: Option, +} + +impl IndexMeta { + fn new(index: &Index) -> Result { + let txn = index.read_txn()?; + Self::new_txn(index, &txn) + } + + fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; + let primary_key = index.primary_key(&txn)?.map(String::from); + Ok(Self { + created_at, + updated_at, + primary_key, + }) + } +} + +#[async_trait::async_trait] +#[cfg_attr(test, automock)] +pub trait IndexActorHandle { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn update( + &self, + uuid: Uuid, + meta: Processing, + data: Option, + ) -> Result>; + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result; + async fn settings(&self, uuid: Uuid) -> Result>; + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result>; + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result; + async fn delete(&self, uuid: Uuid) -> Result<()>; + async fn get_index_meta(&self, uuid: Uuid) -> Result; + async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result; + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn get_index_stats(&self, uuid: Uuid) -> Result; +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::*; + + #[async_trait::async_trait] + /// Useful for passing around an `Arc` in tests. + impl IndexActorHandle for Arc { + async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + self.as_ref().create_index(uuid, primary_key).await + } + + async fn update( + &self, + uuid: Uuid, + meta: Processing, + data: Option, + ) -> Result> { + self.as_ref().update(uuid, meta, data).await + } + + async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result { + self.as_ref().search(uuid, query).await + } + + async fn settings(&self, uuid: Uuid) -> Result> { + self.as_ref().settings(uuid).await + } + + async fn documents( + &self, + uuid: Uuid, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + self.as_ref() + .documents(uuid, offset, limit, attributes_to_retrieve) + .await + } + + async fn document( + &self, + uuid: Uuid, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + self.as_ref() + .document(uuid, doc_id, attributes_to_retrieve) + .await + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + self.as_ref().delete(uuid).await + } + + async fn get_index_meta(&self, uuid: Uuid) -> Result { + self.as_ref().get_index_meta(uuid).await + } + + async fn update_index( + &self, + uuid: Uuid, + index_settings: IndexSettings, + ) -> Result { + self.as_ref().update_index(uuid, index_settings).await + } + + async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + self.as_ref().snapshot(uuid, path).await + } + + async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> { + self.as_ref().dump(uuid, path).await + } + + async fn get_index_stats(&self, uuid: Uuid) -> Result { + self.as_ref().get_index_stats(uuid).await + } + } +} diff --git a/meilisearch-http/src/index_controller/index_actor/store.rs b/meilisearch-http/src/index_controller/index_actor/store.rs new file mode 100644 index 000000000..2cfda61b5 --- /dev/null +++ b/meilisearch-http/src/index_controller/index_actor/store.rs @@ -0,0 +1,103 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use milli::update::UpdateBuilder; +use tokio::fs; +use tokio::sync::RwLock; +use tokio::task::spawn_blocking; +use uuid::Uuid; + +use super::error::{IndexActorError, Result}; +use crate::index::Index; + +type AsyncMap = Arc>>; + +#[async_trait::async_trait] +pub trait IndexStore { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; + async fn get(&self, uuid: Uuid) -> Result>; + async fn delete(&self, uuid: Uuid) -> Result>; +} + +pub struct MapIndexStore { + index_store: AsyncMap, + path: PathBuf, + index_size: usize, +} + +impl MapIndexStore { + pub fn new(path: impl AsRef, index_size: usize) -> Self { + let path = path.as_ref().join("indexes/"); + let index_store = Arc::new(RwLock::new(HashMap::new())); + Self { + index_store, + path, + index_size, + } + } +} + +#[async_trait::async_trait] +impl IndexStore for MapIndexStore { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { + // We need to keep the lock until we are sure the db file has been opened correclty, to + // ensure that another db is not created at the same time. + let mut lock = self.index_store.write().await; + + if let Some(index) = lock.get(&uuid) { + return Ok(index.clone()); + } + let path = self.path.join(format!("index-{}", uuid)); + if path.exists() { + return Err(IndexActorError::IndexAlreadyExists); + } + + let index_size = self.index_size; + let index = spawn_blocking(move || -> Result { + let index = Index::open(path, index_size)?; + if let Some(primary_key) = primary_key { + let mut txn = index.write_txn()?; + + let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index); + builder.set_primary_key(primary_key); + builder.execute(|_, _| ())?; + + txn.commit()?; + } + Ok(index) + }) + .await??; + + lock.insert(uuid, index.clone()); + + Ok(index) + } + + async fn get(&self, uuid: Uuid) -> Result> { + let guard = self.index_store.read().await; + match guard.get(&uuid) { + Some(index) => Ok(Some(index.clone())), + None => { + // drop the guard here so we can perform the write after without deadlocking; + drop(guard); + let path = self.path.join(format!("index-{}", uuid)); + if !path.exists() { + return Ok(None); + } + + let index_size = self.index_size; + let index = spawn_blocking(move || Index::open(path, index_size)).await??; + self.index_store.write().await.insert(uuid, index.clone()); + Ok(Some(index)) + } + } + } + + async fn delete(&self, uuid: Uuid) -> Result> { + let db_path = self.path.join(format!("index-{}", uuid)); + fs::remove_dir_all(db_path).await?; + let index = self.index_store.write().await.remove(&uuid); + Ok(index) + } +} diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs new file mode 100644 index 000000000..a90498b9c --- /dev/null +++ b/meilisearch-http/src/index_controller/mod.rs @@ -0,0 +1,441 @@ +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use actix_web::web::Bytes; +use chrono::{DateTime, Utc}; +use futures::stream::StreamExt; +use log::error; +use log::info; +use milli::FieldDistribution; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio::time::sleep; +use uuid::Uuid; + +use dump_actor::DumpActorHandle; +pub use dump_actor::{DumpInfo, DumpStatus}; +use index_actor::IndexActorHandle; +use snapshot::{load_snapshot, SnapshotService}; +use update_actor::UpdateActorHandle; +pub use updates::*; +use uuid_resolver::{error::UuidResolverError, UuidResolverHandle}; + +use crate::extractors::payload::Payload; +use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings}; +use crate::option::Opt; +use error::Result; + +use self::dump_actor::load_dump; +use self::error::IndexControllerError; + +mod dump_actor; +pub mod error; +pub mod index_actor; +mod snapshot; +mod update_actor; +mod updates; +mod uuid_resolver; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMetadata { + #[serde(skip)] + pub uuid: Uuid, + pub uid: String, + name: String, + #[serde(flatten)] + pub meta: index_actor::IndexMeta, +} + +#[derive(Clone, Debug)] +pub struct IndexSettings { + pub uid: Option, + pub primary_key: Option, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct IndexStats { + #[serde(skip)] + pub size: u64, + pub number_of_documents: u64, + /// Whether the current index is performing an update. It is initially `None` when the + /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is + /// later set to either true or false, we we retrieve the information from the `UpdateStore` + pub is_indexing: Option, + pub field_distribution: FieldDistribution, +} + +#[derive(Clone)] +pub struct IndexController { + uuid_resolver: uuid_resolver::UuidResolverHandleImpl, + index_handle: index_actor::IndexActorHandleImpl, + update_handle: update_actor::UpdateActorHandleImpl, + dump_handle: dump_actor::DumpActorHandleImpl, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Stats { + pub database_size: u64, + pub last_update: Option>, + pub indexes: BTreeMap, +} + +impl IndexController { + pub fn new(path: impl AsRef, options: &Opt) -> anyhow::Result { + let index_size = options.max_index_size.get_bytes() as usize; + let update_store_size = options.max_index_size.get_bytes() as usize; + + if let Some(ref path) = options.import_snapshot { + info!("Loading from snapshot {:?}", path); + load_snapshot( + &options.db_path, + path, + options.ignore_snapshot_if_db_exists, + options.ignore_missing_snapshot, + )?; + } else if let Some(ref src_path) = options.import_dump { + load_dump( + &options.db_path, + src_path, + options.max_index_size.get_bytes() as usize, + options.max_udb_size.get_bytes() as usize, + &options.indexer_options, + )?; + } + + std::fs::create_dir_all(&path)?; + + let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?; + let index_handle = index_actor::IndexActorHandleImpl::new(&path, index_size)?; + let update_handle = update_actor::UpdateActorHandleImpl::new( + index_handle.clone(), + &path, + update_store_size, + )?; + let dump_handle = dump_actor::DumpActorHandleImpl::new( + &options.dumps_dir, + uuid_resolver.clone(), + update_handle.clone(), + options.max_index_size.get_bytes() as usize, + options.max_udb_size.get_bytes() as usize, + )?; + + if options.schedule_snapshot { + let snapshot_service = SnapshotService::new( + uuid_resolver.clone(), + update_handle.clone(), + Duration::from_secs(options.snapshot_interval_sec), + options.snapshot_dir.clone(), + options + .db_path + .file_name() + .map(|n| n.to_owned().into_string().expect("invalid path")) + .unwrap_or_else(|| String::from("data.ms")), + ); + + tokio::task::spawn(snapshot_service.run()); + } + + Ok(Self { + uuid_resolver, + index_handle, + update_handle, + dump_handle, + }) + } + + pub async fn add_documents( + &self, + uid: String, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + payload: Payload, + primary_key: Option, + ) -> Result { + let perform_update = |uuid| async move { + let meta = UpdateMeta::DocumentsAddition { + method, + format, + primary_key, + }; + let (sender, receiver) = mpsc::channel(10); + + // It is necessary to spawn a local task to send the payload to the update handle to + // prevent dead_locking between the update_handle::update that waits for the update to be + // registered and the update_actor that waits for the the payload to be sent to it. + tokio::task::spawn_local(async move { + payload + .for_each(|r| async { + let _ = sender.send(r).await; + }) + .await + }); + + // This must be done *AFTER* spawning the task. + self.update_handle.update(meta, receiver, uuid).await + }; + + match self.uuid_resolver.get(uid).await { + Ok(uuid) => Ok(perform_update(uuid).await?), + Err(UuidResolverError::UnexistingIndex(name)) => { + let uuid = Uuid::new_v4(); + let status = perform_update(uuid).await?; + // ignore if index creation fails now, since it may already have been created + let _ = self.index_handle.create_index(uuid, None).await; + self.uuid_resolver.insert(name, uuid).await?; + Ok(status) + } + Err(e) => Err(e.into()), + } + } + + pub async fn clear_documents(&self, uid: String) -> Result { + let uuid = self.uuid_resolver.get(uid).await?; + let meta = UpdateMeta::ClearDocuments; + let (_, receiver) = mpsc::channel(1); + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) + } + + pub async fn delete_documents( + &self, + uid: String, + documents: Vec, + ) -> Result { + let uuid = self.uuid_resolver.get(uid).await?; + let meta = UpdateMeta::DeleteDocuments { ids: documents }; + let (_, receiver) = mpsc::channel(1); + let status = self.update_handle.update(meta, receiver, uuid).await?; + Ok(status) + } + + pub async fn update_settings( + &self, + uid: String, + settings: Settings, + create: bool, + ) -> Result { + let perform_udpate = |uuid| async move { + let meta = UpdateMeta::Settings(settings.into_unchecked()); + // Nothing so send, drop the sender right away, as not to block the update actor. + let (_, receiver) = mpsc::channel(1); + self.update_handle.update(meta, receiver, uuid).await + }; + + match self.uuid_resolver.get(uid).await { + Ok(uuid) => Ok(perform_udpate(uuid).await?), + Err(UuidResolverError::UnexistingIndex(name)) if create => { + let uuid = Uuid::new_v4(); + let status = perform_udpate(uuid).await?; + // ignore if index creation fails now, since it may already have been created + let _ = self.index_handle.create_index(uuid, None).await; + self.uuid_resolver.insert(name, uuid).await?; + Ok(status) + } + Err(e) => Err(e.into()), + } + } + + pub async fn create_index(&self, index_settings: IndexSettings) -> Result { + let IndexSettings { uid, primary_key } = index_settings; + let uid = uid.ok_or(IndexControllerError::MissingUid)?; + let uuid = Uuid::new_v4(); + let meta = self.index_handle.create_index(uuid, primary_key).await?; + self.uuid_resolver.insert(uid.clone(), uuid).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + + Ok(meta) + } + + pub async fn delete_index(&self, uid: String) -> Result<()> { + let uuid = self.uuid_resolver.delete(uid).await?; + + // We remove the index from the resolver synchronously, and effectively perform the index + // deletion as a background task. + let update_handle = self.update_handle.clone(); + let index_handle = self.index_handle.clone(); + tokio::spawn(async move { + if let Err(e) = update_handle.delete(uuid).await { + error!("Error while deleting index: {}", e); + } + if let Err(e) = index_handle.delete(uuid).await { + error!("Error while deleting index: {}", e); + } + }); + + Ok(()) + } + + pub async fn update_status(&self, uid: String, id: u64) -> Result { + let uuid = self.uuid_resolver.get(uid).await?; + let result = self.update_handle.update_status(uuid, id).await?; + Ok(result) + } + + pub async fn all_update_status(&self, uid: String) -> Result> { + let uuid = self.uuid_resolver.get(uid).await?; + let result = self.update_handle.get_all_updates_status(uuid).await?; + Ok(result) + } + + pub async fn list_indexes(&self) -> Result> { + let uuids = self.uuid_resolver.list().await?; + + let mut ret = Vec::new(); + + for (uid, uuid) in uuids { + let meta = self.index_handle.get_index_meta(uuid).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + ret.push(meta); + } + + Ok(ret) + } + + pub async fn settings(&self, uid: String) -> Result> { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + let settings = self.index_handle.settings(uuid).await?; + Ok(settings) + } + + pub async fn documents( + &self, + uid: String, + offset: usize, + limit: usize, + attributes_to_retrieve: Option>, + ) -> Result> { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + let documents = self + .index_handle + .documents(uuid, offset, limit, attributes_to_retrieve) + .await?; + Ok(documents) + } + + pub async fn document( + &self, + uid: String, + doc_id: String, + attributes_to_retrieve: Option>, + ) -> Result { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + let document = self + .index_handle + .document(uuid, doc_id, attributes_to_retrieve) + .await?; + Ok(document) + } + + pub async fn update_index( + &self, + uid: String, + mut index_settings: IndexSettings, + ) -> Result { + if index_settings.uid.is_some() { + index_settings.uid.take(); + } + + let uuid = self.uuid_resolver.get(uid.clone()).await?; + let meta = self.index_handle.update_index(uuid, index_settings).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + Ok(meta) + } + + pub async fn search(&self, uid: String, query: SearchQuery) -> Result { + let uuid = self.uuid_resolver.get(uid).await?; + let result = self.index_handle.search(uuid, query).await?; + Ok(result) + } + + pub async fn get_index(&self, uid: String) -> Result { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + let meta = self.index_handle.get_index_meta(uuid).await?; + let meta = IndexMetadata { + uuid, + name: uid.clone(), + uid, + meta, + }; + Ok(meta) + } + + pub async fn get_uuids_size(&self) -> Result { + Ok(self.uuid_resolver.get_size().await?) + } + + pub async fn get_index_stats(&self, uid: String) -> Result { + let uuid = self.uuid_resolver.get(uid).await?; + let update_infos = self.update_handle.get_info().await?; + let mut stats = self.index_handle.get_index_stats(uuid).await?; + // Check if the currently indexing update is from out index. + stats.is_indexing = Some(Some(uuid) == update_infos.processing); + Ok(stats) + } + + pub async fn get_all_stats(&self) -> Result { + let update_infos = self.update_handle.get_info().await?; + let mut database_size = self.get_uuids_size().await? + update_infos.size; + let mut last_update: Option> = None; + let mut indexes = BTreeMap::new(); + + for index in self.list_indexes().await? { + let mut index_stats = self.index_handle.get_index_stats(index.uuid).await?; + database_size += index_stats.size; + + last_update = last_update.map_or(Some(index.meta.updated_at), |last| { + Some(last.max(index.meta.updated_at)) + }); + + index_stats.is_indexing = Some(Some(index.uuid) == update_infos.processing); + + indexes.insert(index.uid, index_stats); + } + + Ok(Stats { + database_size, + last_update, + indexes, + }) + } + + pub async fn create_dump(&self) -> Result { + Ok(self.dump_handle.create_dump().await?) + } + + pub async fn dump_info(&self, uid: String) -> Result { + Ok(self.dump_handle.dump_info(uid).await?) + } +} + +pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { + loop { + match Arc::try_unwrap(item) { + Ok(item) => return item, + Err(item_arc) => { + item = item_arc; + sleep(Duration::from_millis(100)).await; + continue; + } + } + } +} diff --git a/meilisearch-http/src/index_controller/snapshot.rs b/meilisearch-http/src/index_controller/snapshot.rs new file mode 100644 index 000000000..7bdedae76 --- /dev/null +++ b/meilisearch-http/src/index_controller/snapshot.rs @@ -0,0 +1,268 @@ +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::bail; +use log::{error, info, trace}; +use tokio::fs; +use tokio::task::spawn_blocking; +use tokio::time::sleep; + +use super::update_actor::UpdateActorHandle; +use super::uuid_resolver::UuidResolverHandle; +use crate::helpers::compression; + +pub struct SnapshotService { + uuid_resolver_handle: R, + update_handle: U, + snapshot_period: Duration, + snapshot_path: PathBuf, + db_name: String, +} + +impl SnapshotService +where + U: UpdateActorHandle, + R: UuidResolverHandle, +{ + pub fn new( + uuid_resolver_handle: R, + update_handle: U, + snapshot_period: Duration, + snapshot_path: PathBuf, + db_name: String, + ) -> Self { + Self { + uuid_resolver_handle, + update_handle, + snapshot_period, + snapshot_path, + db_name, + } + } + + pub async fn run(self) { + info!( + "Snapshot scheduled every {}s.", + self.snapshot_period.as_secs() + ); + loop { + if let Err(e) = self.perform_snapshot().await { + error!("Error while performing snapshot: {}", e); + } + sleep(self.snapshot_period).await; + } + } + + async fn perform_snapshot(&self) -> anyhow::Result<()> { + trace!("Performing snapshot."); + + let snapshot_dir = self.snapshot_path.clone(); + fs::create_dir_all(&snapshot_dir).await?; + let temp_snapshot_dir = + spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??; + let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); + + let uuids = self + .uuid_resolver_handle + .snapshot(temp_snapshot_path.clone()) + .await?; + + if uuids.is_empty() { + return Ok(()); + } + + self.update_handle + .snapshot(uuids, temp_snapshot_path.clone()) + .await?; + let snapshot_dir = self.snapshot_path.clone(); + let snapshot_path = self + .snapshot_path + .join(format!("{}.snapshot", self.db_name)); + let snapshot_path = spawn_blocking(move || -> anyhow::Result { + let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?; + let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); + compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?; + temp_snapshot_file.persist(&snapshot_path)?; + Ok(snapshot_path) + }) + .await??; + + trace!("Created snapshot in {:?}.", snapshot_path); + + Ok(()) + } +} + +pub fn load_snapshot( + db_path: impl AsRef, + snapshot_path: impl AsRef, + ignore_snapshot_if_db_exists: bool, + ignore_missing_snapshot: bool, +) -> anyhow::Result<()> { + if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() { + match compression::from_tar_gz(snapshot_path, &db_path) { + Ok(()) => Ok(()), + Err(e) => { + // clean created db folder + std::fs::remove_dir_all(&db_path)?; + Err(e) + } + } + } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists { + bail!( + "database already exists at {:?}, try to delete it or rename it", + db_path + .as_ref() + .canonicalize() + .unwrap_or_else(|_| db_path.as_ref().to_owned()) + ) + } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot { + bail!( + "snapshot doesn't exist at {:?}", + snapshot_path + .as_ref() + .canonicalize() + .unwrap_or_else(|_| snapshot_path.as_ref().to_owned()) + ) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod test { + use std::iter::FromIterator; + use std::{collections::HashSet, sync::Arc}; + + use futures::future::{err, ok}; + use rand::Rng; + use tokio::time::timeout; + use uuid::Uuid; + + use super::*; + use crate::index_controller::index_actor::MockIndexActorHandle; + use crate::index_controller::update_actor::{ + error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl, + }; + use crate::index_controller::uuid_resolver::{ + error::UuidResolverError, MockUuidResolverHandle, + }; + + #[actix_rt::test] + async fn test_normal() { + let mut rng = rand::thread_rng(); + let uuids_num: usize = rng.gen_range(5, 10); + let uuids = (0..uuids_num) + .map(|_| Uuid::new_v4()) + .collect::>(); + + let mut uuid_resolver = MockUuidResolverHandle::new(); + let uuids_clone = uuids.clone(); + uuid_resolver + .expect_snapshot() + .times(1) + .returning(move |_| Box::pin(ok(uuids_clone.clone()))); + + let uuids_clone = uuids.clone(); + let mut index_handle = MockIndexActorHandle::new(); + index_handle + .expect_snapshot() + .withf(move |uuid, _path| uuids_clone.contains(uuid)) + .times(uuids_num) + .returning(move |_, _| Box::pin(ok(()))); + + let dir = tempfile::tempdir_in(".").unwrap(); + let handle = Arc::new(index_handle); + let update_handle = + UpdateActorHandleImpl::>::new(handle.clone(), dir.path(), 4096 * 100).unwrap(); + + let snapshot_path = tempfile::tempdir_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + "data.ms".to_string(), + ); + + snapshot_service.perform_snapshot().await.unwrap(); + } + + #[actix_rt::test] + async fn error_performing_uuid_snapshot() { + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + .times(1) + // abitrary error + .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist))); + + let update_handle = MockUpdateActorHandle::new(); + + let snapshot_path = tempfile::tempdir_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + "data.ms".to_string(), + ); + + assert!(snapshot_service.perform_snapshot().await.is_err()); + // Nothing was written to the file + assert!(!snapshot_path.path().join("data.ms.snapshot").exists()); + } + + #[actix_rt::test] + async fn error_performing_index_snapshot() { + let uuid = Uuid::new_v4(); + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + .times(1) + .returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid))))); + + let mut update_handle = MockUpdateActorHandle::new(); + update_handle + .expect_snapshot() + // abitrary error + .returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0)))); + + let snapshot_path = tempfile::tempdir_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + "data.ms".to_string(), + ); + + assert!(snapshot_service.perform_snapshot().await.is_err()); + // Nothing was written to the file + assert!(!snapshot_path.path().join("data.ms.snapshot").exists()); + } + + #[actix_rt::test] + async fn test_loop() { + let mut uuid_resolver = MockUuidResolverHandle::new(); + uuid_resolver + .expect_snapshot() + // we expect the funtion to be called between 2 and 3 time in the given interval. + .times(2..4) + // abitrary error, to short-circuit the function + .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist))); + + let update_handle = MockUpdateActorHandle::new(); + + let snapshot_path = tempfile::tempdir_in(".").unwrap(); + let snapshot_service = SnapshotService::new( + uuid_resolver, + update_handle, + Duration::from_millis(100), + snapshot_path.path().to_owned(), + "data.ms".to_string(), + ); + + let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await; + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs new file mode 100644 index 000000000..8ba96dad1 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -0,0 +1,274 @@ +use std::collections::HashSet; +use std::io::SeekFrom; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +use async_stream::stream; +use futures::StreamExt; +use log::trace; +use oxidized_json_checker::JsonChecker; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use tokio::sync::mpsc; +use uuid::Uuid; + +use super::error::{Result, UpdateActorError}; +use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo}; +use crate::index_controller::index_actor::IndexActorHandle; +use crate::index_controller::{UpdateMeta, UpdateStatus}; + +pub struct UpdateActor { + path: PathBuf, + store: Arc, + inbox: Option>>, + index_handle: I, + must_exit: Arc, +} + +impl UpdateActor +where + D: AsRef<[u8]> + Sized + 'static, + I: IndexActorHandle + Clone + Send + Sync + 'static, +{ + pub fn new( + update_db_size: usize, + inbox: mpsc::Receiver>, + path: impl AsRef, + index_handle: I, + ) -> anyhow::Result { + let path = path.as_ref().join("updates"); + + std::fs::create_dir_all(&path)?; + + let mut options = heed::EnvOpenOptions::new(); + options.map_size(update_db_size); + + let must_exit = Arc::new(AtomicBool::new(false)); + + let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?; + std::fs::create_dir_all(path.join("update_files"))?; + let inbox = Some(inbox); + Ok(Self { + path, + store, + inbox, + index_handle, + must_exit, + }) + } + + pub async fn run(mut self) { + use UpdateMsg::*; + + trace!("Started update actor."); + + let mut inbox = self + .inbox + .take() + .expect("A receiver should be present by now."); + + let must_exit = self.must_exit.clone(); + let stream = stream! { + loop { + let msg = inbox.recv().await; + + if must_exit.load(std::sync::atomic::Ordering::Relaxed) { + break; + } + + match msg { + Some(msg) => yield msg, + None => break, + } + } + }; + + stream + .for_each_concurrent(Some(10), |msg| async { + match msg { + Update { + uuid, + meta, + data, + ret, + } => { + let _ = ret.send(self.handle_update(uuid, meta, data).await); + } + ListUpdates { uuid, ret } => { + let _ = ret.send(self.handle_list_updates(uuid).await); + } + GetUpdate { uuid, ret, id } => { + let _ = ret.send(self.handle_get_update(uuid, id).await); + } + Delete { uuid, ret } => { + let _ = ret.send(self.handle_delete(uuid).await); + } + Snapshot { uuids, path, ret } => { + let _ = ret.send(self.handle_snapshot(uuids, path).await); + } + GetInfo { ret } => { + let _ = ret.send(self.handle_get_info().await); + } + Dump { uuids, path, ret } => { + let _ = ret.send(self.handle_dump(uuids, path).await); + } + } + }) + .await; + } + + async fn handle_update( + &self, + uuid: Uuid, + meta: UpdateMeta, + payload: mpsc::Receiver>, + ) -> Result { + let file_path = match meta { + UpdateMeta::DocumentsAddition { .. } => { + let update_file_id = uuid::Uuid::new_v4(); + let path = self + .path + .join(format!("update_files/update_{}", update_file_id)); + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&path) + .await?; + + async fn write_to_file( + file: &mut fs::File, + mut payload: mpsc::Receiver>, + ) -> Result + where + D: AsRef<[u8]> + Sized + 'static, + { + let mut file_len = 0; + + while let Some(bytes) = payload.recv().await { + let bytes = bytes?; + file_len += bytes.as_ref().len(); + file.write_all(bytes.as_ref()).await?; + } + + file.flush().await?; + + Ok(file_len) + } + + let file_len = write_to_file(&mut file, payload).await; + + match file_len { + Ok(len) if len > 0 => { + let file = file.into_std().await; + Some((file, update_file_id)) + } + Err(e) => { + fs::remove_file(&path).await?; + return Err(e); + } + _ => { + fs::remove_file(&path).await?; + None + } + } + } + _ => None, + }; + + let update_store = self.store.clone(); + + tokio::task::spawn_blocking(move || { + use std::io::{copy, sink, BufReader, Seek}; + + // If the payload is empty, ignore the check. + let update_uuid = if let Some((mut file, uuid)) = file_path { + // set the file back to the beginning + file.seek(SeekFrom::Start(0))?; + // Check that the json payload is valid: + let reader = BufReader::new(&mut file); + let mut checker = JsonChecker::new(reader); + + if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { + // The json file is invalid, we use Serde to get a nice error message: + file.seek(SeekFrom::Start(0))?; + let _: serde_json::Value = serde_json::from_reader(file) + .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; + } + Some(uuid) + } else { + None + }; + + // The payload is valid, we can register it to the update store. + let status = update_store + .register_update(meta, update_uuid, uuid) + .map(UpdateStatus::Enqueued)?; + Ok(status) + }) + .await? + } + + async fn handle_list_updates(&self, uuid: Uuid) -> Result> { + let update_store = self.store.clone(); + tokio::task::spawn_blocking(move || { + let result = update_store.list(uuid)?; + Ok(result) + }) + .await? + } + + async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result { + let store = self.store.clone(); + tokio::task::spawn_blocking(move || { + let result = store + .meta(uuid, id)? + .ok_or(UpdateActorError::UnexistingUpdate(id))?; + Ok(result) + }) + .await? + } + + async fn handle_delete(&self, uuid: Uuid) -> Result<()> { + let store = self.store.clone(); + + tokio::task::spawn_blocking(move || store.delete_all(uuid)).await??; + + Ok(()) + } + + async fn handle_snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + let update_store = self.store.clone(); + + tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle)) + .await??; + + Ok(()) + } + + async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + let index_handle = self.index_handle.clone(); + let update_store = self.store.clone(); + + tokio::task::spawn_blocking(move || -> Result<()> { + update_store.dump(&uuids, path.to_path_buf(), index_handle)?; + Ok(()) + }) + .await??; + + Ok(()) + } + + async fn handle_get_info(&self) -> Result { + let update_store = self.store.clone(); + let info = tokio::task::spawn_blocking(move || -> Result { + let info = update_store.get_info()?; + Ok(info) + }) + .await??; + + Ok(info) + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/error.rs b/meilisearch-http/src/index_controller/update_actor/error.rs new file mode 100644 index 000000000..29c1802a8 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/error.rs @@ -0,0 +1,61 @@ +use std::error::Error; + +use meilisearch_error::{Code, ErrorCode}; + +use crate::index_controller::index_actor::error::IndexActorError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +#[allow(clippy::large_enum_variant)] +pub enum UpdateActorError { + #[error("Update {0} not found.")] + UnexistingUpdate(u64), + #[error("Internal error: {0}")] + Internal(Box), + #[error("{0}")] + IndexActor(#[from] IndexActorError), + #[error( + "update store was shut down due to a fatal error, please check your logs for more info." + )] + FatalUpdateStoreError, + #[error("{0}")] + InvalidPayload(Box), + #[error("{0}")] + PayloadError(#[from] actix_web::error::PayloadError), +} + +impl From> for UpdateActorError { + fn from(_: tokio::sync::mpsc::error::SendError) -> Self { + Self::FatalUpdateStoreError + } +} + +impl From for UpdateActorError { + fn from(_: tokio::sync::oneshot::error::RecvError) -> Self { + Self::FatalUpdateStoreError + } +} + +internal_error!( + UpdateActorError: heed::Error, + std::io::Error, + serde_json::Error, + tokio::task::JoinError +); + +impl ErrorCode for UpdateActorError { + fn error_code(&self) -> Code { + match self { + UpdateActorError::UnexistingUpdate(_) => Code::NotFound, + UpdateActorError::Internal(_) => Code::Internal, + UpdateActorError::IndexActor(e) => e.error_code(), + UpdateActorError::FatalUpdateStoreError => Code::Internal, + UpdateActorError::InvalidPayload(_) => Code::BadRequest, + UpdateActorError::PayloadError(error) => match error { + actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge, + _ => Code::Internal, + }, + } + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs new file mode 100644 index 000000000..125c63401 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -0,0 +1,103 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use crate::index_controller::{IndexActorHandle, UpdateStatus}; + +use super::error::Result; +use super::{PayloadData, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo}; + +#[derive(Clone)] +pub struct UpdateActorHandleImpl { + sender: mpsc::Sender>, +} + +impl UpdateActorHandleImpl +where + D: AsRef<[u8]> + Sized + 'static + Sync + Send, +{ + pub fn new( + index_handle: I, + path: impl AsRef, + update_store_size: usize, + ) -> anyhow::Result + where + I: IndexActorHandle + Clone + Send + Sync + 'static, + { + let path = path.as_ref().to_owned(); + let (sender, receiver) = mpsc::channel(100); + let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?; + + tokio::task::spawn(actor.run()); + + Ok(Self { sender }) + } +} + +#[async_trait::async_trait] +impl UpdateActorHandle for UpdateActorHandleImpl +where + D: AsRef<[u8]> + Sized + 'static + Sync + Send, +{ + type Data = D; + + async fn get_all_updates_status(&self, uuid: Uuid) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::ListUpdates { uuid, ret }; + self.sender.send(msg).await?; + receiver.await? + } + async fn update_status(&self, uuid: Uuid, id: u64) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetUpdate { uuid, id, ret }; + self.sender.send(msg).await?; + receiver.await? + } + + async fn delete(&self, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Delete { uuid, ret }; + self.sender.send(msg).await?; + receiver.await? + } + + async fn snapshot(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Snapshot { uuids, path, ret }; + self.sender.send(msg).await?; + receiver.await? + } + + async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Dump { uuids, path, ret }; + self.sender.send(msg).await?; + receiver.await? + } + + async fn get_info(&self) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetInfo { ret }; + self.sender.send(msg).await?; + receiver.await? + } + + async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::Update { + uuid, + data, + meta, + ret, + }; + self.sender.send(msg).await?; + receiver.await? + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs new file mode 100644 index 000000000..6b8a0f73f --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -0,0 +1,43 @@ +use std::collections::HashSet; +use std::path::PathBuf; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use super::error::Result; +use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo}; + +pub enum UpdateMsg { + Update { + uuid: Uuid, + meta: UpdateMeta, + data: mpsc::Receiver>, + ret: oneshot::Sender>, + }, + ListUpdates { + uuid: Uuid, + ret: oneshot::Sender>>, + }, + GetUpdate { + uuid: Uuid, + ret: oneshot::Sender>, + id: u64, + }, + Delete { + uuid: Uuid, + ret: oneshot::Sender>, + }, + Snapshot { + uuids: HashSet, + path: PathBuf, + ret: oneshot::Sender>, + }, + Dump { + uuids: HashSet, + path: PathBuf, + ret: oneshot::Sender>, + }, + GetInfo { + ret: oneshot::Sender>, + }, +} diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs new file mode 100644 index 000000000..cb40311b2 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -0,0 +1,44 @@ +use std::{collections::HashSet, path::PathBuf}; + +use actix_http::error::PayloadError; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::index_controller::{UpdateMeta, UpdateStatus}; + +use actor::UpdateActor; +use error::Result; +use message::UpdateMsg; + +pub use handle_impl::UpdateActorHandleImpl; +pub use store::{UpdateStore, UpdateStoreInfo}; + +mod actor; +pub mod error; +mod handle_impl; +mod message; +pub mod store; + +type PayloadData = std::result::Result; + +#[cfg(test)] +use mockall::automock; + +#[async_trait::async_trait] +#[cfg_attr(test, automock(type Data=Vec;))] +pub trait UpdateActorHandle { + type Data: AsRef<[u8]> + Sized + 'static + Sync + Send; + + async fn get_all_updates_status(&self, uuid: Uuid) -> Result>; + async fn update_status(&self, uuid: Uuid, id: u64) -> Result; + async fn delete(&self, uuid: Uuid) -> Result<()>; + async fn snapshot(&self, uuid: HashSet, path: PathBuf) -> Result<()>; + async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()>; + async fn get_info(&self) -> Result; + async fn update( + &self, + meta: UpdateMeta, + data: mpsc::Receiver>, + uuid: Uuid, + ) -> Result; +} diff --git a/meilisearch-http/src/index_controller/update_actor/store/codec.rs b/meilisearch-http/src/index_controller/update_actor/store/codec.rs new file mode 100644 index 000000000..e07b52eec --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store/codec.rs @@ -0,0 +1,86 @@ +use std::{borrow::Cow, convert::TryInto, mem::size_of}; + +use heed::{BytesDecode, BytesEncode}; +use uuid::Uuid; + +pub struct NextIdCodec; + +pub enum NextIdKey { + Global, + Index(Uuid), +} + +impl<'a> BytesEncode<'a> for NextIdCodec { + type EItem = NextIdKey; + + fn bytes_encode(item: &'a Self::EItem) -> Option> { + match item { + NextIdKey::Global => Some(Cow::Borrowed(b"__global__")), + NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())), + } + } +} + +pub struct PendingKeyCodec; + +impl<'a> BytesEncode<'a> for PendingKeyCodec { + type EItem = (u64, Uuid, u64); + + fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(&global_id.to_be_bytes()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for PendingKeyCodec { + type DItem = (u64, Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let global_id_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let global_id = u64::from_be_bytes(global_id_bytes); + + let uuid_bytes = bytes + .get(size_of::()..(size_of::() + size_of::()))? + .try_into() + .ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes + .get((size_of::() + size_of::())..)? + .try_into() + .ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((global_id, uuid, update_id)) + } +} + +pub struct UpdateKeyCodec; + +impl<'a> BytesEncode<'a> for UpdateKeyCodec { + type EItem = (Uuid, u64); + + fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(size_of::()); + bytes.extend_from_slice(uuid.as_bytes()); + bytes.extend_from_slice(&update_id.to_be_bytes()); + Some(Cow::Owned(bytes)) + } +} + +impl<'a> BytesDecode<'a> for UpdateKeyCodec { + type DItem = (Uuid, u64); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let uuid_bytes = bytes.get(0..size_of::())?.try_into().ok()?; + let uuid = Uuid::from_bytes(uuid_bytes); + + let update_id_bytes = bytes.get(size_of::()..)?.try_into().ok()?; + let update_id = u64::from_be_bytes(update_id_bytes); + + Some((uuid, update_id)) + } +} diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs new file mode 100644 index 000000000..7c46f98fa --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -0,0 +1,184 @@ +use std::{ + collections::HashSet, + fs::{create_dir_all, File}, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, +}; + +use heed::{EnvOpenOptions, RoTxn}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{Result, State, UpdateStore}; +use crate::index_controller::{ + index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued, + UpdateStatus, +}; + +#[derive(Serialize, Deserialize)] +struct UpdateEntry { + uuid: Uuid, + update: UpdateStatus, +} + +impl UpdateStore { + pub fn dump( + &self, + uuids: &HashSet, + path: PathBuf, + handle: impl IndexActorHandle, + ) -> Result<()> { + let state_lock = self.state.write(); + state_lock.swap(State::Dumping); + + // txn must *always* be acquired after state lock, or it will dead lock. + let txn = self.env.write_txn()?; + + let dump_path = path.join("updates"); + create_dir_all(&dump_path)?; + + self.dump_updates(&txn, uuids, &dump_path)?; + + let fut = dump_indexes(uuids, handle, &path); + tokio::runtime::Handle::current().block_on(fut)?; + + state_lock.swap(State::Idle); + + Ok(()) + } + + fn dump_updates( + &self, + txn: &RoTxn, + uuids: &HashSet, + path: impl AsRef, + ) -> Result<()> { + let dump_data_path = path.as_ref().join("data.jsonl"); + let mut dump_data_file = File::create(dump_data_path)?; + + let update_files_path = path.as_ref().join(super::UPDATE_DIR); + create_dir_all(&update_files_path)?; + + self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?; + self.dump_completed(&txn, uuids, &mut dump_data_file)?; + + Ok(()) + } + + fn dump_pending( + &self, + txn: &RoTxn, + uuids: &HashSet, + mut file: &mut File, + dst_path: impl AsRef, + ) -> Result<()> { + let pendings = self.pending_queue.iter(txn)?.lazily_decode_data(); + + for pending in pendings { + let ((_, uuid, _), data) = pending?; + if uuids.contains(&uuid) { + let update = data.decode()?; + + if let Some(ref update_uuid) = update.content { + let src = super::update_uuid_to_file_path(&self.path, *update_uuid); + let dst = super::update_uuid_to_file_path(&dst_path, *update_uuid); + std::fs::copy(src, dst)?; + } + + let update_json = UpdateEntry { + uuid, + update: update.into(), + }; + + serde_json::to_writer(&mut file, &update_json)?; + file.write_all(b"\n")?; + } + } + + Ok(()) + } + + fn dump_completed( + &self, + txn: &RoTxn, + uuids: &HashSet, + mut file: &mut File, + ) -> Result<()> { + let updates = self.updates.iter(txn)?.lazily_decode_data(); + + for update in updates { + let ((uuid, _), data) = update?; + if uuids.contains(&uuid) { + let update = data.decode()?; + + let update_json = UpdateEntry { uuid, update }; + + serde_json::to_writer(&mut file, &update_json)?; + file.write_all(b"\n")?; + } + } + + Ok(()) + } + + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + db_size: usize, + ) -> anyhow::Result<()> { + let dst_update_path = dst.as_ref().join("updates/"); + create_dir_all(&dst_update_path)?; + + let mut options = EnvOpenOptions::new(); + options.map_size(db_size as usize); + let (store, _) = UpdateStore::new(options, &dst_update_path)?; + + let src_update_path = src.as_ref().join("updates"); + let update_data = File::open(&src_update_path.join("data.jsonl"))?; + let mut update_data = BufReader::new(update_data); + + std::fs::create_dir_all(dst_update_path.join("update_files/"))?; + + let mut wtxn = store.env.write_txn()?; + let mut line = String::new(); + loop { + match update_data.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let UpdateEntry { uuid, update } = serde_json::from_str(&line)?; + store.register_raw_updates(&mut wtxn, &update, uuid)?; + + // Copy ascociated update path if it exists + if let UpdateStatus::Enqueued(Enqueued { + content: Some(uuid), + .. + }) = update + { + let src = update_uuid_to_file_path(&src_update_path, uuid); + let dst = update_uuid_to_file_path(&dst_update_path, uuid); + std::fs::copy(src, dst)?; + } + } + _ => break, + } + + line.clear(); + } + + wtxn.commit()?; + + Ok(()) + } +} + +async fn dump_indexes( + uuids: &HashSet, + handle: impl IndexActorHandle, + path: impl AsRef, +) -> Result<()> { + for uuid in uuids { + handle.dump(*uuid, path.as_ref().to_owned()).await?; + } + + Ok(()) +} diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs new file mode 100644 index 000000000..cf5b846c6 --- /dev/null +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -0,0 +1,721 @@ +mod codec; +pub mod dump; + +use std::fs::{copy, create_dir_all, remove_file, File}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::{ + collections::{BTreeMap, HashSet}, + path::PathBuf, +}; + +use arc_swap::ArcSwap; +use futures::StreamExt; +use heed::types::{ByteSlice, OwnedType, SerdeJson}; +use heed::zerocopy::U64; +use heed::{CompactionOption, Database, Env, EnvOpenOptions}; +use log::error; +use parking_lot::{Mutex, MutexGuard}; +use tokio::runtime::Handle; +use tokio::sync::mpsc; +use uuid::Uuid; + +use codec::*; + +use super::error::Result; +use super::UpdateMeta; +use crate::helpers::EnvSizer; +use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; + +#[allow(clippy::upper_case_acronyms)] +type BEU64 = U64; + +const UPDATE_DIR: &str = "update_files"; + +pub struct UpdateStoreInfo { + /// Size of the update store in bytes. + pub size: u64, + /// Uuid of the currently processing update if it exists + pub processing: Option, +} + +/// A data structure that allows concurrent reads AND exactly one writer. +pub struct StateLock { + lock: Mutex<()>, + data: ArcSwap, +} + +pub struct StateLockGuard<'a> { + _lock: MutexGuard<'a, ()>, + state: &'a StateLock, +} + +impl StateLockGuard<'_> { + pub fn swap(&self, state: State) -> Arc { + self.state.data.swap(Arc::new(state)) + } +} + +impl StateLock { + fn from_state(state: State) -> Self { + let lock = Mutex::new(()); + let data = ArcSwap::from(Arc::new(state)); + Self { lock, data } + } + + pub fn read(&self) -> Arc { + self.data.load().clone() + } + + pub fn write(&self) -> StateLockGuard { + let _lock = self.lock.lock(); + let state = &self; + StateLockGuard { _lock, state } + } +} + +#[allow(clippy::large_enum_variant)] +pub enum State { + Idle, + Processing(Uuid, Processing), + Snapshoting, + Dumping, +} + +#[derive(Clone)] +pub struct UpdateStore { + pub env: Env, + /// A queue containing the updates to process, ordered by arrival. + /// The key are built as follow: + /// | global_update_id | index_uuid | update_id | + /// | 8-bytes | 16-bytes | 8-bytes | + pending_queue: Database>, + /// Map indexes to the next available update id. If NextIdKey::Global is queried, then the next + /// global update id is returned + next_update_id: Database>, + /// Contains all the performed updates meta, be they failed, aborted, or processed. + /// The keys are built as follow: + /// | Uuid | id | + /// | 16-bytes | 8-bytes | + updates: Database>, + /// Indicates the current state of the update store, + state: Arc, + /// Wake up the loop when a new event occurs. + notification_sender: mpsc::Sender<()>, + path: PathBuf, +} + +impl UpdateStore { + fn new( + mut options: EnvOpenOptions, + path: impl AsRef, + ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> { + options.max_dbs(5); + + let env = options.open(&path)?; + let pending_queue = env.create_database(Some("pending-queue"))?; + let next_update_id = env.create_database(Some("next-update-id"))?; + let updates = env.create_database(Some("updates"))?; + + let state = Arc::new(StateLock::from_state(State::Idle)); + + let (notification_sender, notification_receiver) = mpsc::channel(10); + + Ok(( + Self { + env, + pending_queue, + next_update_id, + updates, + state, + notification_sender, + path: path.as_ref().to_owned(), + }, + notification_receiver, + )) + } + + pub fn open( + options: EnvOpenOptions, + path: impl AsRef, + index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static, + must_exit: Arc, + ) -> anyhow::Result> { + let (update_store, mut notification_receiver) = Self::new(options, path)?; + let update_store = Arc::new(update_store); + + // Send a first notification to trigger the process. + let _ = update_store.notification_sender.send(()); + + // Init update loop to perform any pending updates at launch. + // Since we just launched the update store, and we still own the receiving end of the + // channel, this call is guaranteed to succeed. + update_store + .notification_sender + .try_send(()) + .expect("Failed to init update store"); + + // We need a weak reference so we can take ownership on the arc later when we + // want to close the index. + let update_store_weak = Arc::downgrade(&update_store); + tokio::task::spawn(async move { + // Block and wait for something to process. + 'outer: while notification_receiver.recv().await.is_some() { + loop { + match update_store_weak.upgrade() { + Some(update_store) => { + let handler = index_handle.clone(); + let res = tokio::task::spawn_blocking(move || { + update_store.process_pending_update(handler) + }) + .await + .expect("Fatal error processing update."); + match res { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => { + error!("Fatal error while processing an update that requires the update store to shutdown: {}", e); + must_exit.store(true, Ordering::SeqCst); + break 'outer; + } + } + } + // the ownership on the arc has been taken, we need to exit. + None => break 'outer, + } + } + } + + error!("Update store loop exited."); + }); + + Ok(update_store) + } + + /// Returns the next global update id and the next update id for a given `index_uuid`. + fn next_update_id(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<(u64, u64)> { + let global_id = self + .next_update_id + .get(txn, &NextIdKey::Global)? + .map(U64::get) + .unwrap_or_default(); + + self.next_update_id + .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?; + + let update_id = self.next_update_id_raw(txn, index_uuid)?; + + Ok((global_id, update_id)) + } + + /// Returns the next next update id for a given `index_uuid` without + /// incrementing the global update id. This is useful for the dumps. + fn next_update_id_raw(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result { + let update_id = self + .next_update_id + .get(txn, &NextIdKey::Index(index_uuid))? + .map(U64::get) + .unwrap_or_default(); + + self.next_update_id.put( + txn, + &NextIdKey::Index(index_uuid), + &BEU64::new(update_id + 1), + )?; + + Ok(update_id) + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: UpdateMeta, + content: Option, + index_uuid: Uuid, + ) -> heed::Result { + let mut txn = self.env.write_txn()?; + + let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; + let meta = Enqueued::new(meta, update_id, content); + + self.pending_queue + .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?; + + txn.commit()?; + + self.notification_sender + .blocking_send(()) + .expect("Update store loop exited."); + Ok(meta) + } + + /// Push already processed update in the UpdateStore without triggering the notification + /// process. This is useful for the dumps. + pub fn register_raw_updates( + &self, + wtxn: &mut heed::RwTxn, + update: &UpdateStatus, + index_uuid: Uuid, + ) -> heed::Result<()> { + match update { + UpdateStatus::Enqueued(enqueued) => { + let (global_id, _update_id) = self.next_update_id(wtxn, index_uuid)?; + self.pending_queue.remap_key_type::().put( + wtxn, + &(global_id, index_uuid, enqueued.id()), + &enqueued, + )?; + } + _ => { + let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; + self.updates + .put(wtxn, &(index_uuid, update.id()), &update)?; + } + } + Ok(()) + } + + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result> { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_queue.first(&rtxn)?; + drop(rtxn); + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some(((global_id, index_uuid, _), mut pending)) => { + let content = pending.content.take(); + let processing = pending.processing(); + // Acquire the state lock and set the current state to processing. + // txn must *always* be acquired after state lock, or it will dead lock. + let state = self.state.write(); + state.swap(State::Processing(index_uuid, processing.clone())); + + let result = + self.perform_update(content, processing, index_handle, index_uuid, global_id); + + state.swap(State::Idle); + + result + } + None => Ok(None), + } + } + + fn perform_update( + &self, + content: Option, + processing: Processing, + index_handle: impl IndexActorHandle, + index_uuid: Uuid, + global_id: u64, + ) -> Result> { + let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid)); + let update_id = processing.id(); + + let file = match content_path { + Some(ref path) => { + let file = File::open(path)?; + Some(file) + } + None => None, + }; + + // Process the pending update using the provided user function. + let handle = Handle::current(); + let result = + match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) { + Ok(result) => result, + Err(e) => Err(processing.fail(e.into())), + }; + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.pending_queue + .delete(&mut wtxn, &(global_id, index_uuid, update_id))?; + + let result = match result { + Ok(res) => res.into(), + Err(res) => res.into(), + }; + + self.updates + .put(&mut wtxn, &(index_uuid, update_id), &result)?; + + wtxn.commit()?; + + if let Some(ref path) = content_path { + remove_file(&path)?; + } + + Ok(Some(())) + } + + /// List the updates for `index_uuid`. + pub fn list(&self, index_uuid: Uuid) -> Result> { + let mut update_list = BTreeMap::::new(); + + let txn = self.env.read_txn()?; + + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + for entry in pendings { + let ((_, uuid, id), pending) = entry?; + if uuid == index_uuid { + update_list.insert(id, pending.decode()?.into()); + } + } + + let updates = self + .updates + .remap_key_type::() + .prefix_iter(&txn, index_uuid.as_bytes())?; + + for entry in updates { + let (_, update) = entry?; + update_list.insert(update.id(), update); + } + + // If the currently processing update is from this index, replace the corresponding pending update with this one. + match *self.state.read() { + State::Processing(uuid, ref processing) if uuid == index_uuid => { + update_list.insert(processing.id(), processing.clone().into()); + } + _ => (), + } + + Ok(update_list.into_iter().map(|(_, v)| v).collect()) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, index_uuid: Uuid, update_id: u64) -> heed::Result> { + // Check if the update is the one currently processing + match *self.state.read() { + State::Processing(uuid, ref processing) + if uuid == index_uuid && processing.id() == update_id => + { + return Ok(Some(processing.clone().into())); + } + _ => (), + } + + let txn = self.env.read_txn()?; + // Else, check if it is in the updates database: + let update = self.updates.get(&txn, &(index_uuid, update_id))?; + + if let Some(update) = update { + return Ok(Some(update)); + } + + // If nothing was found yet, we resolve to iterate over the pending queue. + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + + for entry in pendings { + let ((_, uuid, id), pending) = entry?; + if uuid == index_uuid && id == update_id { + return Ok(Some(pending.decode()?.into())); + } + } + + // No update was found. + Ok(None) + } + + /// Delete all updates for an index from the update store. If the currently processing update + /// is for `index_uuid`, the call will block until the update is terminated. + pub fn delete_all(&self, index_uuid: Uuid) -> Result<()> { + let mut txn = self.env.write_txn()?; + // Contains all the content file paths that we need to be removed if the deletion was successful. + let mut uuids_to_remove = Vec::new(); + + let mut pendings = self.pending_queue.iter_mut(&mut txn)?.lazily_decode_data(); + + while let Some(Ok(((_, uuid, _), pending))) = pendings.next() { + if uuid == index_uuid { + unsafe { + pendings.del_current()?; + } + let mut pending = pending.decode()?; + if let Some(update_uuid) = pending.content.take() { + uuids_to_remove.push(update_uuid); + } + } + } + + drop(pendings); + + let mut updates = self + .updates + .remap_key_type::() + .prefix_iter_mut(&mut txn, index_uuid.as_bytes())? + .lazily_decode_data(); + + while let Some(_) = updates.next() { + unsafe { + updates.del_current()?; + } + } + + drop(updates); + + txn.commit()?; + + uuids_to_remove + .iter() + .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) + .for_each(|path| { + let _ = remove_file(path); + }); + + // If the currently processing update is from our index, we wait until it is + // finished before returning. This ensure that no write to the index occurs after we delete it. + if let State::Processing(uuid, _) = *self.state.read() { + if uuid == index_uuid { + // wait for a write lock, do nothing with it. + self.state.write(); + } + } + + Ok(()) + } + + pub fn snapshot( + &self, + uuids: &HashSet, + path: impl AsRef, + handle: impl IndexActorHandle + Clone, + ) -> Result<()> { + let state_lock = self.state.write(); + state_lock.swap(State::Snapshoting); + + let txn = self.env.write_txn()?; + + let update_path = path.as_ref().join("updates"); + create_dir_all(&update_path)?; + + // acquire write lock to prevent further writes during snapshot + create_dir_all(&update_path)?; + let db_path = update_path.join("data.mdb"); + + // create db snapshot + self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; + + let update_files_path = update_path.join(UPDATE_DIR); + create_dir_all(&update_files_path)?; + + let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); + + for entry in pendings { + let ((_, uuid, _), pending) = entry?; + if uuids.contains(&uuid) { + if let Enqueued { + content: Some(uuid), + .. + } = pending.decode()? + { + let path = update_uuid_to_file_path(&self.path, uuid); + copy(path, &update_files_path)?; + } + } + } + + let path = &path.as_ref().to_path_buf(); + let handle = &handle; + // Perform the snapshot of each index concurently. Only a third of the capabilities of + // the index actor at a time not to put too much pressure on the index actor + let mut stream = futures::stream::iter(uuids.iter()) + .map(move |uuid| handle.snapshot(*uuid, path.clone())) + .buffer_unordered(CONCURRENT_INDEX_MSG / 3); + + Handle::current().block_on(async { + while let Some(res) = stream.next().await { + res?; + } + Ok(()) as Result<()> + })?; + + Ok(()) + } + + pub fn get_info(&self) -> Result { + let mut size = self.env.size(); + let txn = self.env.read_txn()?; + for entry in self.pending_queue.iter(&txn)? { + let (_, pending) = entry?; + if let Enqueued { + content: Some(uuid), + .. + } = pending + { + let path = update_uuid_to_file_path(&self.path, uuid); + size += File::open(path)?.metadata()?.len(); + } + } + let processing = match *self.state.read() { + State::Processing(uuid, _) => Some(uuid), + _ => None, + }; + + Ok(UpdateStoreInfo { size, processing }) + } +} + +fn update_uuid_to_file_path(root: impl AsRef, uuid: Uuid) -> PathBuf { + root.as_ref() + .join(UPDATE_DIR) + .join(format!("update_{}", uuid)) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::index_controller::{ + index_actor::{error::IndexActorError, MockIndexActorHandle}, + UpdateResult, + }; + + use futures::future::ok; + + #[actix_rt::test] + async fn test_next_id() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut options = EnvOpenOptions::new(); + let handle = Arc::new(MockIndexActorHandle::new()); + options.map_size(4096 * 100); + let update_store = UpdateStore::open( + options, + dir.path(), + handle, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + + let index1_uuid = Uuid::new_v4(); + let index2_uuid = Uuid::new_v4(); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((0, 0), ids); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index2_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((1, 0), ids); + + let mut txn = update_store.env.write_txn().unwrap(); + let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap(); + txn.commit().unwrap(); + assert_eq!((2, 1), ids); + } + + #[actix_rt::test] + async fn test_register_update() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut options = EnvOpenOptions::new(); + let handle = Arc::new(MockIndexActorHandle::new()); + options.map_size(4096 * 100); + let update_store = UpdateStore::open( + options, + dir.path(), + handle, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + let meta = UpdateMeta::ClearDocuments; + let uuid = Uuid::new_v4(); + let store_clone = update_store.clone(); + tokio::task::spawn_blocking(move || { + store_clone.register_update(meta, None, uuid).unwrap(); + }) + .await + .unwrap(); + + let txn = update_store.env.read_txn().unwrap(); + assert!(update_store + .pending_queue + .get(&txn, &(0, uuid, 0)) + .unwrap() + .is_some()); + } + + #[actix_rt::test] + async fn test_process_update() { + let dir = tempfile::tempdir_in(".").unwrap(); + let mut handle = MockIndexActorHandle::new(); + + handle + .expect_update() + .times(2) + .returning(|_index_uuid, processing, _file| { + if processing.id() == 0 { + Box::pin(ok(Ok(processing.process(UpdateResult::Other)))) + } else { + Box::pin(ok(Err( + processing.fail(IndexActorError::ExistingPrimaryKey.into()) + ))) + } + }); + + let handle = Arc::new(handle); + + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let store = UpdateStore::open( + options, + dir.path(), + handle.clone(), + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + + // wait a bit for the event loop exit. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let mut txn = store.env.write_txn().unwrap(); + + let update = Enqueued::new(UpdateMeta::ClearDocuments, 0, None); + let uuid = Uuid::new_v4(); + + store + .pending_queue + .put(&mut txn, &(0, uuid, 0), &update) + .unwrap(); + + let update = Enqueued::new(UpdateMeta::ClearDocuments, 1, None); + + store + .pending_queue + .put(&mut txn, &(1, uuid, 1), &update) + .unwrap(); + + txn.commit().unwrap(); + + // Process the pending, and check that it has been moved to the update databases, and + // removed from the pending database. + let store_clone = store.clone(); + tokio::task::spawn_blocking(move || { + store_clone.process_pending_update(handle.clone()).unwrap(); + store_clone.process_pending_update(handle).unwrap(); + }) + .await + .unwrap(); + + let txn = store.env.read_txn().unwrap(); + + assert!(store.pending_queue.first(&txn).unwrap().is_none()); + let update = store.updates.get(&txn, &(uuid, 0)).unwrap().unwrap(); + + assert!(matches!(update, UpdateStatus::Processed(_))); + let update = store.updates.get(&txn, &(uuid, 1)).unwrap().unwrap(); + + assert!(matches!(update, UpdateStatus::Failed(_))); + } +} diff --git a/meilisearch-http/src/index_controller/updates.rs b/meilisearch-http/src/index_controller/updates.rs new file mode 100644 index 000000000..d02438d3c --- /dev/null +++ b/meilisearch-http/src/index_controller/updates.rs @@ -0,0 +1,233 @@ +use chrono::{DateTime, Utc}; +use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{ + error::ResponseError, + index::{Settings, Unchecked}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: u64 }, + Other, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum UpdateMeta { + DocumentsAddition { + method: IndexDocumentsMethod, + format: UpdateFormat, + primary_key: Option, + }, + ClearDocuments, + DeleteDocuments { + ids: Vec, + }, + Settings(Settings), +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Enqueued { + pub update_id: u64, + pub meta: UpdateMeta, + pub enqueued_at: DateTime, + pub content: Option, +} + +impl Enqueued { + pub fn new(meta: UpdateMeta, update_id: u64, content: Option) -> Self { + Self { + enqueued_at: Utc::now(), + meta, + update_id, + content, + } + } + + pub fn processing(self) -> Processing { + Processing { + from: self, + started_processing_at: Utc::now(), + } + } + + pub fn abort(self) -> Aborted { + Aborted { + from: self, + aborted_at: Utc::now(), + } + } + + pub fn meta(&self) -> &UpdateMeta { + &self.meta + } + + pub fn id(&self) -> u64 { + self.update_id + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Processed { + pub success: UpdateResult, + pub processed_at: DateTime, + #[serde(flatten)] + pub from: Processing, +} + +impl Processed { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Processing { + #[serde(flatten)] + pub from: Enqueued, + pub started_processing_at: DateTime, +} + +impl Processing { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } + + pub fn process(self, success: UpdateResult) -> Processed { + Processed { + success, + from: self, + processed_at: Utc::now(), + } + } + + pub fn fail(self, error: ResponseError) -> Failed { + Failed { + from: self, + error, + failed_at: Utc::now(), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Aborted { + #[serde(flatten)] + from: Enqueued, + aborted_at: DateTime, +} + +impl Aborted { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Failed { + #[serde(flatten)] + pub from: Processing, + pub error: ResponseError, + pub failed_at: DateTime, +} + +impl Failed { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &UpdateMeta { + self.from.meta() + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "camelCase")] +pub enum UpdateStatus { + Processing(Processing), + Enqueued(Enqueued), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), +} + +impl UpdateStatus { + pub fn id(&self) -> u64 { + match self { + UpdateStatus::Processing(u) => u.id(), + UpdateStatus::Enqueued(u) => u.id(), + UpdateStatus::Processed(u) => u.id(), + UpdateStatus::Aborted(u) => u.id(), + UpdateStatus::Failed(u) => u.id(), + } + } + + pub fn meta(&self) -> &UpdateMeta { + match self { + UpdateStatus::Processing(u) => u.meta(), + UpdateStatus::Enqueued(u) => u.meta(), + UpdateStatus::Processed(u) => u.meta(), + UpdateStatus::Aborted(u) => u.meta(), + UpdateStatus::Failed(u) => u.meta(), + } + } + + pub fn processed(&self) -> Option<&Processed> { + match self { + UpdateStatus::Processed(p) => Some(p), + _ => None, + } + } +} + +impl From for UpdateStatus { + fn from(other: Enqueued) -> Self { + Self::Enqueued(other) + } +} + +impl From for UpdateStatus { + fn from(other: Aborted) -> Self { + Self::Aborted(other) + } +} + +impl From for UpdateStatus { + fn from(other: Processed) -> Self { + Self::Processed(other) + } +} + +impl From for UpdateStatus { + fn from(other: Processing) -> Self { + Self::Processing(other) + } +} + +impl From for UpdateStatus { + fn from(other: Failed) -> Self { + Self::Failed(other) + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs new file mode 100644 index 000000000..d221bd4f2 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -0,0 +1,98 @@ +use std::{collections::HashSet, path::PathBuf}; + +use log::{trace, warn}; +use tokio::sync::mpsc; +use uuid::Uuid; + +use super::{error::UuidResolverError, Result, UuidResolveMsg, UuidStore}; + +pub struct UuidResolverActor { + inbox: mpsc::Receiver, + store: S, +} + +impl UuidResolverActor { + pub fn new(inbox: mpsc::Receiver, store: S) -> Self { + Self { inbox, store } + } + + pub async fn run(mut self) { + use UuidResolveMsg::*; + + trace!("uuid resolver started"); + + loop { + match self.inbox.recv().await { + Some(Get { uid: name, ret }) => { + let _ = ret.send(self.handle_get(name).await); + } + Some(Delete { uid: name, ret }) => { + let _ = ret.send(self.handle_delete(name).await); + } + Some(List { ret }) => { + let _ = ret.send(self.handle_list().await); + } + Some(Insert { ret, uuid, name }) => { + let _ = ret.send(self.handle_insert(name, uuid).await); + } + Some(SnapshotRequest { path, ret }) => { + let _ = ret.send(self.handle_snapshot(path).await); + } + Some(GetSize { ret }) => { + let _ = ret.send(self.handle_get_size().await); + } + Some(DumpRequest { path, ret }) => { + let _ = ret.send(self.handle_dump(path).await); + } + // all senders have been dropped, need to quit. + None => break, + } + } + + warn!("exiting uuid resolver loop"); + } + + async fn handle_get(&self, uid: String) -> Result { + self.store + .get_uuid(uid.clone()) + .await? + .ok_or(UuidResolverError::UnexistingIndex(uid)) + } + + async fn handle_delete(&self, uid: String) -> Result { + self.store + .delete(uid.clone()) + .await? + .ok_or(UuidResolverError::UnexistingIndex(uid)) + } + + async fn handle_list(&self) -> Result> { + let result = self.store.list().await?; + Ok(result) + } + + async fn handle_snapshot(&self, path: PathBuf) -> Result> { + self.store.snapshot(path).await + } + + async fn handle_dump(&self, path: PathBuf) -> Result> { + self.store.dump(path).await + } + + async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { + if !is_index_uid_valid(&uid) { + return Err(UuidResolverError::BadlyFormatted(uid)); + } + self.store.insert(uid, uuid).await?; + Ok(()) + } + + async fn handle_get_size(&self) -> Result { + self.store.get_size().await + } +} + +fn is_index_uid_valid(uid: &str) -> bool { + uid.chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/error.rs b/meilisearch-http/src/index_controller/uuid_resolver/error.rs new file mode 100644 index 000000000..de3dc662e --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/error.rs @@ -0,0 +1,34 @@ +use meilisearch_error::{Code, ErrorCode}; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum UuidResolverError { + #[error("Index already exists.")] + NameAlreadyExist, + #[error("Index \"{0}\" not found.")] + UnexistingIndex(String), + #[error("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")] + BadlyFormatted(String), + #[error("Internal error: {0}")] + Internal(Box), +} + +internal_error!( + UuidResolverError: heed::Error, + uuid::Error, + std::io::Error, + tokio::task::JoinError, + serde_json::Error +); + +impl ErrorCode for UuidResolverError { + fn error_code(&self) -> Code { + match self { + UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists, + UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound, + UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid, + UuidResolverError::Internal(_) => Code::Internal, + } + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs new file mode 100644 index 000000000..1296264e0 --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -0,0 +1,87 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use super::{HeedUuidStore, Result, UuidResolveMsg, UuidResolverActor, UuidResolverHandle}; + +#[derive(Clone)] +pub struct UuidResolverHandleImpl { + sender: mpsc::Sender, +} + +impl UuidResolverHandleImpl { + pub fn new(path: impl AsRef) -> Result { + let (sender, reveiver) = mpsc::channel(100); + let store = HeedUuidStore::new(path)?; + let actor = UuidResolverActor::new(reveiver, store); + tokio::spawn(actor.run()); + Ok(Self { sender }) + } +} + +#[async_trait::async_trait] +impl UuidResolverHandle for UuidResolverHandleImpl { + async fn get(&self, name: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Get { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn delete(&self, name: String) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Delete { uid: name, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn list(&self) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::List { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::Insert { ret, name, uuid }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn snapshot(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::SnapshotRequest { path, ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + + async fn get_size(&self) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetSize { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } + async fn dump(&self, path: PathBuf) -> Result> { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::DumpRequest { ret, path }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs new file mode 100644 index 000000000..46d9b585f --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -0,0 +1,37 @@ +use std::collections::HashSet; +use std::path::PathBuf; + +use tokio::sync::oneshot; +use uuid::Uuid; + +use super::Result; + +pub enum UuidResolveMsg { + Get { + uid: String, + ret: oneshot::Sender>, + }, + Delete { + uid: String, + ret: oneshot::Sender>, + }, + List { + ret: oneshot::Sender>>, + }, + Insert { + uuid: Uuid, + name: String, + ret: oneshot::Sender>, + }, + SnapshotRequest { + path: PathBuf, + ret: oneshot::Sender>>, + }, + GetSize { + ret: oneshot::Sender>, + }, + DumpRequest { + path: PathBuf, + ret: oneshot::Sender>>, + }, +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs new file mode 100644 index 000000000..da6c1264d --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -0,0 +1,35 @@ +mod actor; +pub mod error; +mod handle_impl; +mod message; +pub mod store; + +use std::collections::HashSet; +use std::path::PathBuf; + +use uuid::Uuid; + +use actor::UuidResolverActor; +use error::Result; +use message::UuidResolveMsg; +use store::UuidStore; + +#[cfg(test)] +use mockall::automock; + +pub use handle_impl::UuidResolverHandleImpl; +pub use store::HeedUuidStore; + +const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB + +#[async_trait::async_trait] +#[cfg_attr(test, automock)] +pub trait UuidResolverHandle { + async fn get(&self, name: String) -> Result; + async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; + async fn delete(&self, name: String) -> Result; + async fn list(&self) -> Result>; + async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; + async fn dump(&self, path: PathBuf) -> Result>; +} diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs new file mode 100644 index 000000000..f02d22d7f --- /dev/null +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -0,0 +1,224 @@ +use std::collections::HashSet; +use std::fs::{create_dir_all, File}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; + +use heed::types::{ByteSlice, Str}; +use heed::{CompactionOption, Database, Env, EnvOpenOptions}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{error::UuidResolverError, Result, UUID_STORE_SIZE}; +use crate::helpers::EnvSizer; + +#[derive(Serialize, Deserialize)] +struct DumpEntry { + uuid: Uuid, + uid: String, +} + +const UUIDS_DB_PATH: &str = "index_uuids"; + +#[async_trait::async_trait] +pub trait UuidStore: Sized { + // Create a new entry for `name`. Return an error if `err` and the entry already exists, return + // the uuid otherwise. + async fn get_uuid(&self, uid: String) -> Result>; + async fn delete(&self, uid: String) -> Result>; + async fn list(&self) -> Result>; + async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; + async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; + async fn dump(&self, path: PathBuf) -> Result>; +} + +#[derive(Clone)] +pub struct HeedUuidStore { + env: Env, + db: Database, +} + +impl HeedUuidStore { + pub fn new(path: impl AsRef) -> Result { + let path = path.as_ref().join(UUIDS_DB_PATH); + create_dir_all(&path)?; + let mut options = EnvOpenOptions::new(); + options.map_size(UUID_STORE_SIZE); // 1GB + let env = options.open(path)?; + let db = env.create_database(None)?; + Ok(Self { env, db }) + } + + pub fn get_uuid(&self, name: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + let txn = env.read_txn()?; + match db.get(&txn, &name)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + Ok(Some(uuid)) + } + None => Ok(None), + } + } + + pub fn delete(&self, uid: String) -> Result> { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; + match db.get(&txn, &uid)? { + Some(uuid) => { + let uuid = Uuid::from_slice(uuid)?; + db.delete(&mut txn, &uid)?; + txn.commit()?; + Ok(Some(uuid)) + } + None => Ok(None), + } + } + + pub fn list(&self) -> Result> { + let env = self.env.clone(); + let db = self.db; + let txn = env.read_txn()?; + let mut entries = Vec::new(); + for entry in db.iter(&txn)? { + let (name, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.push((name.to_owned(), uuid)) + } + Ok(entries) + } + + pub fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let env = self.env.clone(); + let db = self.db; + let mut txn = env.write_txn()?; + + if db.get(&txn, &name)?.is_some() { + return Err(UuidResolverError::NameAlreadyExist); + } + + db.put(&mut txn, &name, uuid.as_bytes())?; + txn.commit()?; + Ok(()) + } + + pub fn snapshot(&self, mut path: PathBuf) -> Result> { + let env = self.env.clone(); + let db = self.db; + // Write transaction to acquire a lock on the database. + let txn = env.write_txn()?; + let mut entries = HashSet::new(); + for entry in db.iter(&txn)? { + let (_, uuid) = entry?; + let uuid = Uuid::from_slice(uuid)?; + entries.insert(uuid); + } + + // only perform snapshot if there are indexes + if !entries.is_empty() { + path.push(UUIDS_DB_PATH); + create_dir_all(&path).unwrap(); + path.push("data.mdb"); + env.copy_to_path(path, CompactionOption::Enabled)?; + } + Ok(entries) + } + + pub fn get_size(&self) -> Result { + Ok(self.env.size()) + } + + pub fn dump(&self, path: PathBuf) -> Result> { + let dump_path = path.join(UUIDS_DB_PATH); + create_dir_all(&dump_path)?; + let dump_file_path = dump_path.join("data.jsonl"); + let mut dump_file = File::create(&dump_file_path)?; + let mut uuids = HashSet::new(); + + let txn = self.env.read_txn()?; + for entry in self.db.iter(&txn)? { + let (uid, uuid) = entry?; + let uid = uid.to_string(); + let uuid = Uuid::from_slice(uuid)?; + + let entry = DumpEntry { uuid, uid }; + serde_json::to_writer(&mut dump_file, &entry)?; + dump_file.write_all(b"\n").unwrap(); + + uuids.insert(uuid); + } + + Ok(uuids) + } + + pub fn load_dump(src: impl AsRef, dst: impl AsRef) -> Result<()> { + let uuid_resolver_path = dst.as_ref().join(UUIDS_DB_PATH); + std::fs::create_dir_all(&uuid_resolver_path)?; + + let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl"); + let indexes = File::open(&src_indexes)?; + let mut indexes = BufReader::new(indexes); + let mut line = String::new(); + + let db = Self::new(dst)?; + let mut txn = db.env.write_txn()?; + + loop { + match indexes.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let DumpEntry { uuid, uid } = serde_json::from_str(&line)?; + println!("importing {} {}", uid, uuid); + db.db.put(&mut txn, &uid, uuid.as_bytes())?; + } + Err(e) => return Err(e.into()), + } + + line.clear(); + } + txn.commit()?; + + db.env.prepare_for_closing().wait(); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl UuidStore for HeedUuidStore { + async fn get_uuid(&self, name: String) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.get_uuid(name)).await? + } + + async fn delete(&self, uid: String) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.delete(uid)).await? + } + + async fn list(&self) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.list()).await? + } + + async fn insert(&self, name: String, uuid: Uuid) -> Result<()> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.insert(name, uuid)).await? + } + + async fn snapshot(&self, path: PathBuf) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.snapshot(path)).await? + } + + async fn get_size(&self) -> Result { + self.get_size() + } + + async fn dump(&self, path: PathBuf) -> Result> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.dump(path)).await? + } +} diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 12a2f85a8..0eb61f84c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,105 +1,138 @@ -#![allow(clippy::or_fun_call)] - pub mod data; +#[macro_use] pub mod error; +#[macro_use] +pub mod extractors; pub mod helpers; -pub mod models; +mod index; +mod index_controller; pub mod option; pub mod routes; + +#[cfg(all(not(debug_assertions), feature = "analytics"))] pub mod analytics; -pub mod snapshot; -pub mod dump; -use actix_http::Error; -use actix_service::ServiceFactory; -use actix_web::{dev, web, App}; -use chrono::Utc; -use log::error; +use crate::extractors::authentication::AuthConfig; -use meilisearch_core::{Index, MainWriter, ProcessedUpdateResult}; - -pub use option::Opt; pub use self::data::Data; -use self::error::{payload_error_handler, ResponseError}; +pub use option::Opt; -pub fn create_app( - data: &Data, - enable_frontend: bool, -) -> App< - impl ServiceFactory< - Config = (), - Request = dev::ServiceRequest, - Response = dev::ServiceResponse, - Error = Error, - InitError = (), - >, - actix_http::body::Body, -> { - let app = App::new() +use actix_web::web; + +use extractors::authentication::policies::*; +use extractors::payload::PayloadConfig; + +pub fn configure_data(config: &mut web::ServiceConfig, data: Data) { + let http_payload_size_limit = data.http_payload_size_limit(); + config .data(data.clone()) + .app_data(data) .app_data( web::JsonConfig::default() - .limit(data.http_payload_size_limit) + .limit(http_payload_size_limit) .content_type(|_mime| true) // Accept all mime types - .error_handler(|err, _req| payload_error_handler(err).into()), + .error_handler(|err, _req| error::payload_error_handler(err).into()), ) + .app_data(PayloadConfig::new(http_payload_size_limit)) .app_data( web::QueryConfig::default() - .error_handler(|err, _req| payload_error_handler(err).into()) - ) - .configure(routes::document::services) - .configure(routes::index::services) - .configure(routes::search::services) - .configure(routes::setting::services) - .configure(routes::stop_words::services) - .configure(routes::synonym::services) - .configure(routes::health::services) - .configure(routes::stats::services) - .configure(routes::key::services) - .configure(routes::dump::services); - if enable_frontend { - app - .service(routes::load_html) - .service(routes::load_css) + .error_handler(|err, _req| error::payload_error_handler(err).into()), + ); +} + +pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) { + let keys = data.api_keys(); + let auth_config = if let Some(ref master_key) = keys.master { + let private_key = keys.private.as_ref().unwrap(); + let public_key = keys.public.as_ref().unwrap(); + let mut policies = init_policies!(Public, Private, Admin); + create_users!( + policies, + master_key.as_bytes() => { Admin, Private, Public }, + private_key.as_bytes() => { Private, Public }, + public_key.as_bytes() => { Public } + ); + AuthConfig::Auth(policies) } else { - app - .service(routes::running) - } + AuthConfig::NoAuth + }; + + config.app_data(auth_config); } -pub fn index_update_callback_txn(index: Index, index_uid: &str, data: &Data, mut writer: &mut MainWriter) -> Result<(), String> { - if let Err(e) = data.db.compute_stats(&mut writer, index_uid) { - return Err(format!("Impossible to compute stats; {}", e)); +#[cfg(feature = "mini-dashboard")] +pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { + use actix_web::HttpResponse; + use actix_web_static_files::Resource; + + mod generated { + include!(concat!(env!("OUT_DIR"), "/generated.rs")); } - if let Err(e) = data.db.set_last_update(&mut writer, &Utc::now()) { - return Err(format!("Impossible to update last_update; {}", e)); - } - - if let Err(e) = index.main.put_updated_at(&mut writer) { - return Err(format!("Impossible to update updated_at; {}", e)); - } - - Ok(()) -} - -pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) { - if status.error.is_some() { - return; - } - - if let Some(index) = data.db.open_index(index_uid) { - let db = &data.db; - let res = db.main_write::<_, _, ResponseError>(|mut writer| { - if let Err(e) = index_update_callback_txn(index, index_uid, data, &mut writer) { - error!("{}", e); + if enable_frontend { + let generated = generated::generate(); + let mut scope = web::scope("/"); + // Generate routes for mini-dashboard assets + for (path, resource) in generated.into_iter() { + let Resource { + mime_type, data, .. + } = resource; + // Redirect index.html to / + if path == "index.html" { + config.service(web::resource("/").route( + web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), + )); + } else { + scope = scope.service(web::resource(path).route( + web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), + )); } - - Ok(()) - }); - match res { - Ok(_) => (), - Err(e) => error!("{}", e), } + config.service(scope); + } else { + config.service(web::resource("/").route(web::get().to(routes::running))); } } + +#[cfg(not(feature = "mini-dashboard"))] +pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { + config.service(web::resource("/").route(web::get().to(routes::running))); +} + +#[macro_export] +macro_rules! create_app { + ($data:expr, $enable_frontend:expr) => {{ + use actix_cors::Cors; + use actix_web::middleware::TrailingSlash; + use actix_web::App; + use actix_web::{middleware, web}; + use meilisearch_http::routes::*; + use meilisearch_http::{configure_auth, configure_data, dashboard}; + + App::new() + .configure(|s| configure_data(s, $data.clone())) + .configure(|s| configure_auth(s, &$data)) + .configure(document::services) + .configure(index::services) + .configure(search::services) + .configure(settings::services) + .configure(health::services) + .configure(stats::services) + .configure(key::services) + .configure(dump::services) + .configure(|s| dashboard(s, $enable_frontend)) + .wrap( + Cors::default() + .send_wildcard() + .allowed_headers(vec!["content-type", "x-meili-api-key"]) + .allow_any_origin() + .allow_any_method() + .max_age(86_400), // 24h + ) + .wrap(middleware::Logger::default()) + .wrap(middleware::Compress::default()) + .wrap(middleware::NormalizePath::new( + middleware::TrailingSlash::Trim, + )) + }}; +} diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 0fd84fac1..5638c453f 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -1,35 +1,32 @@ -use std::{env, thread}; +use std::env; -use actix_cors::Cors; -use actix_web::{middleware, HttpServer}; +use actix_web::HttpServer; use main_error::MainError; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::{create_app, index_update_callback, Data, Opt}; +use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; -use meilisearch_http::{snapshot, dump}; -mod analytics; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +use meilisearch_http::analytics; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +use std::sync::Arc; #[cfg(target_os = "linux")] #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337"; + #[actix_web::main] async fn main() -> Result<(), MainError> { let opt = Opt::from_args(); - #[cfg(all(not(debug_assertions), feature = "sentry"))] - let _sentry = sentry::init(( - if !opt.no_sentry { - Some(opt.sentry_dsn.clone()) - } else { - None - }, - sentry::ClientOptions { - release: sentry::release_name!(), - ..Default::default() - }, - )); + let mut log_builder = env_logger::Builder::new(); + log_builder.parse_filters(&opt.log_level); + if opt.log_level == "info" { + // if we are in info we only allow the warn log_level for milli + log_builder.filter_module("milli", log::LevelFilter::Warn); + } match opt.env.as_ref() { "production" => { @@ -40,61 +37,60 @@ async fn main() -> Result<(), MainError> { ); } - #[cfg(all(not(debug_assertions), feature = "sentry"))] - if !opt.no_sentry && _sentry.is_enabled() { - sentry::integrations::panic::register_panic_handler(); // TODO: This shouldn't be needed when upgrading to sentry 0.19.0. These integrations are turned on by default when using `sentry::init`. - sentry::integrations::env_logger::init(None, Default::default()); + #[cfg(all(not(debug_assertions), feature = "analytics"))] + if !opt.no_analytics { + let logger = + sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); + log::set_boxed_logger(Box::new(logger)) + .map(|()| log::set_max_level(log::LevelFilter::Info)) + .unwrap(); + + let sentry = sentry::init(sentry::ClientOptions { + release: sentry::release_name!(), + dsn: Some(SENTRY_DSN.parse()?), + before_send: Some(Arc::new(|event| { + event + .message + .as_ref() + .map(|msg| msg.to_lowercase().contains("no space left on device")) + .unwrap_or(false) + .then(|| event) + })), + ..Default::default() + }); + // sentry must stay alive as long as possible + std::mem::forget(sentry); + } else { + log_builder.init(); } } "development" => { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + log_builder.init(); } _ => unreachable!(), } - if let Some(path) = &opt.import_snapshot { - snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; - } - let data = Data::new(opt.clone())?; + #[cfg(all(not(debug_assertions), feature = "analytics"))] if !opt.no_analytics { let analytics_data = data.clone(); let analytics_opt = opt.clone(); - thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); - } - - let data_cloned = data.clone(); - data.db.set_update_callback(Box::new(move |name, status| { - index_update_callback(name, &data_cloned, status); - })); - - - if let Some(path) = &opt.import_dump { - dump::import_dump(&data, path, opt.dump_batch_size)?; - } - - if opt.schedule_snapshot { - snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt)); } print_launch_resume(&opt, &data); - let enable_frontend = opt.env != "production"; - let http_server = HttpServer::new(move || { - let cors = Cors::default() - .send_wildcard() - .allowed_headers(vec!["content-type", "x-meili-api-key"]) - .allow_any_origin() - .allow_any_method() - .max_age(86_400); // 24h + run_http(data, opt).await?; - create_app(&data, enable_frontend) - .wrap(cors) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(NormalizePath) - }); + Ok(()) +} + +async fn run_http(data: Data, opt: Opt) -> Result<(), Box> { + let _enable_dashboard = &opt.env == "development"; + let http_server = HttpServer::new(move || create_app!(data, _enable_dashboard)) + // Disable signals allows the server to terminate immediately when a user enter CTRL-C + .disable_signals(); if let Some(config) = opt.get_ssl_config()? { http_server @@ -104,11 +100,19 @@ async fn main() -> Result<(), MainError> { } else { http_server.bind(opt.http_addr)?.run().await?; } - Ok(()) } pub fn print_launch_resume(opt: &Opt, data: &Data) { + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_SHA"), + Some(commit_sha) => commit_sha, + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some(commit_date) => commit_date, + }; + let ascii_name = r#" 888b d888 d8b 888 d8b .d8888b. 888 8888b d8888 Y8P 888 Y8P d88P Y88b 888 @@ -125,38 +129,32 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!("Database path:\t\t{:?}", opt.db_path); eprintln!("Server listening on:\t\"http://{}\"", opt.http_addr); eprintln!("Environment:\t\t{:?}", opt.env); - eprintln!("Commit SHA:\t\t{:?}", env!("VERGEN_SHA").to_string()); - eprintln!( - "Build date:\t\t{:?}", - env!("VERGEN_BUILD_TIMESTAMP").to_string() - ); + eprintln!("Commit SHA:\t\t{:?}", commit_sha.to_string()); + eprintln!("Commit date:\t\t{:?}", commit_date.to_string()); eprintln!( "Package version:\t{:?}", env!("CARGO_PKG_VERSION").to_string() ); - #[cfg(all(not(debug_assertions), feature = "sentry"))] - eprintln!( - "Sentry DSN:\t\t{:?}", - if !opt.no_sentry { - &opt.sentry_dsn + #[cfg(all(not(debug_assertions), feature = "analytics"))] + { + if opt.no_analytics { + eprintln!("Anonymous telemetry:\t\"Disabled\""); } else { - "Disabled" - } - ); + eprintln!( + " +Thank you for using MeiliSearch! - eprintln!( - "Anonymous telemetry:\t{:?}", - if !opt.no_analytics { - "Enabled" - } else { - "Disabled" +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics + +Anonymous telemetry: \"Enabled\"" + ); } - ); + } eprintln!(); - if data.api_keys.master.is_some() { + if data.api_keys().master.is_some() { eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key."); } else { eprintln!("No master key found; The server will accept unidentified requests. \ @@ -166,6 +164,6 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { eprintln!(); eprintln!("Documentation:\t\thttps://docs.meilisearch.com"); eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch"); - eprintln!("Contact:\t\thttps://docs.meilisearch.com/learn/what_is_meilisearch/contact.html or bonjour@meilisearch.com"); + eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html or bonjour@meilisearch.com"); eprintln!(); } diff --git a/meilisearch-http/src/models/mod.rs b/meilisearch-http/src/models/mod.rs deleted file mode 100644 index 82e7e77c4..000000000 --- a/meilisearch-http/src/models/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod update_operation; diff --git a/meilisearch-http/src/models/update_operation.rs b/meilisearch-http/src/models/update_operation.rs deleted file mode 100644 index e7a41b10b..000000000 --- a/meilisearch-http/src/models/update_operation.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::fmt; - -#[allow(dead_code)] -#[derive(Debug)] -pub enum UpdateOperation { - ClearAllDocuments, - DocumentsAddition, - DocumentsDeletion, - SynonymsUpdate, - SynonymsDeletion, - StopWordsAddition, - StopWordsDeletion, - Schema, - Config, -} - -impl fmt::Display for UpdateOperation { - fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result { - use UpdateOperation::*; - - match self { - ClearAllDocuments => write!(f, "ClearAllDocuments"), - DocumentsAddition => write!(f, "DocumentsAddition"), - DocumentsDeletion => write!(f, "DocumentsDeletion"), - SynonymsUpdate => write!(f, "SynonymsUpdate"), - SynonymsDeletion => write!(f, "SynonymsDelettion"), - StopWordsAddition => write!(f, "StopWordsAddition"), - StopWordsDeletion => write!(f, "StopWordsDeletion"), - Schema => write!(f, "Schema"), - Config => write!(f, "Config"), - } - } -} diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 62adc4c69..0e75b63c8 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -1,8 +1,10 @@ -use std::{error, fs}; use std::io::{BufReader, Read}; use std::path::PathBuf; use std::sync::Arc; +use std::{error, fs}; +use byte_unit::Byte; +use grenad::CompressionType; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, @@ -10,13 +12,81 @@ use rustls::{ }; use structopt::StructOpt; +#[derive(Debug, Clone, StructOpt)] +pub struct IndexerOpts { + /// The amount of documents to skip before printing + /// a log regarding the indexing advancement. + #[structopt(long, default_value = "100000")] // 100k + pub log_every_n: usize, + + /// Grenad max number of chunks in bytes. + #[structopt(long)] + pub max_nb_chunks: Option, + + /// The maximum amount of memory to use for the Grenad buffer. It is recommended + /// to use something like 80%-90% of the available memory. + /// + /// It is automatically split by the number of jobs e.g. if you use 7 jobs + /// and 7 GB of max memory, each thread will use a maximum of 1 GB. + #[structopt(long, default_value = "7 GiB")] + pub max_memory: Byte, + + /// Size of the linked hash map cache when indexing. + /// The bigger it is, the faster the indexing is but the more memory it takes. + #[structopt(long, default_value = "500")] + pub linked_hash_map_size: usize, + + /// The name of the compression algorithm to use when compressing intermediate + /// Grenad chunks while indexing documents. + /// + /// Choosing a fast algorithm will make the indexing faster but may consume more memory. + #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] + pub chunk_compression_type: CompressionType, + + /// The level of compression of the chosen algorithm. + #[structopt(long, requires = "chunk-compression-type")] + pub chunk_compression_level: Option, + + /// The number of bytes to remove from the begining of the chunks while reading/sorting + /// or merging them. + /// + /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, + /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. + #[structopt(long, default_value = "4 GiB")] + pub chunk_fusing_shrink_size: Byte, + + /// Enable the chunk fusing or not, this reduces the amount of disk space used. + #[structopt(long)] + pub enable_chunk_fusing: bool, + + /// Number of parallel jobs for indexing, defaults to # of CPUs. + #[structopt(long)] + pub indexing_jobs: Option, +} + +impl Default for IndexerOpts { + fn default() -> Self { + Self { + log_every_n: 100_000, + max_nb_chunks: None, + max_memory: Byte::from_str("1GiB").unwrap(), + linked_hash_map_size: 500, + chunk_compression_type: CompressionType::None, + chunk_compression_level: None, + chunk_fusing_shrink_size: Byte::from_str("4GiB").unwrap(), + enable_chunk_fusing: false, + indexing_jobs: None, + } + } +} + const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; -#[derive(Debug, Default, Clone, StructOpt)] +#[derive(Debug, Clone, StructOpt)] pub struct Opt { /// The destination where the database must be created. #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] - pub db_path: String, + pub db_path: PathBuf, /// The address on which the http server will listen. #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")] @@ -26,17 +96,6 @@ pub struct Opt { #[structopt(long, env = "MEILI_MASTER_KEY")] pub master_key: Option, - /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. - /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. - #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt(long, env = "SENTRY_DSN", default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337")] - pub sentry_dsn: String, - - /// Disable Sentry error reporting. - #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt(long, env = "MEILI_NO_SENTRY")] - pub no_sentry: bool, - /// This environment variable must be set to `production` if you are running in production. /// If the server is running in development mode more logs will be displayed, /// and the master key can be avoided which implies that there is no security on the updates routes. @@ -45,20 +104,21 @@ pub struct Opt { pub env: String, /// Do not send analytics to Meili. + #[cfg(all(not(debug_assertions), feature = "analytics"))] #[structopt(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[structopt(long, env = "MEILI_MAX_MDB_SIZE", default_value = "107374182400")] // 100GB - pub max_mdb_size: usize, + #[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")] + pub max_index_size: Byte, /// The maximum size, in bytes, of the update lmdb database directory - #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "107374182400")] // 100GB - pub max_udb_size: usize, + #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "100 GiB")] + pub max_udb_size: Byte, /// The maximum size, in bytes, of accepted JSON payloads - #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "104857600")] // 100MB - pub http_payload_size_limit: usize, + #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")] + pub http_payload_size_limit: Byte, /// Read server certificates from CERTFILE. /// This should contain PEM-format certificates @@ -117,20 +177,23 @@ pub struct Opt { pub schedule_snapshot: bool, /// Defines time interval, in seconds, between each snapshot creation. - #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC")] - pub snapshot_interval_sec: Option, + #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h + pub snapshot_interval_sec: u64, /// Folder where dumps are created when the dump route is called. #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] pub dumps_dir: PathBuf, - /// Import a dump from the specified path, must be a `.tar.gz` file. + /// Import a dump from the specified path, must be a `.dump` file. #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, - /// The batch size used in the importation process, the bigger it is the faster the dump is created. - #[structopt(long, env = "MEILI_DUMP_BATCH_SIZE", default_value = "1024")] - pub dump_batch_size: usize, + /// Set the log level + #[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")] + pub log_level: String, + + #[structopt(skip)] + pub indexer_options: IndexerOpts, } impl Opt { diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/document.rs index 202575cc3..418c67462 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/document.rs @@ -1,18 +1,48 @@ -use std::collections::{BTreeSet, HashSet}; - -use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; -use indexmap::IndexMap; -use meilisearch_core::{update, MainReader}; -use serde_json::Value; +use log::debug; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; use serde::Deserialize; +use serde_json::Value; +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::payload::Payload; +use crate::routes::IndexParam; use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; -type Document = IndexMap; +const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; +const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; + +/* +macro_rules! guard_content_type { + ($fn_name:ident, $guard_value:literal) => { + fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { + if let Some(content_type) = head.headers.get("Content-Type") { + content_type + .to_str() + .map(|v| v.contains($guard_value)) + .unwrap_or(false) + } else { + false + } + } + }; +} + +guard_content_type!(guard_json, "application/json"); +*/ + +fn guard_json(head: &actix_web::dev::RequestHead) -> bool { + if let Some(content_type) = head.headers.get("Content-Type") { + content_type + .to_str() + .map(|v| v.contains("application/json")) + .unwrap_or(false) + } else { + // if no content-type is specified we still accept the data as json! + true + } +} #[derive(Deserialize)] struct DocumentParam { @@ -21,64 +51,50 @@ struct DocumentParam { } pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get_document) - .service(delete_document) - .service(get_all_documents) - .service(add_documents) - .service(update_documents) - .service(delete_documents) - .service(clear_all_documents); + cfg.service( + web::scope("/indexes/{index_uid}/documents") + .service( + web::resource("") + .route(web::get().to(get_all_documents)) + .route(web::post().guard(guard_json).to(add_documents)) + .route(web::put().guard(guard_json).to(update_documents)) + .route(web::delete().to(clear_all_documents)), + ) + // this route needs to be before the /documents/{document_id} to match properly + .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) + .service( + web::resource("/{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), + ), + ); } -#[get( - "/indexes/{index_uid}/documents/{document_id}", - wrap = "Authentication::Public" -)] async fn get_document( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - - let internal_id = index - .main - .external_to_internal_docid(&reader, &path.document_id)? - .ok_or(Error::document_not_found(&path.document_id))?; - - let document: Document = index - .document(&reader, None, internal_id)? - .ok_or(Error::document_not_found(&path.document_id))?; - + let index = path.index_uid.clone(); + let id = path.document_id.clone(); + let document = data + .retrieve_document(index, id, None as Option>) + .await?; + debug!("returns: {:?}", document); Ok(HttpResponse::Ok().json(document)) } -#[delete( - "/indexes/{index_uid}/documents/{document_id}", - wrap = "Authentication::Private" -)] async fn delete_document( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let mut documents_deletion = index.documents_deletion(); - documents_deletion.delete_document_by_external_docid(path.document_id.clone()); - - let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + let update_status = data + .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()]) + .await?; + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct BrowseQuery { offset: Option, @@ -86,181 +102,112 @@ struct BrowseQuery { attributes_to_retrieve: Option, } -pub fn get_all_documents_sync( - data: &web::Data, - reader: &MainReader, - index_uid: &str, - offset: usize, - limit: usize, - attributes_to_retrieve: Option<&String> -) -> Result, Error> { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - - let documents_ids: Result, _> = index - .documents_fields_counts - .documents_ids(reader)? - .skip(offset) - .take(limit) - .collect(); - - let attributes: Option> = attributes_to_retrieve - .map(|a| a.split(',').collect()); - - let mut documents = Vec::new(); - for document_id in documents_ids? { - if let Ok(Some(document)) = - index.document::(reader, attributes.as_ref(), document_id) - { - documents.push(document); - } - } - - Ok(documents) -} - -#[get("/indexes/{index_uid}/documents", wrap = "Authentication::Public")] async fn get_all_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { - let offset = params.offset.unwrap_or(0); - let limit = params.limit.unwrap_or(20); - let index_uid = &path.index_uid; - let reader = data.db.main_read_txn()?; - - let documents = get_all_documents_sync( - &data, - &reader, - index_uid, - offset, - limit, - params.attributes_to_retrieve.as_ref() - )?; + debug!("called with params: {:?}", params); + let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| { + let mut names = Vec::new(); + for name in attrs.split(',').map(String::from) { + if name == "*" { + return None; + } + names.push(name); + } + Some(names) + }); + let documents = data + .retrieve_documents( + path.index_uid.clone(), + params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), + params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), + attributes_to_retrieve, + ) + .await?; + debug!("returns: {:?}", documents); Ok(HttpResponse::Ok().json(documents)) } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateDocumentsQuery { primary_key: Option, } -async fn update_multiple_documents( - data: web::Data, - path: web::Path, - params: web::Query, - body: web::Json>, - is_partial: bool, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - - let mut document_addition = if is_partial { - index.documents_partial_addition() - } else { - index.documents_addition() - }; - - // Return an early error if primary key is already set, otherwise, try to set it up in the - // update later. - let reader = data.db.main_read_txn()?; - let schema = index - .main - .schema(&reader)? - .ok_or(meilisearch_core::Error::SchemaMissing)?; - - match (params.into_inner().primary_key, schema.primary_key()) { - (Some(key), None) => document_addition.set_primary_key(key), - (None, None) => { - let key = body - .first() - .and_then(find_primary_key) - .ok_or(meilisearch_core::Error::MissingPrimaryKey)?; - document_addition.set_primary_key(key); - } - _ => () - } - - for document in body.into_inner() { - document_addition.update_document(document); - } - - Ok(data.db.update_write(|w| document_addition.finalize(w))?) - })?; - return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))); -} - -fn find_primary_key(document: &IndexMap) -> Option { - for key in document.keys() { - if key.to_lowercase().contains("id") { - return Some(key.to_string()); - } - } - None -} - -#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +/// Route used when the payload type is "application/json" +/// Used to add or replace documents async fn add_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, - body: web::Json>, + body: Payload, ) -> Result { - update_multiple_documents(data, path, params, body, false).await + debug!("called with params: {:?}", params); + let update_status = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::ReplaceDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ) + .await?; + + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -#[put("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] +/// Route used when the payload type is "application/json" +/// Used to add or replace documents async fn update_documents( - data: web::Data, + data: GuardedData, path: web::Path, params: web::Query, - body: web::Json>, + body: Payload, ) -> Result { - update_multiple_documents(data, path, params, body, true).await + debug!("called with params: {:?}", params); + let update = data + .add_documents( + path.into_inner().index_uid, + IndexDocumentsMethod::UpdateDocuments, + UpdateFormat::Json, + body, + params.primary_key.clone(), + ) + .await?; + + debug!("returns: {:?}", update); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } -#[post( - "/indexes/{index_uid}/documents/delete-batch", - wrap = "Authentication::Private" -)] async fn delete_documents( - data: web::Data, + data: GuardedData, path: web::Path, body: web::Json>, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; + debug!("called with params: {:?}", body); + let ids = body + .iter() + .map(|v| { + v.as_str() + .map(String::from) + .unwrap_or_else(|| v.to_string()) + }) + .collect(); - let mut documents_deletion = index.documents_deletion(); - - for document_id in body.into_inner() { - let document_id = update::value_to_string(&document_id); - documents_deletion.delete_document_by_external_docid(document_id); - } - - let update_id = data.db.update_write(|w| documents_deletion.finalize(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + let update_status = data.delete_documents(path.index_uid.clone(), ids).await?; + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -#[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")] async fn clear_all_documents( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let update_id = data.db.update_write(|w| index.clear_all(w))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) + let update_status = data.clear_documents(path.index_uid.clone()).await?; + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 3c6d0e060..1f987a588 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,29 +1,21 @@ -use std::fs::File; -use std::path::Path; - -use actix_web::{get, post}; -use actix_web::{HttpResponse, web}; +use actix_web::{web, HttpResponse}; +use log::debug; use serde::{Deserialize, Serialize}; -use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process}; +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(trigger_dump) - .service(get_dump_status); + cfg.service(web::resource("/dumps").route(web::post().to(create_dump))) + .service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status))); } -#[post("/dumps", wrap = "Authentication::Private")] -async fn trigger_dump( - data: web::Data, -) -> Result { - let dumps_dir = Path::new(&data.dumps_dir); - match init_dump_process(&data, &dumps_dir) { - Ok(resume) => Ok(HttpResponse::Accepted().json(resume)), - Err(e) => Err(e.into()) - } +async fn create_dump(data: GuardedData) -> Result { + let res = data.create_dump().await?; + + debug!("returns: {:?}", res); + Ok(HttpResponse::Accepted().json(res)) } #[derive(Debug, Serialize)] @@ -37,28 +29,12 @@ struct DumpParam { dump_uid: String, } -#[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] async fn get_dump_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { - let dumps_dir = Path::new(&data.dumps_dir); - let dump_uid = &path.dump_uid; + let res = data.dump_status(path.dump_uid.clone()).await?; - if let Some(resume) = data.get_current_dump_info() { - if &resume.uid == dump_uid { - return Ok(HttpResponse::Ok().json(resume)); - } - } - - if File::open(compressed_dumps_dir(Path::new(dumps_dir), dump_uid)).is_ok() { - let resume = DumpInfo::new( - dump_uid.into(), - DumpStatus::Done - ); - - Ok(HttpResponse::Ok().json(resume)) - } else { - Err(Error::not_found("dump does not exist").into()) - } + debug!("returns: {:?}", res); + Ok(HttpResponse::Ok().json(res)) } diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs index 8d42b79bc..54237de1a 100644 --- a/meilisearch-http/src/routes/health.rs +++ b/meilisearch-http/src/routes/health.rs @@ -1,14 +1,11 @@ -use actix_web::get; use actix_web::{web, HttpResponse}; use crate::error::ResponseError; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get_health); + cfg.service(web::resource("/health").route(web::get().to(get_health))); } -#[get("/health")] async fn get_health() -> Result { - let payload = serde_json::json!({ "status": "available" }); - Ok(HttpResponse::Ok().json(payload)) + Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index aa0496920..badbdcc10 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -1,254 +1,51 @@ -use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; -use meilisearch_core::{Database, MainReader, UpdateReader}; -use meilisearch_core::update::UpdateStatus; -use rand::seq::SliceRandom; +use log::debug; use serde::{Deserialize, Serialize}; +use super::{IndexParam, UpdateStatusResponse}; +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::routes::IndexParam; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(list_indexes) - .service(get_index) - .service(create_index) - .service(update_index) - .service(delete_index) - .service(get_update_status) - .service(get_all_updates_status); -} - -fn generate_uid() -> String { - let mut rng = rand::thread_rng(); - let sample = b"abcdefghijklmnopqrstuvwxyz0123456789"; - sample - .choose_multiple(&mut rng, 8) - .map(|c| *c as char) - .collect() -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct IndexResponse { - pub name: String, - pub uid: String, - created_at: DateTime, - updated_at: DateTime, - pub primary_key: Option, -} - -pub fn list_indexes_sync(data: &web::Data, reader: &MainReader) -> Result, ResponseError> { - let mut indexes = Vec::new(); - - for index_uid in data.db.indexes_uids() { - let index = data.db.open_index(&index_uid); - - match index { - Some(index) => { - let name = index.main.name(reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - - let index_response = IndexResponse { - name, - uid: index_uid, - created_at, - updated_at, - primary_key, - }; - indexes.push(index_response); - } - None => error!( - "Index {} is referenced in the indexes list but cannot be found", - index_uid - ), - } - } - - Ok(indexes) -} - -#[get("/indexes", wrap = "Authentication::Private")] -async fn list_indexes(data: web::Data) -> Result { - let reader = data.db.main_read_txn()?; - let indexes = list_indexes_sync(&data, &reader)?; - - Ok(HttpResponse::Ok().json(indexes)) -} - -#[get("/indexes/{index_uid}", wrap = "Authentication::Private")] -async fn get_index( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - let name = index.main.name(&reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(&reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - let index_response = IndexResponse { - name, - uid: path.index_uid.clone(), - created_at, - updated_at, - primary_key, - }; - - Ok(HttpResponse::Ok().json(index_response)) + cfg.service( + web::resource("indexes") + .route(web::get().to(list_indexes)) + .route(web::post().to(create_index)), + ) + .service( + web::resource("/indexes/{index_uid}") + .route(web::get().to(get_index)) + .route(web::put().to(update_index)) + .route(web::delete().to(delete_index)), + ) + .service( + web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)), + ) + .service( + web::resource("/indexes/{index_uid}/updates/{update_id}") + .route(web::get().to(get_update_status)), + ); } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { - name: Option, - uid: Option, - primary_key: Option, -} - - -pub fn create_index_sync( - database: &std::sync::Arc, uid: String, - name: String, primary_key: Option, -) -> Result { - - let created_index = database - .create_index(&uid) - .map_err(|e| match e { - meilisearch_core::Error::IndexAlreadyExists => Error::IndexAlreadyExists(uid.clone()), - _ => Error::create_index(e) - })?; - - let index_response = database.main_write::<_, _, Error>(|mut write_txn| { - created_index.main.put_name(&mut write_txn, &name)?; - - let created_at = created_index - .main - .created_at(&write_txn)? - .ok_or(Error::internal("Impossible to read created at"))?; - - let updated_at = created_index - .main - .updated_at(&write_txn)? - .ok_or(Error::internal("Impossible to read updated at"))?; - - if let Some(id) = primary_key.clone() { - if let Some(mut schema) = created_index.main.schema(&write_txn)? { - schema - .set_primary_key(&id) - .map_err(Error::bad_request)?; - created_index.main.put_schema(&mut write_txn, &schema)?; - } - } - let index_response = IndexResponse { - name, - uid, - created_at, - updated_at, - primary_key, - }; - Ok(index_response) - })?; - - Ok(index_response) -} - -#[post("/indexes", wrap = "Authentication::Private")] -async fn create_index( - data: web::Data, - body: web::Json, -) -> Result { - if let (None, None) = (body.name.clone(), body.uid.clone()) { - return Err(Error::bad_request( - "Index creation must have an uid", - ).into()); - } - - let uid = match &body.uid { - Some(uid) => { - if uid - .chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') - { - uid.to_owned() - } else { - return Err(Error::InvalidIndexUid.into()); - } - } - None => loop { - let uid = generate_uid(); - if data.db.open_index(&uid).is_none() { - break uid; - } - }, - }; - - let name = body.name.as_ref().unwrap_or(&uid).to_string(); - - let index_response = create_index_sync(&data.db, uid, name, body.primary_key.clone())?; - - Ok(HttpResponse::Created().json(index_response)) } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { - name: Option, + uid: Option, primary_key: Option, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct UpdateIndexResponse { +pub struct UpdateIndexResponse { name: String, uid: String, created_at: DateTime, @@ -256,78 +53,50 @@ struct UpdateIndexResponse { primary_key: Option, } -#[put("/indexes/{index_uid}", wrap = "Authentication::Private")] -async fn update_index( - data: web::Data, - path: web::Path, - body: web::Json, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - data.db.main_write::<_, _, ResponseError>(|writer| { - if let Some(name) = &body.name { - index.main.put_name(writer, name)?; - } - - if let Some(id) = body.primary_key.clone() { - if let Some(mut schema) = index.main.schema(writer)? { - schema.set_primary_key(&id)?; - index.main.put_schema(writer, &schema)?; - } - } - index.main.put_updated_at(writer)?; - Ok(()) - })?; - - let reader = data.db.main_read_txn()?; - let name = index.main.name(&reader)?.ok_or(Error::internal( - "Impossible to get the name of an index", - ))?; - let created_at = index - .main - .created_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the create date of an index", - ))?; - let updated_at = index - .main - .updated_at(&reader)? - .ok_or(Error::internal( - "Impossible to get the last update date of an index", - ))?; - - let primary_key = match index.main.schema(&reader) { - Ok(Some(schema)) => match schema.primary_key() { - Some(primary_key) => Some(primary_key.to_owned()), - None => None, - }, - _ => None, - }; - - let index_response = IndexResponse { - name, - uid: path.index_uid.clone(), - created_at, - updated_at, - primary_key, - }; - - Ok(HttpResponse::Ok().json(index_response)) +async fn list_indexes(data: GuardedData) -> Result { + let indexes = data.list_indexes().await?; + debug!("returns: {:?}", indexes); + Ok(HttpResponse::Ok().json(indexes)) } -#[delete("/indexes/{index_uid}", wrap = "Authentication::Private")] -async fn delete_index( - data: web::Data, +async fn create_index( + data: GuardedData, + body: web::Json, +) -> Result { + let body = body.into_inner(); + let meta = data.create_index(body.uid, body.primary_key).await?; + Ok(HttpResponse::Ok().json(meta)) +} + +async fn get_index( + data: GuardedData, path: web::Path, ) -> Result { - if data.db.delete_index(&path.index_uid)? { - Ok(HttpResponse::NoContent().finish()) - } else { - Err(Error::index_not_found(&path.index_uid).into()) - } + let meta = data.index(path.index_uid.clone()).await?; + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) +} + +async fn update_index( + data: GuardedData, + path: web::Path, + body: web::Json, +) -> Result { + debug!("called with params: {:?}", body); + let body = body.into_inner(); + let meta = data + .update_index(path.into_inner().index_uid, body.primary_key, body.uid) + .await?; + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) +} + +async fn delete_index( + data: GuardedData, + path: web::Path, +) -> Result { + data.delete_index(path.index_uid.clone()).await?; + Ok(HttpResponse::NoContent().finish()) } #[derive(Deserialize)] @@ -336,53 +105,29 @@ struct UpdateParam { update_id: u64, } -#[get( - "/indexes/{index_uid}/updates/{update_id}", - wrap = "Authentication::Private" -)] async fn get_update_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.update_read_txn()?; - - let status = index.update_status(&reader, path.update_id)?; - - match status { - Some(status) => Ok(HttpResponse::Ok().json(status)), - None => Err(Error::NotFound(format!( - "Update {}", - path.update_id - )).into()), - } -} -pub fn get_all_updates_status_sync( - data: &web::Data, - reader: &UpdateReader, - index_uid: &str, -) -> Result, Error> { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - Ok(index.all_updates_status(reader)?) + let params = path.into_inner(); + let meta = data + .get_update_status(params.index_uid, params.update_id) + .await?; + let meta = UpdateStatusResponse::from(meta); + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) } -#[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - data: web::Data, + data: GuardedData, path: web::Path, ) -> Result { + let metas = data.get_updates_status(path.into_inner().index_uid).await?; + let metas = metas + .into_iter() + .map(UpdateStatusResponse::from) + .collect::>(); - let reader = data.db.update_read_txn()?; - - let response = get_all_updates_status_sync(&data, &reader, &path.index_uid)?; - - Ok(HttpResponse::Ok().json(response)) + debug!("returns: {:?}", metas); + Ok(HttpResponse::Ok().json(metas)) } diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs index a0cbaccc3..d47e264be 100644 --- a/meilisearch-http/src/routes/key.rs +++ b/meilisearch-http/src/routes/key.rs @@ -1,13 +1,11 @@ -use actix_web::web; -use actix_web::HttpResponse; -use actix_web::get; +use actix_web::{web, HttpResponse}; use serde::Serialize; -use crate::helpers::Authentication; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(list); + cfg.service(web::resource("/keys").route(web::get().to(list))); } #[derive(Serialize)] @@ -16,10 +14,9 @@ struct KeysResponse { public: Option, } -#[get("/keys", wrap = "Authentication::Admin")] -async fn list(data: web::Data) -> HttpResponse { +async fn list(data: GuardedData) -> HttpResponse { let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(KeysResponse { + HttpResponse::Ok().json(&KeysResponse { private: api_keys.private, public: api_keys.public, }) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e2aeb8171..520949cd8 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,16 +1,201 @@ -use actix_web::{get, HttpResponse}; +use std::time::Duration; + +use actix_web::HttpResponse; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use crate::error::ResponseError; +use crate::index::{Settings, Unchecked}; +use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; + pub mod document; +pub mod dump; pub mod health; pub mod index; pub mod key; pub mod search; -pub mod setting; +pub mod settings; pub mod stats; -pub mod stop_words; -pub mod synonym; -pub mod dump; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(clippy::large_enum_variant)] +#[serde(tag = "name")] +pub enum UpdateType { + ClearAll, + Customs, + DocumentsAddition { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option, + }, + DocumentsPartial { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option, + }, + DocumentsDeletion { + #[serde(skip_serializing_if = "Option::is_none")] + number: Option, + }, + Settings { + settings: Settings, + }, +} + +impl From<&UpdateStatus> for UpdateType { + fn from(other: &UpdateStatus) -> Self { + use milli::update::IndexDocumentsMethod::*; + + match other.meta() { + UpdateMeta::DocumentsAddition { method, .. } => { + let number = match other { + UpdateStatus::Processed(processed) => match processed.success { + UpdateResult::DocumentsAddition(ref addition) => { + Some(addition.nb_documents) + } + _ => None, + }, + _ => None, + }; + + match method { + ReplaceDocuments => UpdateType::DocumentsAddition { number }, + UpdateDocuments => UpdateType::DocumentsPartial { number }, + _ => unreachable!(), + } + } + UpdateMeta::ClearDocuments => UpdateType::ClearAll, + UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion { + number: Some(ids.len()), + }, + UpdateMeta::Settings(settings) => UpdateType::Settings { + settings: settings.clone(), + }, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + pub duration: f64, // in seconds + pub enqueued_at: DateTime, + pub processed_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FailedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + #[serde(flatten)] + pub response: ResponseError, + pub duration: f64, // in seconds + pub enqueued_at: DateTime, + pub processed_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EnqueuedUpdateResult { + pub update_id: u64, + #[serde(rename = "type")] + pub update_type: UpdateType, + pub enqueued_at: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub started_processing_at: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "status")] +pub enum UpdateStatusResponse { + Enqueued { + #[serde(flatten)] + content: EnqueuedUpdateResult, + }, + Processing { + #[serde(flatten)] + content: EnqueuedUpdateResult, + }, + Failed { + #[serde(flatten)] + content: FailedUpdateResult, + }, + Processed { + #[serde(flatten)] + content: ProcessedUpdateResult, + }, +} + +impl From for UpdateStatusResponse { + fn from(other: UpdateStatus) -> Self { + let update_type = UpdateType::from(&other); + + match other { + UpdateStatus::Processing(processing) => { + let content = EnqueuedUpdateResult { + update_id: processing.id(), + update_type, + enqueued_at: processing.from.enqueued_at, + started_processing_at: Some(processing.started_processing_at), + }; + UpdateStatusResponse::Processing { content } + } + UpdateStatus::Enqueued(enqueued) => { + let content = EnqueuedUpdateResult { + update_id: enqueued.id(), + update_type, + enqueued_at: enqueued.enqueued_at, + started_processing_at: None, + }; + UpdateStatusResponse::Enqueued { content } + } + UpdateStatus::Processed(processed) => { + let duration = processed + .processed_at + .signed_duration_since(processed.from.started_processing_at) + .num_milliseconds(); + + // necessary since chrono::duration don't expose a f64 secs method. + let duration = Duration::from_millis(duration as u64).as_secs_f64(); + + let content = ProcessedUpdateResult { + update_id: processed.id(), + update_type, + duration, + enqueued_at: processed.from.from.enqueued_at, + processed_at: processed.processed_at, + }; + UpdateStatusResponse::Processed { content } + } + UpdateStatus::Aborted(_) => unreachable!(), + UpdateStatus::Failed(failed) => { + let duration = failed + .failed_at + .signed_duration_since(failed.from.started_processing_at) + .num_milliseconds(); + + // necessary since chrono::duration don't expose a f64 secs method. + let duration = Duration::from_millis(duration as u64).as_secs_f64(); + + let update_id = failed.id(); + let response = failed.error; + + let content = FailedUpdateResult { + update_id, + update_type, + response, + duration, + enqueued_at: failed.from.from.enqueued_at, + processed_at: failed.failed_at, + }; + UpdateStatusResponse::Failed { content } + } + } + } +} #[derive(Deserialize)] pub struct IndexParam { @@ -29,28 +214,12 @@ impl IndexUpdateResponse { } } -/// Return the dashboard, should not be used in production. See [running] -#[get("/")] -pub async fn load_html() -> HttpResponse { - HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .body(include_str!("../../public/interface.html").to_string()) -} - /// Always return a 200 with: /// ```json /// { /// "status": "Meilisearch is running" /// } /// ``` -#[get("/")] pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) } - -#[get("/bulma.min.css")] -pub async fn load_css() -> HttpResponse { - HttpResponse::Ok() - .content_type("text/css; charset=utf-8") - .body(include_str!("../../public/bulma.min.css").to_string()) -} diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/search.rs index 0f86cafc8..31a7dbd03 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/search.rs @@ -1,270 +1,103 @@ -use std::collections::{HashMap, HashSet, BTreeSet}; +use std::collections::{BTreeSet, HashSet}; -use actix_web::{get, post, web, HttpResponse}; -use log::warn; -use serde::{Deserialize, Serialize}; +use actix_web::{web, HttpResponse}; +use log::debug; +use serde::Deserialize; use serde_json::Value; -use crate::error::{Error, FacetCountError, ResponseError}; -use crate::helpers::meilisearch::{IndexSearchExt, SearchResult}; -use crate::helpers::Authentication; +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -use meilisearch_core::facets::FacetFilter; -use meilisearch_schema::{FieldId, Schema}; - pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(search_with_post).service(search_with_url_query); + cfg.service( + web::resource("/indexes/{index_uid}/search") + .route(web::get().to(search_with_url_query)) + .route(web::post().to(search_with_post)), + ); } -#[derive(Serialize, Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SearchQuery { +pub struct SearchQueryGet { q: Option, offset: Option, limit: Option, attributes_to_retrieve: Option, attributes_to_crop: Option, - crop_length: Option, + #[serde(default = "default_crop_length")] + crop_length: usize, attributes_to_highlight: Option, - filters: Option, - matches: Option, - facet_filters: Option, + filter: Option, + #[serde(default = "Default::default")] + matches: bool, facets_distribution: Option, } -#[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] -async fn search_with_url_query( - data: web::Data, - path: web::Path, - params: web::Query, -) -> Result { - let search_result = params.search(&path.index_uid, data)?; - Ok(HttpResponse::Ok().json(search_result)) -} +impl From for SearchQuery { + fn from(other: SearchQueryGet) -> Self { + let attributes_to_retrieve = other + .attributes_to_retrieve + .map(|attrs| attrs.split(',').map(String::from).collect::>()); -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SearchQueryPost { - q: Option, - offset: Option, - limit: Option, - attributes_to_retrieve: Option>, - attributes_to_crop: Option>, - crop_length: Option, - attributes_to_highlight: Option>, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option>, -} + let attributes_to_crop = other + .attributes_to_crop + .map(|attrs| attrs.split(',').map(String::from).collect::>()); -impl From for SearchQuery { - fn from(other: SearchQueryPost) -> SearchQuery { - SearchQuery { + let attributes_to_highlight = other + .attributes_to_highlight + .map(|attrs| attrs.split(',').map(String::from).collect::>()); + + let facets_distribution = other + .facets_distribution + .map(|attrs| attrs.split(',').map(String::from).collect::>()); + + let filter = match other.filter { + Some(f) => match serde_json::from_str(&f) { + Ok(v) => Some(v), + _ => Some(Value::String(f)), + }, + None => None, + }; + + Self { q: other.q, offset: other.offset, - limit: other.limit, - attributes_to_retrieve: other.attributes_to_retrieve.map(|attrs| attrs.join(",")), - attributes_to_crop: other.attributes_to_crop.map(|attrs| attrs.join(",")), + limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT), + attributes_to_retrieve, + attributes_to_crop, crop_length: other.crop_length, - attributes_to_highlight: other.attributes_to_highlight.map(|attrs| attrs.join(",")), - filters: other.filters, + attributes_to_highlight, + filter, matches: other.matches, - facet_filters: other.facet_filters.map(|f| f.to_string()), - facets_distribution: other.facets_distribution.map(|f| format!("{:?}", f)), + facets_distribution, } } } -#[post("/indexes/{index_uid}/search", wrap = "Authentication::Public")] -async fn search_with_post( - data: web::Data, +async fn search_with_url_query( + data: GuardedData, path: web::Path, - params: web::Json, + params: web::Query, ) -> Result { - let query: SearchQuery = params.0.into(); - let search_result = query.search(&path.index_uid, data)?; + debug!("called with params: {:?}", params); + let query = params.into_inner().into(); + let search_result = data.search(path.into_inner().index_uid, query).await?; + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } -impl SearchQuery { - fn search( - &self, - index_uid: &str, - data: web::Data, - ) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let reader = data.db.main_read_txn()?; - let schema = index - .main - .schema(&reader)? - .ok_or(Error::internal("Impossible to retrieve the schema"))?; - - let query = self - .q - .clone() - .and_then(|q| if q.is_empty() { None } else { Some(q) }); - - let mut search_builder = index.new_search(query); - - if let Some(offset) = self.offset { - search_builder.offset(offset); - } - if let Some(limit) = self.limit { - search_builder.limit(limit); - } - - let available_attributes = schema.displayed_names(); - let mut restricted_attributes: BTreeSet<&str>; - match &self.attributes_to_retrieve { - Some(attributes_to_retrieve) => { - let attributes_to_retrieve: HashSet<&str> = - attributes_to_retrieve.split(',').collect(); - if attributes_to_retrieve.contains("*") { - restricted_attributes = available_attributes.clone(); - } else { - restricted_attributes = BTreeSet::new(); - search_builder.attributes_to_retrieve(HashSet::new()); - for attr in attributes_to_retrieve { - if available_attributes.contains(attr) { - restricted_attributes.insert(attr); - search_builder.add_retrievable_field(attr.to_string()); - } else { - warn!("The attributes {:?} present in attributesToRetrieve parameter doesn't exist", attr); - } - } - } - } - None => { - restricted_attributes = available_attributes.clone(); - } - } - - if let Some(ref facet_filters) = self.facet_filters { - let attrs = index - .main - .attributes_for_faceting(&reader)? - .unwrap_or_default(); - search_builder.add_facet_filters(FacetFilter::from_str( - facet_filters, - &schema, - &attrs, - )?); - } - - if let Some(facets) = &self.facets_distribution { - match index.main.attributes_for_faceting(&reader)? { - Some(ref attrs) => { - let field_ids = prepare_facet_list(&facets, &schema, attrs)?; - search_builder.add_facets(field_ids); - } - None => return Err(FacetCountError::NoFacetSet.into()), - } - } - - if let Some(attributes_to_crop) = &self.attributes_to_crop { - let default_length = self.crop_length.unwrap_or(200); - let mut final_attributes: HashMap = HashMap::new(); - - for attribute in attributes_to_crop.split(',') { - let mut attribute = attribute.split(':'); - let attr = attribute.next(); - let length = attribute - .next() - .and_then(|s| s.parse().ok()) - .unwrap_or(default_length); - match attr { - Some("*") => { - for attr in &restricted_attributes { - final_attributes.insert(attr.to_string(), length); - } - } - Some(attr) => { - if available_attributes.contains(attr) { - final_attributes.insert(attr.to_string(), length); - } else { - warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr); - } - } - None => (), - } - } - search_builder.attributes_to_crop(final_attributes); - } - - if let Some(attributes_to_highlight) = &self.attributes_to_highlight { - let mut final_attributes: HashSet = HashSet::new(); - for attribute in attributes_to_highlight.split(',') { - if attribute == "*" { - for attr in &restricted_attributes { - final_attributes.insert(attr.to_string()); - } - } else if available_attributes.contains(attribute) { - final_attributes.insert(attribute.to_string()); - } else { - warn!("The attributes {:?} present in attributesToHighlight parameter doesn't exist", attribute); - } - } - - search_builder.attributes_to_highlight(final_attributes); - } - - if let Some(filters) = &self.filters { - search_builder.filters(filters.to_string()); - } - - if let Some(matches) = self.matches { - if matches { - search_builder.get_matches(); - } - } - search_builder.search(&reader) - } -} - -/// Parses the incoming string into an array of attributes for which to return a count. It returns -/// a Vec of attribute names ascociated with their id. -/// -/// An error is returned if the array is malformed, or if it contains attributes that are -/// unexisting, or not set as facets. -fn prepare_facet_list( - facets: &str, - schema: &Schema, - facet_attrs: &[FieldId], -) -> Result, FacetCountError> { - let json_array = serde_json::from_str(facets)?; - match json_array { - Value::Array(vals) => { - let wildcard = Value::String("*".to_string()); - if vals.iter().any(|f| f == &wildcard) { - let attrs = facet_attrs - .iter() - .filter_map(|&id| schema.name(id).map(|n| (id, n.to_string()))) - .collect(); - return Ok(attrs); - } - let mut field_ids = Vec::with_capacity(facet_attrs.len()); - for facet in vals { - match facet { - Value::String(facet) => { - if let Some(id) = schema.id(&facet) { - if !facet_attrs.contains(&id) { - return Err(FacetCountError::AttributeNotSet(facet)); - } - field_ids.push((id, facet)); - } - } - bad_val => return Err(FacetCountError::unexpected_token(bad_val, &["String"])), - } - } - Ok(field_ids) - } - bad_val => Err(FacetCountError::unexpected_token(bad_val, &["[String]"])), - } +async fn search_with_post( + data: GuardedData, + path: web::Path, + params: web::Json, +) -> Result { + debug!("search called with params: {:?}", params); + let search_result = data + .search(path.into_inner().index_uid, params.into_inner()) + .await?; + debug!("returns: {:?}", search_result); + Ok(HttpResponse::Ok().json(search_result)) } diff --git a/meilisearch-http/src/routes/setting.rs b/meilisearch-http/src/routes/setting.rs deleted file mode 100644 index f7fae0a6c..000000000 --- a/meilisearch-http/src/routes/setting.rs +++ /dev/null @@ -1,547 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use actix_web::{delete, get, post}; -use actix_web::{web, HttpResponse}; -use meilisearch_core::{MainReader, UpdateWriter}; -use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES}; -use meilisearch_schema::Schema; - -use crate::Data; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(update_all) - .service(get_all) - .service(delete_all) - .service(get_rules) - .service(update_rules) - .service(delete_rules) - .service(get_distinct) - .service(update_distinct) - .service(delete_distinct) - .service(get_searchable) - .service(update_searchable) - .service(delete_searchable) - .service(get_displayed) - .service(update_displayed) - .service(delete_displayed) - .service(get_attributes_for_faceting) - .service(delete_attributes_for_faceting) - .service(update_attributes_for_faceting); -} - -pub fn update_all_settings_txn( - data: &web::Data, - settings: SettingsUpdate, - index_uid: &str, - write_txn: &mut UpdateWriter, -) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let update_id = index.settings_update(write_txn, settings)?; - Ok(update_id) -} - -#[post("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn update_all( - data: web::Data, - path: web::Path, - body: web::Json, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - Ok(data.db.update_write::<_, _, ResponseError>(|writer| { - let settings = body.into_inner().to_update().map_err(Error::bad_request)?; - let update_id = index.settings_update(writer, settings)?; - Ok(update_id) - })?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -pub fn get_all_sync(data: &web::Data, reader: &MainReader, index_uid: &str) -> Result { - let index = data - .db - .open_index(index_uid) - .ok_or(Error::index_not_found(index_uid))?; - - let stop_words: BTreeSet = index.main.stop_words(&reader)?.into_iter().collect(); - - let synonyms_list = index.main.synonyms(reader)?; - - let mut synonyms = BTreeMap::new(); - let index_synonyms = &index.synonyms; - for synonym in synonyms_list { - let list = index_synonyms.synonyms(reader, synonym.as_bytes())?; - synonyms.insert(synonym, list); - } - - let ranking_rules = index - .main - .ranking_rules(reader)? - .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) - .into_iter() - .map(|r| r.to_string()) - .collect(); - - let schema = index.main.schema(&reader)?; - - let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) { - (Some(id), Some(schema)) => schema.name(id).map(str::to_string), - _ => None, - }; - - let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) { - (Some(schema), Some(attrs)) => attrs - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect(), - _ => vec![], - }; - - let searchable_attributes = schema.as_ref().map(get_indexed_attributes); - let displayed_attributes = schema.as_ref().map(get_displayed_attributes); - - Ok(Settings { - ranking_rules: Some(Some(ranking_rules)), - distinct_attribute: Some(distinct_attribute), - searchable_attributes: Some(searchable_attributes), - displayed_attributes: Some(displayed_attributes), - stop_words: Some(Some(stop_words)), - synonyms: Some(Some(synonyms)), - attributes_for_faceting: Some(Some(attributes_for_faceting)), - }) -} - -#[get("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn get_all( - data: web::Data, - path: web::Path, -) -> Result { - let reader = data.db.main_read_txn()?; - let settings = get_all_sync(&data, &reader, &path.index_uid)?; - - Ok(HttpResponse::Ok().json(settings)) -} - -#[delete("/indexes/{index_uid}/settings", wrap = "Authentication::Private")] -async fn delete_all( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - ranking_rules: UpdateState::Clear, - distinct_attribute: UpdateState::Clear, - primary_key: UpdateState::Clear, - searchable_attributes: UpdateState::Clear, - displayed_attributes: UpdateState::Clear, - stop_words: UpdateState::Clear, - synonyms: UpdateState::Clear, - attributes_for_faceting: UpdateState::Clear, - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[get( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn get_rules( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - - let ranking_rules = index - .main - .ranking_rules(&reader)? - .unwrap_or(DEFAULT_RANKING_RULES.to_vec()) - .into_iter() - .map(|r| r.to_string()) - .collect::>(); - - Ok(HttpResponse::Ok().json(ranking_rules)) -} - -#[post( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn update_rules( - data: web::Data, - path: web::Path, - body: web::Json>>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - ranking_rules: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/ranking-rules", - wrap = "Authentication::Private" -)] -async fn delete_rules( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - ranking_rules: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[get( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn get_distinct( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let distinct_attribute_id = index.main.distinct_attribute(&reader)?; - let schema = index.main.schema(&reader)?; - let distinct_attribute = match (schema, distinct_attribute_id) { - (Some(schema), Some(id)) => schema.name(id).map(str::to_string), - _ => None, - }; - - Ok(HttpResponse::Ok().json(distinct_attribute)) -} - -#[post( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn update_distinct( - data: web::Data, - path: web::Path, - body: web::Json>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - distinct_attribute: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/distinct-attribute", - wrap = "Authentication::Private" -)] -async fn delete_distinct( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - distinct_attribute: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[get( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn get_searchable( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let schema = index.main.schema(&reader)?; - let searchable_attributes: Option> = schema.as_ref().map(get_indexed_attributes); - - Ok(HttpResponse::Ok().json(searchable_attributes)) -} - -#[post( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn update_searchable( - data: web::Data, - path: web::Path, - body: web::Json>>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - searchable_attributes: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/searchable-attributes", - wrap = "Authentication::Private" -)] -async fn delete_searchable( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - searchable_attributes: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[get( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn get_displayed( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - - let schema = index.main.schema(&reader)?; - - let displayed_attributes = schema.as_ref().map(get_displayed_attributes); - - Ok(HttpResponse::Ok().json(displayed_attributes)) -} - -#[post( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn update_displayed( - data: web::Data, - path: web::Path, - body: web::Json>>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - displayed_attributes: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/displayed-attributes", - wrap = "Authentication::Private" -)] -async fn delete_displayed( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - displayed_attributes: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[get( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn get_attributes_for_faceting( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| { - let schema = index.main.schema(reader)?; - let attrs = index.main.attributes_for_faceting(reader)?; - let attr_names = match (&schema, &attrs) { - (Some(schema), Some(attrs)) => attrs - .iter() - .filter_map(|&id| schema.name(id)) - .map(str::to_string) - .collect(), - _ => vec![], - }; - Ok(attr_names) - })?; - - Ok(HttpResponse::Ok().json(attributes_for_faceting)) -} - -#[post( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn update_attributes_for_faceting( - data: web::Data, - path: web::Path, - body: web::Json>>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = Settings { - attributes_for_faceting: Some(body.into_inner()), - ..Settings::default() - }; - - let settings = settings.to_update().map_err(Error::bad_request)?; - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/attributes-for-faceting", - wrap = "Authentication::Private" -)] -async fn delete_attributes_for_faceting( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - attributes_for_faceting: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -fn get_indexed_attributes(schema: &Schema) -> Vec { - if schema.is_searchable_all() { - vec!["*".to_string()] - } else { - schema - .searchable_names() - .iter() - .map(|s| s.to_string()) - .collect() - } -} - -fn get_displayed_attributes(schema: &Schema) -> BTreeSet { - if schema.is_displayed_all() { - ["*"].iter().map(|s| s.to_string()).collect() - } else { - schema - .displayed_names() - .iter() - .map(|s| s.to_string()) - .collect() - } -} diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/settings.rs new file mode 100644 index 000000000..812e37b58 --- /dev/null +++ b/meilisearch-http/src/routes/settings.rs @@ -0,0 +1,177 @@ +use actix_web::{web, HttpResponse}; +use log::debug; + +use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::index::Settings; +use crate::Data; +use crate::{error::ResponseError, index::Unchecked}; + +#[macro_export] +macro_rules! make_setting_route { + ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { + mod $attr { + use log::debug; + use actix_web::{web, HttpResponse, Resource}; + + use crate::data; + use crate::error::ResponseError; + use crate::index::Settings; + use crate::extractors::authentication::{GuardedData, policies::*}; + + async fn delete( + data: GuardedData, + index_uid: web::Path, + ) -> Result { + use crate::index::Settings; + let settings = Settings { + $attr: Some(None), + ..Default::default() + }; + let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?; + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } + + async fn update( + data: GuardedData, + index_uid: actix_web::web::Path, + body: actix_web::web::Json>, + ) -> std::result::Result { + let settings = Settings { + $attr: Some(body.into_inner()), + ..Default::default() + }; + + let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?; + debug!("returns: {:?}", update_status); + Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + } + + async fn get( + data: GuardedData, + index_uid: actix_web::web::Path, + ) -> std::result::Result { + let settings = data.settings(index_uid.into_inner()).await?; + debug!("returns: {:?}", settings); + let mut json = serde_json::json!(&settings); + let val = json[$camelcase_attr].take(); + Ok(HttpResponse::Ok().json(val)) + } + + pub fn resources() -> Resource { + Resource::new($route) + .route(web::get().to(get)) + .route(web::post().to(update)) + .route(web::delete().to(delete)) + } + } + }; +} + +make_setting_route!( + "/indexes/{index_uid}/settings/filterable-attributes", + std::collections::HashSet, + filterable_attributes, + "filterableAttributes" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/displayed-attributes", + Vec, + displayed_attributes, + "displayedAttributes" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/searchable-attributes", + Vec, + searchable_attributes, + "searchableAttributes" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/stop-words", + std::collections::BTreeSet, + stop_words, + "stopWords" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/synonyms", + std::collections::BTreeMap>, + synonyms, + "synonyms" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/distinct-attribute", + String, + distinct_attribute, + "distinctAttribute" +); + +make_setting_route!( + "/indexes/{index_uid}/settings/ranking-rules", + Vec, + ranking_rules, + "rankingRules" +); + +macro_rules! create_services { + ($($mod:ident),*) => { + pub fn services(cfg: &mut web::ServiceConfig) { + cfg + .service(web::resource("/indexes/{index_uid}/settings") + .route(web::post().to(update_all)) + .route(web::get().to(get_all)) + .route(web::delete().to(delete_all))) + $(.service($mod::resources()))*; + } + }; +} + +create_services!( + filterable_attributes, + displayed_attributes, + searchable_attributes, + distinct_attribute, + stop_words, + synonyms, + ranking_rules +); + +async fn update_all( + data: GuardedData, + index_uid: web::Path, + body: web::Json>, +) -> Result { + let settings = body.into_inner().check(); + let update_result = data + .update_settings(index_uid.into_inner(), settings, true) + .await?; + let json = serde_json::json!({ "updateId": update_result.id() }); + debug!("returns: {:?}", json); + Ok(HttpResponse::Accepted().json(json)) +} + +async fn get_all( + data: GuardedData, + index_uid: web::Path, +) -> Result { + let settings = data.settings(index_uid.into_inner()).await?; + debug!("returns: {:?}", settings); + Ok(HttpResponse::Ok().json(settings)) +} + +async fn delete_all( + data: GuardedData, + index_uid: web::Path, +) -> Result { + let settings = Settings::cleared(); + let update_result = data + .update_settings(index_uid.into_inner(), settings, false) + .await?; + let json = serde_json::json!({ "updateId": update_result.id() }); + debug!("returns: {:?}", json); + Ok(HttpResponse::Accepted().json(json)) +} diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index f8c531732..a0078d76a 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -1,134 +1,56 @@ -use std::collections::{HashMap, BTreeMap}; - -use actix_web::web; -use actix_web::HttpResponse; -use actix_web::get; -use chrono::{DateTime, Utc}; -use log::error; +use actix_web::{web, HttpResponse}; +use log::debug; use serde::Serialize; -use walkdir::WalkDir; -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::routes::IndexParam; use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(index_stats) - .service(get_stats) - .service(get_version); + cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats))) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))); } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct IndexStatsResponse { - number_of_documents: u64, - is_indexing: bool, - fields_distribution: BTreeMap, -} - -#[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")] -async fn index_stats( - data: web::Data, +async fn get_index_stats( + data: GuardedData, path: web::Path, ) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; + let response = data.get_index_stats(path.index_uid.clone()).await?; - let reader = data.db.main_read_txn()?; - - let number_of_documents = index.main.number_of_documents(&reader)?; - - let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); - - let update_reader = data.db.update_read_txn()?; - - let is_indexing = - data.db.is_indexing(&update_reader, &path.index_uid)? - .ok_or(Error::internal( - "Impossible to know if the database is indexing", - ))?; - - Ok(HttpResponse::Ok().json(IndexStatsResponse { - number_of_documents, - is_indexing, - fields_distribution, - })) + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct StatsResult { - database_size: u64, - last_update: Option>, - indexes: HashMap, -} +async fn get_stats(data: GuardedData) -> Result { + let response = data.get_all_stats().await?; -#[get("/stats", wrap = "Authentication::Private")] -async fn get_stats(data: web::Data) -> Result { - let mut index_list = HashMap::new(); - - let reader = data.db.main_read_txn()?; - let update_reader = data.db.update_read_txn()?; - - let indexes_set = data.db.indexes_uids(); - for index_uid in indexes_set { - let index = data.db.open_index(&index_uid); - match index { - Some(index) => { - let number_of_documents = index.main.number_of_documents(&reader)?; - - let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); - - let is_indexing = data.db.is_indexing(&update_reader, &index_uid)?.ok_or( - Error::internal("Impossible to know if the database is indexing"), - )?; - - let response = IndexStatsResponse { - number_of_documents, - is_indexing, - fields_distribution, - }; - index_list.insert(index_uid, response); - } - None => error!( - "Index {:?} is referenced in the indexes list but cannot be found", - index_uid - ), - } - } - - let database_size = WalkDir::new(&data.db_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()); - - let last_update = data.db.last_update(&reader)?; - - Ok(HttpResponse::Ok().json(StatsResult { - database_size, - last_update, - indexes: index_list, - })) + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) } #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct VersionResponse { commit_sha: String, - build_date: String, + commit_date: String, pkg_version: String, } -#[get("/version", wrap = "Authentication::Private")] -async fn get_version() -> HttpResponse { +async fn get_version(_data: GuardedData) -> HttpResponse { + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_SHA"), + Some(commit_sha) => commit_sha, + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some(commit_date) => commit_date, + }; + HttpResponse::Ok().json(VersionResponse { - commit_sha: env!("VERGEN_SHA").to_string(), - build_date: env!("VERGEN_BUILD_TIMESTAMP").to_string(), + commit_sha: commit_sha.to_string(), + commit_date: commit_date.to_string(), pkg_version: env!("CARGO_PKG_VERSION").to_string(), }) } diff --git a/meilisearch-http/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs deleted file mode 100644 index c757b4d14..000000000 --- a/meilisearch-http/src/routes/stop_words.rs +++ /dev/null @@ -1,79 +0,0 @@ -use actix_web::{web, HttpResponse}; -use actix_web::{delete, get, post}; -use meilisearch_core::settings::{SettingsUpdate, UpdateState}; -use std::collections::BTreeSet; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get).service(update).service(delete); -} - -#[get( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - let reader = data.db.main_read_txn()?; - let stop_words = index.main.stop_words(&reader)?; - - Ok(HttpResponse::Ok().json(stop_words)) -} - -#[post( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn update( - data: web::Data, - path: web::Path, - body: web::Json>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = SettingsUpdate { - stop_words: UpdateState::Update(body.into_inner()), - ..SettingsUpdate::default() - }; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/stop-words", - wrap = "Authentication::Private" -)] -async fn delete( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - stop_words: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} diff --git a/meilisearch-http/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs deleted file mode 100644 index 5aefaaca5..000000000 --- a/meilisearch-http/src/routes/synonym.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::collections::BTreeMap; - -use actix_web::{web, HttpResponse}; -use actix_web::{delete, get, post}; -use indexmap::IndexMap; -use meilisearch_core::settings::{SettingsUpdate, UpdateState}; - -use crate::error::{Error, ResponseError}; -use crate::helpers::Authentication; -use crate::routes::{IndexParam, IndexUpdateResponse}; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(get).service(update).service(delete); -} - -#[get( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn get( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let reader = data.db.main_read_txn()?; - - let synonyms_list = index.main.synonyms(&reader)?; - - let mut synonyms = IndexMap::new(); - let index_synonyms = &index.synonyms; - for synonym in synonyms_list { - let list = index_synonyms.synonyms(&reader, synonym.as_bytes())?; - synonyms.insert(synonym, list); - } - - Ok(HttpResponse::Ok().json(synonyms)) -} - -#[post( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn update( - data: web::Data, - path: web::Path, - body: web::Json>>, -) -> Result { - let update_id = data.get_or_create_index(&path.index_uid, |index| { - let settings = SettingsUpdate { - synonyms: UpdateState::Update(body.into_inner()), - ..SettingsUpdate::default() - }; - - Ok(data - .db - .update_write(|w| index.settings_update(w, settings))?) - })?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} - -#[delete( - "/indexes/{index_uid}/settings/synonyms", - wrap = "Authentication::Private" -)] -async fn delete( - data: web::Data, - path: web::Path, -) -> Result { - let index = data - .db - .open_index(&path.index_uid) - .ok_or(Error::index_not_found(&path.index_uid))?; - - let settings = SettingsUpdate { - synonyms: UpdateState::Clear, - ..SettingsUpdate::default() - }; - - let update_id = data - .db - .update_write(|w| index.settings_update(w, settings))?; - - Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id))) -} diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs deleted file mode 100644 index 90db8460a..000000000 --- a/meilisearch-http/src/snapshot.rs +++ /dev/null @@ -1,106 +0,0 @@ -use crate::Data; -use crate::error::Error; -use crate::helpers::compression; - -use log::error; -use std::fs::create_dir_all; -use std::path::Path; -use std::thread; -use std::time::Duration; - -pub fn load_snapshot( - db_path: &str, - snapshot_path: &Path, - ignore_snapshot_if_db_exists: bool, - ignore_missing_snapshot: bool -) -> Result<(), Error> { - let db_path = Path::new(db_path); - - if !db_path.exists() && snapshot_path.exists() { - compression::from_tar_gz(snapshot_path, db_path) - } else if db_path.exists() && !ignore_snapshot_if_db_exists { - Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path.canonicalize().unwrap_or(db_path.into())))) - } else if !snapshot_path.exists() && !ignore_missing_snapshot { - Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path.canonicalize().unwrap_or(snapshot_path.into())))) - } else { - Ok(()) - } -} - -pub fn create_snapshot(data: &Data, snapshot_dir: impl AsRef, snapshot_name: impl AsRef) -> Result<(), Error> { - create_dir_all(&snapshot_dir)?; - let tmp_dir = tempfile::tempdir_in(&snapshot_dir)?; - - data.db.copy_and_compact_to_path(tmp_dir.path())?; - - let temp_snapshot_file = tempfile::NamedTempFile::new_in(&snapshot_dir)?; - - compression::to_tar_gz(tmp_dir.path(), temp_snapshot_file.path()) - .map_err(|e| Error::Internal(format!("something went wrong during snapshot compression: {}", e)))?; - - let snapshot_path = snapshot_dir.as_ref().join(snapshot_name.as_ref()); - - temp_snapshot_file.persist(snapshot_path).map_err(|e| Error::Internal(e.to_string()))?; - - Ok(()) -} - -pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Result<(), Error> { - if snapshot_dir.file_name().is_none() { - return Err(Error::Internal("invalid snapshot file path".to_string())); - } - let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?; - create_dir_all(snapshot_dir)?; - let snapshot_name = format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms")); - let snapshot_dir = snapshot_dir.to_owned(); - - thread::spawn(move || loop { - if let Err(e) = create_snapshot(&data, &snapshot_dir, &snapshot_name) { - error!("Unsuccessful snapshot creation: {}", e); - } - thread::sleep(Duration::from_secs(time_gap_s)); - }); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::prelude::*; - use std::fs; - - #[test] - fn test_pack_unpack() { - let tempdir = tempfile::tempdir().unwrap(); - - let test_dir = tempdir.path(); - let src_dir = test_dir.join("src"); - let dest_dir = test_dir.join("complex/destination/path/"); - let archive_path = test_dir.join("archive.snapshot"); - - let file_1_relative = Path::new("file1.txt"); - let subdir_relative = Path::new("subdir/"); - let file_2_relative = Path::new("subdir/file2.txt"); - - create_dir_all(src_dir.join(subdir_relative)).unwrap(); - fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap(); - fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap(); - - - assert!(compression::to_tar_gz(&src_dir, &archive_path).is_ok()); - assert!(archive_path.exists()); - assert!(load_snapshot(&dest_dir.to_str().unwrap(), &archive_path, false, false).is_ok()); - - assert!(dest_dir.exists()); - assert!(dest_dir.join(file_1_relative).exists()); - assert!(dest_dir.join(subdir_relative).exists()); - assert!(dest_dir.join(file_2_relative).exists()); - - let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap(); - assert_eq!(contents, "Hello_file_1"); - - let contents = fs::read_to_string(dest_dir.join(file_2_relative)).unwrap(); - assert_eq!(contents, "Hello_file_2"); - } -} diff --git a/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl index 19539cedd..7af80f342 100644 --- a/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl +++ b/meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl @@ -74,4 +74,4 @@ {"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} {"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} {"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} -{"id":77,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Red","name":"孫武","gender":"male","email":"SunTzu@chorizon.com","phone":"+1 (810) 407-3258","address":"吴國","about":"孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} +{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/meilisearch-http/tests/assets/dumps/v1/test/settings.json b/meilisearch-http/tests/assets/dumps/v1/test/settings.json index 918cfab53..c000bc7f6 100644 --- a/meilisearch-http/tests/assets/dumps/v1/test/settings.json +++ b/meilisearch-http/tests/assets/dumps/v1/test/settings.json @@ -50,7 +50,7 @@ "wolverine": ["xmen", "logan"], "logan": ["wolverine", "xmen"] }, - "attributesForFaceting": [ + "filterableAttributes": [ "gender", "color", "tags", diff --git a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl index 5bba3b9f0..9eb50e43e 100644 --- a/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl +++ b/meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl @@ -1,3 +1,2 @@ -{"status":"processed","updateId":0,"type":{"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":"Nothing","displayed_attributes":"Nothing","stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} -{"status":"processed","updateId":1,"type":{"name":"DocumentsAddition","number":77}} - +{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}} +{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} diff --git a/meilisearch-http/tests/assets/test_set.json b/meilisearch-http/tests/assets/test_set.json index cd3ed9633..63534c896 100644 --- a/meilisearch-http/tests/assets/test_set.json +++ b/meilisearch-http/tests/assets/test_set.json @@ -1590,18 +1590,18 @@ "tags": [] }, { - "id": 77, + "id": 76, "isActive": false, "balance": "$1,274.29", "picture": "http://placehold.it/32x32", "age": 25, - "color": "Red", - "name": "孫武", - "gender": "male", - "email": "SunTzu@chorizon.com", + "color": "Green", + "name": "Clarice Gardner", + "gender": "female", + "email": "claricegardner@chorizon.com", "phone": "+1 (810) 407-3258", - "address": "吴國", - "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n", + "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", + "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", "registered": "2014-10-20T10:13:32 -02:00", "latitude": 17.11935, "longitude": 65.38197, diff --git a/meilisearch-http/tests/common.rs b/meilisearch-http/tests/common.rs deleted file mode 100644 index 057baa1b9..000000000 --- a/meilisearch-http/tests/common.rs +++ /dev/null @@ -1,535 +0,0 @@ -#![allow(dead_code)] - -use actix_web::{http::StatusCode, test}; -use serde_json::{json, Value}; -use std::time::Duration; -use tempdir::TempDir; -use tokio::time::delay_for; - -use meilisearch_core::DatabaseOptions; -use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::option::Opt; - -/// Performs a search test on both post and get routes -#[macro_export] -macro_rules! test_post_get_search { - ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { - let post_query: meilisearch_http::routes::search::SearchQueryPost = - serde_json::from_str(&$query.clone().to_string()).unwrap(); - let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); - let get_query = ::serde_url_params::to_string(&get_query).unwrap(); - let ($response, $status_code) = $server.search_get(&get_query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in get route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - let ($response, $status_code) = $server.search_post($query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in post route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - }; -} - -pub struct Server { - pub uid: String, - pub data: Data, -} - -impl Server { - pub fn with_uid(uid: &str) -> Server { - let tmp_dir = TempDir::new("meilisearch").unwrap(); - - let default_db_options = DatabaseOptions::default(); - - let opt = Opt { - db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(), - dumps_dir: tmp_dir.path().join("dump"), - dump_batch_size: 16, - http_addr: "127.0.0.1:7700".to_owned(), - master_key: None, - env: "development".to_owned(), - no_analytics: true, - max_mdb_size: default_db_options.main_map_size, - max_udb_size: default_db_options.update_map_size, - http_payload_size_limit: 100000000, - ..Opt::default() - }; - - let data = Data::new(opt).unwrap(); - - Server { - uid: uid.to_string(), - data, - } - } - - pub async fn test_server() -> Self { - let mut server = Self::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - ], - }); - - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - server.add_or_replace_multiple_documents(body).await; - server - } - - pub fn data(&self) -> &Data { - &self.data - } - - pub async fn wait_update_id(&mut self, update_id: u64) { - // try 10 times to get status, or panic to not wait forever - for _ in 0..10 { - let (response, status_code) = self.get_update_status(update_id).await; - assert_eq!(status_code, 200); - - if response["status"] == "processed" || response["status"] == "failed" { - // eprintln!("{:#?}", response); - return; - } - - delay_for(Duration::from_secs(1)).await; - } - panic!("Timeout waiting for update id"); - } - - // Global Http request GET/POST/DELETE async or sync - - pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("get_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::get().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request(&self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::post() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request_async: {}", url); - - let (response, status_code) = self.post_request(url, body).await; - eprintln!("response: {}", response); - assert!(response["updateId"].as_u64().is_some()); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::put() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request_async: {}", url); - - let (response, status_code) = self.put_request(url, body).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::delete().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request_async: {}", url); - - let (response, status_code) = self.delete_request(url).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - // All Routes - - pub async fn list_indexes(&mut self) -> (Value, StatusCode) { - self.get_request("/indexes").await - } - - pub async fn create_index(&mut self, body: Value) -> (Value, StatusCode) { - self.post_request("/indexes", body).await - } - - pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/search?{}", query); - self.get_request(&url).await - } - - pub async fn get_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.get_request(&url).await - } - - pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.put_request(&url, body).await - } - - pub async fn delete_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.delete_request(&url).await - } - - pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search?{}", self.uid, query); - self.get_request(&url).await - } - - pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search", self.uid); - self.post_request(&url, body).await - } - - pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates", self.uid); - self.get_request(&url).await - } - - pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates/{}", self.uid, update_id); - self.get_request(&url).await - } - - pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.get_request(&url).await - } - - pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn add_or_replace_multiple_documents_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request(&url, body).await - } - - pub async fn add_or_update_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.put_request_async(&url, body).await; - } - - pub async fn clear_all_documents(&mut self) { - let url = format!("/indexes/{}/documents", self.uid); - self.delete_request_async(&url).await; - } - - pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.get_request(&url).await - } - - pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.delete_request_async(&url).await - } - - pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/delete-batch", self.uid); - self.post_request_async(&url, body).await - } - - pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.get_request(&url).await - } - - pub async fn update_all_settings(&mut self, body: Value) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.get_request(&url).await - } - - pub async fn update_ranking_rules(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.get_request(&url).await - } - - pub async fn update_distinct_attribute(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/primary_key", self.uid); - self.get_request(&url).await - } - - pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_searchable_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_displayed_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.get_request(&url).await - } - - pub async fn update_attributes_for_faceting(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_attributes_for_faceting_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.get_request(&url).await - } - - pub async fn update_synonyms(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.get_request(&url).await - } - - pub async fn update_stop_words(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/stats", self.uid); - self.get_request(&url).await - } - - pub async fn list_keys(&mut self) -> (Value, StatusCode) { - self.get_request("/keys").await - } - - pub async fn get_health(&mut self) -> (Value, StatusCode) { - self.get_request("/health").await - } - - pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { - self.put_request("/health", body).await - } - - pub async fn get_version(&mut self) -> (Value, StatusCode) { - self.get_request("/version").await - } - - pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info").await - } - - pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info/pretty").await - } - - pub async fn trigger_dump(&self) -> (Value, StatusCode) { - self.post_request("/dumps", Value::Null).await - } - - pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/status", dump_uid); - self.get_request(&url).await - } - - pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/import", dump_uid); - self.get_request(&url).await - } -} diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs new file mode 100644 index 000000000..7d98d0733 --- /dev/null +++ b/meilisearch-http/tests/common/index.rs @@ -0,0 +1,198 @@ +use std::time::Duration; + +use actix_web::http::StatusCode; +use paste::paste; +use serde_json::{json, Value}; +use tokio::time::sleep; + +use super::service::Service; + +macro_rules! make_settings_test_routes { + ($($name:ident),+) => { + $(paste! { + pub async fn [](&self, value: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-")); + self.service.post(url, value).await + } + + pub async fn [](&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-")); + self.service.get(url).await + } + })* + }; +} + +pub struct Index<'a> { + pub uid: String, + pub service: &'a Service, +} + +#[allow(dead_code)] +impl Index<'_> { + pub async fn get(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.service.get(url).await + } + + pub async fn load_test_set(&self) -> u64 { + let url = format!("/indexes/{}/documents", self.uid); + let (response, code) = self + .service + .post_str(url, include_str!("../assets/test_set.json")) + .await; + assert_eq!(code, 202); + let update_id = response["updateId"].as_i64().unwrap(); + self.wait_update_id(update_id as u64).await; + update_id as u64 + } + + pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) { + let body = json!({ + "uid": self.uid, + "primaryKey": primary_key, + }); + self.service.post("/indexes", body).await + } + + pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) { + let body = json!({ + "primaryKey": primary_key, + }); + let url = format!("/indexes/{}", self.uid); + + self.service.put(url, body).await + } + + pub async fn delete(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", self.uid); + self.service.delete(url).await + } + + pub async fn add_documents( + &self, + documents: Value, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { + let url = match primary_key { + Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), + None => format!("/indexes/{}/documents", self.uid), + }; + self.service.post(url, documents).await + } + + pub async fn update_documents( + &self, + documents: Value, + primary_key: Option<&str>, + ) -> (Value, StatusCode) { + let url = match primary_key { + Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key), + None => format!("/indexes/{}/documents", self.uid), + }; + self.service.put(url, documents).await + } + + pub async fn wait_update_id(&self, update_id: u64) -> Value { + // try 10 times to get status, or panic to not wait forever + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); + for _ in 0..10 { + let (response, status_code) = self.service.get(&url).await; + assert_eq!(status_code, 200, "response: {}", response); + + if response["status"] == "processed" || response["status"] == "failed" { + return response; + } + + sleep(Duration::from_secs(1)).await; + } + panic!("Timeout waiting for update id"); + } + + pub async fn get_update(&self, update_id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates/{}", self.uid, update_id); + self.service.get(url).await + } + + pub async fn list_updates(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/updates", self.uid); + self.service.get(url).await + } + + pub async fn get_document( + &self, + id: u64, + _options: Option, + ) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/{}", self.uid, id); + self.service.get(url).await + } + + pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { + let mut url = format!("/indexes/{}/documents?", self.uid); + if let Some(limit) = options.limit { + url.push_str(&format!("limit={}&", limit)); + } + + if let Some(offset) = options.offset { + url.push_str(&format!("offset={}&", offset)); + } + + if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { + url.push_str(&format!( + "attributesToRetrieve={}&", + attributes_to_retrieve.join(",") + )); + } + + self.service.get(url).await + } + + pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/{}", self.uid, id); + self.service.delete(url).await + } + + pub async fn clear_all_documents(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents", self.uid); + self.service.delete(url).await + } + + pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { + let url = format!("/indexes/{}/documents/delete-batch", self.uid); + self.service + .post(url, serde_json::to_value(&ids).unwrap()) + .await + } + + pub async fn settings(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.get(url).await + } + + pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.post(url, settings).await + } + + pub async fn delete_settings(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings", self.uid); + self.service.delete(url).await + } + + pub async fn stats(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}/stats", self.uid); + self.service.get(url).await + } + + make_settings_test_routes!(distinct_attribute); +} + +pub struct GetDocumentOptions; + +#[derive(Debug, Default)] +pub struct GetAllDocumentsOptions { + pub limit: Option, + pub offset: Option, + pub attributes_to_retrieve: Option>, +} diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs new file mode 100644 index 000000000..e734b3621 --- /dev/null +++ b/meilisearch-http/tests/common/mod.rs @@ -0,0 +1,31 @@ +pub mod index; +pub mod server; +pub mod service; + +pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; +pub use server::Server; + +/// Performs a search test on both post and get routes +#[macro_export] +macro_rules! test_post_get_search { + ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { + let post_query: meilisearch_http::routes::search::SearchQueryPost = + serde_json::from_str(&$query.clone().to_string()).unwrap(); + let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); + let get_query = ::serde_url_params::to_string(&get_query).unwrap(); + let ($response, $status_code) = $server.search_get(&get_query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in get route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + let ($response, $status_code) = $server.search_post($query).await; + let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { + panic!( + "panic in post route: {:?}", + e.downcast_ref::<&str>().unwrap() + ) + }); + }; +} diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs new file mode 100644 index 000000000..6cf1acb6a --- /dev/null +++ b/meilisearch-http/tests/common/server.rs @@ -0,0 +1,96 @@ +use std::path::Path; + +use actix_web::http::StatusCode; +use byte_unit::{Byte, ByteUnit}; +use serde_json::Value; +use tempdir::TempDir; +use urlencoding::encode; + +use meilisearch_http::data::Data; +use meilisearch_http::option::{IndexerOpts, Opt}; + +use super::index::Index; +use super::service::Service; + +pub struct Server { + pub service: Service, + // hold ownership to the tempdir while we use the server instance. + _dir: Option, +} + +impl Server { + pub async fn new() -> Self { + let dir = TempDir::new("meilisearch").unwrap(); + + let opt = default_settings(dir.path()); + + let data = Data::new(opt).unwrap(); + let service = Service(data); + + Server { + service, + _dir: Some(dir), + } + } + + pub async fn new_with_options(opt: Opt) -> Self { + let data = Data::new(opt).unwrap(); + let service = Service(data); + + Server { + service, + _dir: None, + } + } + + /// Returns a view to an index. There is no guarantee that the index exists. + pub fn index(&self, uid: impl AsRef) -> Index<'_> { + Index { + uid: encode(uid.as_ref()), + service: &self.service, + } + } + + pub async fn list_indexes(&self) -> (Value, StatusCode) { + self.service.get("/indexes").await + } + + pub async fn version(&self) -> (Value, StatusCode) { + self.service.get("/version").await + } + + pub async fn stats(&self) -> (Value, StatusCode) { + self.service.get("/stats").await + } +} + +pub fn default_settings(dir: impl AsRef) -> Opt { + Opt { + db_path: dir.as_ref().join("db"), + dumps_dir: dir.as_ref().join("dump"), + http_addr: "127.0.0.1:7700".to_owned(), + master_key: None, + env: "development".to_owned(), + #[cfg(all(not(debug_assertions), feature = "analytics"))] + no_analytics: true, + max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), + ssl_cert_path: None, + ssl_key_path: None, + ssl_auth_path: None, + ssl_ocsp_path: None, + ssl_require_auth: false, + ssl_resumption: false, + ssl_tickets: false, + import_snapshot: None, + ignore_missing_snapshot: false, + ignore_snapshot_if_db_exists: false, + snapshot_dir: ".".into(), + schedule_snapshot: false, + snapshot_interval_sec: 0, + import_dump: None, + indexer_options: IndexerOpts::default(), + log_level: "off".into(), + } +} diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs new file mode 100644 index 000000000..08db5b9dc --- /dev/null +++ b/meilisearch-http/tests/common/service.rs @@ -0,0 +1,84 @@ +use actix_web::{http::StatusCode, test}; +use serde_json::Value; + +use meilisearch_http::create_app; +use meilisearch_http::data::Data; + +pub struct Service(pub Data); + +impl Service { + pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { + let app = test::init_service(create_app!(&self.0, true)).await; + + let req = test::TestRequest::post() + .uri(url.as_ref()) + .set_json(&body) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + /// Send a test post request from a text body, with a `content-type:application/json` header. + pub async fn post_str( + &self, + url: impl AsRef, + body: impl AsRef, + ) -> (Value, StatusCode) { + let app = test::init_service(create_app!(&self.0, true)).await; + + let req = test::TestRequest::post() + .uri(url.as_ref()) + .set_payload(body.as_ref().to_string()) + .insert_header(("content-type", "application/json")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { + let app = test::init_service(create_app!(&self.0, true)).await; + + let req = test::TestRequest::get().uri(url.as_ref()).to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { + let app = test::init_service(create_app!(&self.0, true)).await; + + let req = test::TestRequest::put() + .uri(url.as_ref()) + .set_json(&body) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } + + pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { + let app = test::init_service(create_app!(&self.0, true)).await; + + let req = test::TestRequest::delete().uri(url.as_ref()).to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + + let body = test::read_body(res).await; + let response = serde_json::from_slice(&body).unwrap_or_default(); + (response, status_code) + } +} diff --git a/meilisearch-http/tests/dashboard.rs b/meilisearch-http/tests/dashboard.rs deleted file mode 100644 index 2dbaf8f7d..000000000 --- a/meilisearch-http/tests/dashboard.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn dashboard() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/bulma.min.css").await; - assert_eq!(status_code, 200); -} diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs new file mode 100644 index 000000000..66e475172 --- /dev/null +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -0,0 +1,428 @@ +use crate::common::{GetAllDocumentsOptions, Server}; +use actix_web::test; +use chrono::DateTime; +use meilisearch_http::create_app; +use serde_json::{json, Value}; + +/// This is the basic usage of our API and every other tests uses the content-type application/json +#[actix_rt::test] +async fn add_documents_test_json_content_types() { + let document = json!([ + { + "id": 1, + "content": "Bouvier Bernois", + } + ]); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "application/json")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response, json!({ "updateId": 0 })); +} + +/// no content type is still supposed to be accepted as json +#[actix_rt::test] +async fn add_documents_test_no_content_types() { + let document = json!([ + { + "id": 1, + "content": "Montagne des Pyrénées", + } + ]); + + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "application/json")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response, json!({ "updateId": 0 })); +} + +/// any other content-type is must be refused +#[actix_rt::test] +async fn add_documents_test_bad_content_types() { + let document = json!([ + { + "id": 1, + "content": "Leonberg", + } + ]); + + let server = Server::new().await; + let app = test::init_service(create_app!(&server.service.0, true)).await; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(document.to_string()) + .insert_header(("content-type", "text/plain")) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + assert_eq!(status_code, 405); + assert!(body.is_empty()); +} + +#[actix_rt::test] +async fn add_documents_no_index_creation() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); + /* + * currently we don’t check these field to stay ISO with meilisearch + * assert_eq!(response["status"], "pending"); + * assert_eq!(response["meta"]["type"], "DocumentsAddition"); + * assert_eq!(response["meta"]["format"], "Json"); + * assert_eq!(response["meta"]["primaryKey"], Value::Null); + * assert!(response.get("enqueuedAt").is_some()); + */ + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 1); + + let processed_at = + DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap(); + let enqueued_at = + DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); + assert!(processed_at > enqueued_at); + + // index was created, and primary key was infered. + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "id"); +} + +#[actix_rt::test] +async fn document_add_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn document_update_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (_response, code) = index.update_documents(json!([]), None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn document_addition_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "primary": 1, + "content": "foo", + } + ]); + let (response, code) = index.add_documents(documents, Some("primary")).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn document_update_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "primary": 1, + "content": "foo", + } + ]); + let (_response, code) = index.update_documents(documents, Some("primary")).await; + assert_eq!(code, 202); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["type"]["name"], "DocumentsPartial"); + assert_eq!(response["type"]["number"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn add_documents_with_primary_key_and_primary_key_already_exists() { + let server = Server::new().await; + let index = server.index("test"); + + index.create(Some("primary")).await; + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, Some("id")).await; + assert_eq!(code, 202); + + index.wait_update_id(0).await; + + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "failed"); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn update_documents_with_primary_key_and_primary_key_already_exists() { + let server = Server::new().await; + let index = server.index("test"); + + index.create(Some("primary")).await; + let documents = json!([ + { + "id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.update_documents(documents, Some("id")).await; + assert_eq!(code, 202); + + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + // Documents without a primary key are not accepted. + assert_eq!(response["status"], "failed"); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn replace_document() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_update_id(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + + index.wait_update_id(1).await; + + let (response, code) = index.get_update(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!(response.to_string(), r##"{"doc_id":1,"other":"bar"}"##); +} + +// test broken, see issue milli#92 +#[actix_rt::test] +#[ignore] +async fn add_no_documents() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.add_documents(json!([]), None).await; + assert_eq!(code, 200); + + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["updateId"], 0); + assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 0); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn update_document() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + + index.wait_update_id(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_update_id(1).await; + + let (response, code) = index.get_update(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); +} + +#[actix_rt::test] +async fn add_larger_dataset() { + let server = Server::new().await; + let index = server.index("test"); + let update_id = index.load_test_set().await; + let (response, code) = index.get_update(update_id).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "processed"); + assert_eq!(response["type"]["name"], "DocumentsAddition"); + assert_eq!(response["type"]["number"], 77); + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 77); +} + +#[actix_rt::test] +async fn update_larger_dataset() { + let server = Server::new().await; + let index = server.index("test"); + let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(); + index.update_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["type"]["name"], "DocumentsPartial"); + assert_eq!(response["type"]["number"], 77); + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 77); +} + +#[actix_rt::test] +async fn add_documents_bad_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "docid": "foo & bar", + "content": "foobar" + } + ]); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "failed"); +} + +#[actix_rt::test] +async fn update_documents_bad_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "docid": "foo & bar", + "content": "foobar" + } + ]); + index.update_documents(documents, None).await; + index.wait_update_id(0).await; + let (response, code) = index.get_update(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "failed"); +} diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs new file mode 100644 index 000000000..eb6fa040b --- /dev/null +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -0,0 +1,125 @@ +use serde_json::json; + +use crate::common::{GetAllDocumentsOptions, Server}; + +#[actix_rt::test] +async fn delete_one_document_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").delete_document(0).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn delete_one_unexisting_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (response, code) = index.delete_document(0).await; + assert_eq!(code, 202, "{}", response); + let update = index.wait_update_id(0).await; + assert_eq!(update["status"], "processed"); +} + +#[actix_rt::test] +async fn delete_one_document() { + let server = Server::new().await; + let index = server.index("test"); + index + .add_documents(json!([{ "id": 0, "content": "foobar" }]), None) + .await; + index.wait_update_id(0).await; + let (_response, code) = server.index("test").delete_document(0).await; + assert_eq!(code, 202); + index.wait_update_id(1).await; + + let (_response, code) = index.get_document(0, None).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn clear_all_documents_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").clear_all_documents().await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn clear_all_documents() { + let server = Server::new().await; + let index = server.index("test"); + index + .add_documents( + json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]), + None, + ) + .await; + index.wait_update_id(0).await; + let (_response, code) = index.clear_all_documents().await; + assert_eq!(code, 202); + + let _update = index.wait_update_id(1).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn clear_all_documents_empty_index() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + + let (_response, code) = index.clear_all_documents().await; + assert_eq!(code, 202); + + let _update = index.wait_update_id(0).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn delete_batch_unexisting_index() { + let server = Server::new().await; + let (response, code) = server.index("test").delete_batch(vec![]).await; + assert_eq!(code, 404, "{}", response); +} + +#[actix_rt::test] +async fn delete_batch() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; + index.wait_update_id(0).await; + let (_response, code) = index.delete_batch(vec![1, 0]).await; + assert_eq!(code, 202); + + let _update = index.wait_update_id(1).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 1); + assert_eq!(response.as_array().unwrap()[0]["id"], 3); +} + +#[actix_rt::test] +async fn delete_no_document_batch() { + let server = Server::new().await; + let index = server.index("test"); + index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await; + index.wait_update_id(0).await; + let (_response, code) = index.delete_batch(vec![]).await; + assert_eq!(code, 202, "{}", _response); + + let _update = index.wait_update_id(1).await; + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 3); +} diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs new file mode 100644 index 000000000..945bd6b5c --- /dev/null +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -0,0 +1,286 @@ +use crate::common::GetAllDocumentsOptions; +use crate::common::Server; + +use serde_json::json; + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn get_unexisting_index_single_document() { + let server = Server::new().await; + let (_response, code) = server.index("test").get_document(1, None).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn get_unexisting_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (_response, code) = index.get_document(1, None).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn get_document() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let documents = serde_json::json!([ + { + "id": 0, + "content": "foobar", + } + ]); + let (_, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + index.wait_update_id(0).await; + let (response, code) = index.get_document(0, None).await; + assert_eq!(code, 200); + assert_eq!( + response, + serde_json::json!( { + "id": 0, + "content": "foobar", + }) + ); +} + +#[actix_rt::test] +async fn get_unexisting_index_all_documents() { + let server = Server::new().await; + let (_response, code) = server + .index("test") + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn get_no_documents() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + assert_eq!(code, 200); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn get_all_documents_no_options() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + let arr = response.as_array().unwrap(); + assert_eq!(arr.len(), 20); + let first = serde_json::json!({ + "id":0, + "isActive":false, + "balance":"$2,668.55", + "picture":"http://placehold.it/32x32", + "age":36, + "color":"Green", + "name":"Lucas Hess", + "gender":"male", + "email":"lucashess@chorizon.com", + "phone":"+1 (998) 478-2597", + "address":"412 Losee Terrace, Blairstown, Georgia, 2825", + "about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", + "registered":"2016-06-21T09:30:25 -02:00", + "latitude":-44.174957, + "longitude":-145.725388, + "tags":["bug" + ,"bug"]}); + assert_eq!(first, arr[0]); +} + +#[actix_rt::test] +async fn test_get_all_documents_limit() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(5), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 5); + assert_eq!(response.as_array().unwrap()[0]["id"], 0); +} + +#[actix_rt::test] +async fn test_get_all_documents_offset() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + offset: Some(5), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response.as_array().unwrap()[0]["id"], 13); +} + +#[actix_rt::test] +async fn test_get_all_documents_attributes_to_retrieve() { + let server = Server::new().await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("name") + .is_some()); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec![]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 0 + ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["wrong"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 0 + ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["name", "tags"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 2 + ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["*"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 16 + ); + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + attributes_to_retrieve: Some(vec!["*", "wrong"]), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 16 + ); +} + +#[actix_rt::test] +async fn get_documents_displayed_attributes() { + let server = Server::new().await; + let index = server.index("test"); + index + .update_settings(json!({"displayedAttributes": ["gender"]})) + .await; + index.load_test_set().await; + + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!( + response.as_array().unwrap()[0] + .as_object() + .unwrap() + .keys() + .count(), + 1 + ); + assert!(response.as_array().unwrap()[0] + .as_object() + .unwrap() + .get("gender") + .is_some()); + + let (response, code) = index.get_document(0, None).await; + assert_eq!(code, 200); + assert_eq!(response.as_object().unwrap().keys().count(), 1); + assert!(response.as_object().unwrap().get("gender").is_some()); +} diff --git a/meilisearch-http/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs new file mode 100644 index 000000000..a791a596f --- /dev/null +++ b/meilisearch-http/tests/documents/mod.rs @@ -0,0 +1,3 @@ +mod add_documents; +mod delete_documents; +mod get_documents; diff --git a/meilisearch-http/tests/documents_add.rs b/meilisearch-http/tests/documents_add.rs deleted file mode 100644 index 382a1ed43..000000000 --- a/meilisearch-http/tests/documents_add.rs +++ /dev/null @@ -1,222 +0,0 @@ -use serde_json::json; - -mod common; - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 -#[actix_rt::test] -async fn check_add_documents_with_primary_key_param() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 -#[actix_rt::test] -async fn check_add_documents_with_nested_boolean() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a boolean in a nested object - - let body = json!([{ - "id": 12161, - "created_at": "2019-04-10T14:57:57.522Z", - "foo": { - "bar": { - "id": 121, - "crash": false - }, - "id": 45912 - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 -#[actix_rt::test] -async fn check_add_documents_with_nested_null() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a null in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": null - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 -#[actix_rt::test] -async fn check_add_documents_with_nested_sequence() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a seq in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": [123,456], - "fez": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }], - "foz": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }, - { - "id": 256, - "baz": "loss", - "fuzz": { - "fax": [235] - }, - "sas": [321, 321] - }] - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body.clone()).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); - - let url = "/indexes/tasks/search?q=leesz"; - let (response, status_code) = server.get_request(&url).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"], body); -} - -#[actix_rt::test] -// test sample from #807 -async fn add_document_with_long_field() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let body = json!([{ - "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", - "rank":1, - "relurl":"/configuration/app/web.html#locations", - "section":"Web", - "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", - "title":"Locations", - "url":"/configuration/app/web.html#locations" - }]); - server.add_or_replace_multiple_documents(body).await; - let (response, _status) = server - .search_post(json!({ "q": "request_buffering" })) - .await; - assert!(!response["hits"].as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn documents_with_same_id_are_overwritten() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test"})).await; - let documents = json!([ - { - "id": 1, - "content": "test1" - }, - { - "id": 1, - "content": "test2" - }, - ]); - server.add_or_replace_multiple_documents(documents).await; - let (response, _status) = server.get_all_documents().await; - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!( - response.as_array().unwrap()[0].as_object().unwrap()["content"], - "test2" - ); -} diff --git a/meilisearch-http/tests/documents_delete.rs b/meilisearch-http/tests/documents_delete.rs deleted file mode 100644 index 4353a5355..000000000 --- a/meilisearch-http/tests/documents_delete.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod common; - -use serde_json::json; - -#[actix_rt::test] -async fn delete() { - let mut server = common::Server::test_server().await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 200); - - server.delete_document(50).await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 404); -} - -// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 -#[actix_rt::test] -async fn delete_batch() { - let mut server = common::Server::test_server().await; - - let doc_ids = vec!(50, 55, 60); - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 200); - } - - let body = serde_json::json!(&doc_ids); - server.delete_multiple_documents(body).await; - - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 404); - } -} - -#[actix_rt::test] -async fn text_clear_all_placeholder_search() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - }); - - server.update_all_settings(settings).await; - - let documents = json!([ - { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, - { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, - { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, - { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, - { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, - { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } - ]); - - server.add_or_update_multiple_documents(documents).await; - server.clear_all_documents().await; - let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; - assert_eq!(response["nbHits"], 0); - let (response, _) = server.search_post(json!({ "q": "" })).await; - assert_eq!(response["nbHits"], 0); -} diff --git a/meilisearch-http/tests/documents_get.rs b/meilisearch-http/tests/documents_get.rs deleted file mode 100644 index 35e04f494..000000000 --- a/meilisearch-http/tests/documents_get.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde_json::json; -use actix_web::http::StatusCode; - -mod common; - -#[actix_rt::test] -async fn get_documents_from_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); - assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); -} - -#[actix_rt::test] -async fn get_empty_documents_list() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::OK); - assert!(response.as_array().unwrap().is_empty()); -} diff --git a/meilisearch-http/tests/dump.rs b/meilisearch-http/tests/dump.rs deleted file mode 100644 index e50be866a..000000000 --- a/meilisearch-http/tests/dump.rs +++ /dev/null @@ -1,372 +0,0 @@ -use assert_json_diff::{assert_json_eq, assert_json_include}; -use meilisearch_http::helpers::compression; -use serde_json::{json, Value}; -use std::fs::File; -use std::path::Path; -use std::thread; -use std::time::Duration; -use tempfile::TempDir; - -#[macro_use] mod common; - -async fn trigger_and_wait_dump(server: &mut common::Server) -> String { - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - for _ in 0..20_u8 { - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - assert_ne!(value["status"].as_str(), Some("dump_process_failed")); - - if value["status"].as_str() == Some("done") { return dump_uid } - thread::sleep(Duration::from_millis(100)); - } - - unreachable!("dump creation runned out of time") -} - -fn current_db_version() -> (String, String, String) { - let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); - let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); - let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); - - (current_version_major, current_version_minor, current_version_patch) -} - -fn current_dump_version() -> String { - "V1".into() -} - -fn read_all_jsonline(r: R) -> Value { - let deserializer = serde_json::Deserializer::from_reader(r); let iterator = deserializer.into_iter::(); - - json!(iterator.map(|v| v.unwrap()).collect::>()) -} - -#[actix_rt::test] -async fn trigger_dump_should_return_ok() { - let server = common::Server::test_server().await; - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); -} - -#[actix_rt::test] -async fn trigger_dump_twice_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let (value, status_code) = server.trigger_dump().await; - - - assert_json_eq!(expected, value, ordered: false); - assert_eq!(status_code, 409); -} - -#[actix_rt::test] -async fn trigger_dump_concurently_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); - - assert_json_eq!(expected, value_2, ordered: false); - assert_eq!(status_code_2, 409); -} - -#[actix_rt::test] -async fn get_dump_status_early_should_return_in_progress() { - let mut server = common::Server::test_server().await; - - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - let expected = json!({ - "uid": dump_uid, - "status": "in_progress" - }); - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -async fn get_dump_status_should_return_done() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "done" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -async fn get_dump_status_should_return_error_provoking_it() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - // removing destination directory provoking `No such file or directory` error - std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "failed", - "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", - "errorCode": "dump_process_failed", - "errorType": "internal_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -async fn dump_metadata_should_be_valid() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "uid": "test2", - "primaryKey": "test2_id", - }); - - server.create_index(body).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); - let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); - - // fields are randomly ordered - metadata.get_mut("indexes").unwrap() - .as_array_mut().unwrap() - .sort_by(|a, b| - a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) - ); - - let (major, minor, patch) = current_db_version(); - - let expected = json!({ - "indexes": [{ - "uid": "test", - "primaryKey": "id", - }, { - "uid": "test2", - "primaryKey": "test2_id", - } - ], - "dbVersion": format!("{}.{}.{}", major, minor, patch), - "dumpVersion": current_dump_version() - }); - - assert_json_include!(expected: expected, actual: metadata); -} - -#[actix_rt::test] -async fn dump_gzip_should_have_been_created() { - let mut server = common::Server::test_server().await; - - - let dump_uid = trigger_and_wait_dump(&mut server).await; - let dumps_dir = Path::new(&server.data().dumps_dir); - - let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); - assert!(File::open(compressed_path).is_ok()); -} - -#[actix_rt::test] -async fn dump_index_settings_should_be_valid() { - let mut server = common::Server::test_server().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags" - ] - }); - - server.update_all_settings(expected.clone()).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); - let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); - - assert_json_eq!(expected, settings, ordered: false); -} - -#[actix_rt::test] -async fn dump_index_documents_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); - let documents = read_all_jsonline(file); - - assert_json_eq!(expected, documents, ordered: false); -} - -#[actix_rt::test] -async fn dump_index_updates_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); - let updates = read_all_jsonline(file); - - eprintln!("{}\n", updates); - eprintln!("{}", expected); - assert_json_include!(expected: expected, actual: updates); -} - -#[actix_rt::test] -async fn get_unexisting_dump_status_should_return_not_found() { - let mut server = common::Server::test_server().await; - - let (_, status_code) = server.get_dump_status("4242").await; - - assert_eq!(status_code, 404); -} diff --git a/meilisearch-http/tests/errors.rs b/meilisearch-http/tests/errors.rs deleted file mode 100644 index 2aac614f5..000000000 --- a/meilisearch-http/tests/errors.rs +++ /dev/null @@ -1,200 +0,0 @@ -mod common; - -use std::thread; -use std::time::Duration; - -use actix_http::http::StatusCode; -use serde_json::{json, Map, Value}; - -macro_rules! assert_error { - ($code:literal, $type:literal, $status:path, $req:expr) => { - let (response, status_code) = $req; - assert_eq!(status_code, $status); - assert_eq!(response["errorCode"].as_str().unwrap(), $code); - assert_eq!(response["errorType"].as_str().unwrap(), $type); - }; -} - -macro_rules! assert_error_async { - ($code:literal, $type:literal, $server:expr, $req:expr) => { - let (response, _) = $req; - let update_id = response["updateId"].as_u64().unwrap(); - for _ in 1..10 { - let (response, status_code) = $server.get_update_status(update_id).await; - assert_eq!(status_code, StatusCode::OK); - if response["status"] == "processed" || response["status"] == "failed" { - println!("response: {}", response); - assert_eq!(response["status"], "failed"); - assert_eq!(response["errorCode"], $code); - assert_eq!(response["errorType"], $type); - return - } - thread::sleep(Duration::from_secs(1)); - } - }; -} - -#[actix_rt::test] -async fn index_already_exists_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test" - }); - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - assert_eq!(status_code, StatusCode::CREATED); - - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - - assert_error!( - "index_already_exists", - "invalid_request_error", - StatusCode::BAD_REQUEST, - (response, status_code)); -} - -#[actix_rt::test] -async fn index_not_found_error() { - let mut server = common::Server::with_uid("test"); - assert_error!( - "index_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_index().await); -} - -#[actix_rt::test] -async fn primary_key_already_present_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body.clone()).await; - let body = json!({ - "primaryKey": "t" - }); - assert_error!( - "primary_key_already_present", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.update_index(body).await); -} - -#[actix_rt::test] -async fn max_field_limit_exceeded_error() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - let mut doc = Map::with_capacity(70_000); - doc.insert("id".into(), Value::String("foo".into())); - for i in 0..69_999 { - doc.insert(format!("field{}", i), Value::String("foo".into())); - } - let docs = json!([doc]); - assert_error_async!( - "max_fields_limit_exceeded", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn missing_document_id() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body).await; - let docs = json!([ - { - "foo": "bar", - } - ]); - assert_error_async!( - "missing_document_id", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn facet_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "facetFilters": ["test:hello"] - }); - assert_error!( - "invalid_facet", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn filters_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "filters": "fo:12" - }); - assert_error!( - "invalid_filter", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn bad_request_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "foo": "bar", - }); - assert_error!( - "bad_request", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(body).await); -} - -#[actix_rt::test] -async fn document_not_found_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - assert_error!( - "document_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_document(100).await); -} - -#[actix_rt::test] -async fn payload_too_large_error() { - let mut server = common::Server::with_uid("test"); - let bigvec = vec![0u64; 100_000_000]; // 800mb - assert_error!( - "payload_too_large", - "invalid_request_error", - StatusCode::PAYLOAD_TOO_LARGE, - server.create_index(json!(bigvec)).await); -} - -#[actix_rt::test] -async fn missing_primary_key_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - let document = json!([{ - "content": "test" - }]); - assert_error!( - "missing_primary_key", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.add_or_replace_multiple_documents_sync(document).await); -} diff --git a/meilisearch-http/tests/health.rs b/meilisearch-http/tests/health.rs deleted file mode 100644 index 2be66887f..000000000 --- a/meilisearch-http/tests/health.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn test_healthyness() { - let mut server = common::Server::with_uid("movies"); - - // Check that the server is healthy - - let (response, status_code) = server.get_health().await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "available"); -} diff --git a/meilisearch-http/tests/index.rs b/meilisearch-http/tests/index.rs deleted file mode 100644 index 050ffe813..000000000 --- a/meilisearch-http/tests/index.rs +++ /dev/null @@ -1,811 +0,0 @@ -use actix_web::http::StatusCode; -use assert_json_diff::assert_json_eq; -use serde_json::{json, Value}; - -mod common; - -#[actix_rt::test] -async fn create_index_with_name() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body.clone()).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid, "movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 1.5 verify that error is thrown when trying to create the same index - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - assert_eq!( - response["errorCode"].as_str().unwrap(), - "index_already_exists" - ); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_name_and_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "Films", - "uid": "fr_movies", - }); - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "Films"); - assert_eq!(r1_uid, "fr_movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn rename_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Update an index name - - let body = json!({ - "name": "TV Shows", - }); - - let (res2_value, status_code) = server.update_index(body).await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_object().unwrap().len(), 5); - let r2_name = res2_value["name"].as_str().unwrap(); - let r2_uid = res2_value["uid"].as_str().unwrap(); - let r2_created_at = res2_value["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, "TV Shows"); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at, r1_created_at); - assert!(r2_updated_at.len() > 1); - - // 3 - Check the list of indexes - - let (res3_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res3_value.as_array().unwrap().len(), 1); - assert_eq!(res3_value[0].as_object().unwrap().len(), 5); - let r3_name = res3_value[0]["name"].as_str().unwrap(); - let r3_uid = res3_value[0]["uid"].as_str().unwrap(); - let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, r2_name); - assert_eq!(r3_uid.len(), r1_uid.len()); - assert_eq!(r3_created_at.len(), r1_created_at.len()); - assert_eq!(r3_updated_at.len(), r2_updated_at.len()); -} - -#[actix_rt::test] -async fn delete_index_and_recreate_it() { - let mut server = common::Server::with_uid("movies"); - - // 0 - delete unexisting index is error - - let (response, status_code) = server.delete_request("/indexes/test").await; - assert_eq!(status_code, 404); - assert_eq!(&response["errorCode"], "index_not_found"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); - - // 3- Delete an index - - let (_res2_value, status_code) = server.delete_index().await; - - assert_eq!(status_code, 204); - - // 4 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 0); - - // 5 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 6 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn check_multiples_indexes() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_0_name = res2_value[0]["name"].as_str().unwrap(); - let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_0_name, r1_name); - assert_eq!(r2_0_uid.len(), r1_uid.len()); - assert_eq!(r2_0_created_at.len(), r1_created_at.len()); - assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); - - // 3 - Create a new index - - let body = json!({ - "name": "films", - }); - - let (res3_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res3_value.as_object().unwrap().len(), 5); - let r3_name = res3_value["name"].as_str().unwrap(); - let r3_uid = res3_value["uid"].as_str().unwrap(); - let r3_created_at = res3_value["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, "films"); - assert_eq!(r3_uid.len(), 8); - assert!(r3_created_at.len() > 1); - assert!(r3_updated_at.len() > 1); - - // 4 - Check the list of indexes - - let (res4_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res4_value.as_array().unwrap().len(), 2); - assert_eq!(res4_value[0].as_object().unwrap().len(), 5); - let r4_0_name = res4_value[0]["name"].as_str().unwrap(); - let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); - let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); - let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(res4_value[1].as_object().unwrap().len(), 5); - let r4_1_name = res4_value[1]["name"].as_str().unwrap(); - let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); - let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); - let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); - if r4_0_name == r1_name { - assert_eq!(r4_0_name, r1_name); - assert_eq!(r4_0_uid.len(), r1_uid.len()); - assert_eq!(r4_0_created_at.len(), r1_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_0_name, r3_name); - assert_eq!(r4_0_uid.len(), r3_uid.len()); - assert_eq!(r4_0_created_at.len(), r3_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); - } - if r4_1_name == r1_name { - assert_eq!(r4_1_name, r1_name); - assert_eq!(r4_1_uid.len(), r1_uid.len()); - assert_eq!(r4_1_created_at.len(), r1_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_1_name, r3_name); - assert_eq!(r4_1_uid.len(), r3_uid.len()); - assert_eq!(r4_1_created_at.len(), r3_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); - } -} - -#[actix_rt::test] -async fn create_index_failed() { - let mut server = common::Server::with_uid("movies"); - - // 2 - Push index creation with empty json body - - let body = json!({}); - - let (res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = res_value["message"].as_str().unwrap(); - assert_eq!(res_value.as_object().unwrap().len(), 4); - assert_eq!(message, "Index creation must have an uid"); - - // 3 - Create a index with extra data - - let body = json!({ - "name": "movies", - "active": true - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - - // 3 - Create a index with wrong data type - - let body = json!({ - "name": "movies", - "uid": 0 - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 -#[actix_rt::test] -async fn create_index_with_primary_key_and_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - - let (_response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - - // 2 - Add content - - let body = json!([{ - "id": 123, - "text": "The mask" - }]); - - server.add_or_replace_multiple_documents(body.clone()).await; - - // 3 - Retreive document - - let (response, _status_code) = server.get_document(123).await; - - let expect = json!({ - "id": 123, - "text": "The mask" - }); - - assert_json_eq!(response, expect, ordered: false); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 -// Test when the given index uid is not valid -// Should have a 400 status code -// Should have the right error message -#[actix_rt::test] -async fn create_index_with_invalid_uid() { - let mut server = common::Server::with_uid(""); - - // 1 - Create the index with invalid uid - - let body = json!({ - "uid": "the movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 2 - Create the index with invalid uid - - let body = json!({ - "uid": "%$#" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 3 - Create the index with invalid uid - - let body = json!({ - "uid": "the~movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 4 - Create the index with invalid uid - - let body = json!({ - "uid": "🎉" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); -} - -// Test that it's possible to add primary_key if it's not already set on index creation -#[actix_rt::test] -async fn create_index_and_add_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "id", - }); - - let (response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 200); - eprintln!("response: {:#?}", response); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that it's impossible to change the primary_key -#[actix_rt::test] -async fn create_index_and_update_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "skuid", - }); - - let (_response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 400); - - // 3 - Get index to verify if the primary_key still the first one - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that schema inference work well -#[actix_rt::test] -async fn create_index_without_primary_key_and_add_document() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document - - let body = json!([{ - "id": 123, - "title": "I'm a legend", - }]); - - server.add_or_update_multiple_documents(body).await; - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test search with no primary_key -#[actix_rt::test] -async fn create_index_without_primary_key_and_search() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Search - - let query = "q=captain&limit=3"; - - let (response, status_code) = server.search_get(&query).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 0); -} - -// Test the error message when we push an document update and impossibility to find primary key -// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 -#[actix_rt::test] -async fn check_add_documents_without_primary_key() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2- Add document - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; - - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(response["errorCode"], "missing_primary_key"); - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("movies"); - - let body = json!({ - "uid": "movies", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("./assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn get_empty_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.list_indexes().await; - assert!(response.as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn create_and_list_multiple_indices() { - let mut server = common::Server::with_uid("test"); - for i in 0..10 { - server - .create_index(json!({ "uid": format!("test{}", i) })) - .await; - } - let (response, _status) = server.list_indexes().await; - assert_eq!(response.as_array().unwrap().len(), 10); -} - -#[actix_rt::test] -async fn get_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_index().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn create_index_twice_is_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "index_already_exists"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn badly_formatted_index_name_is_error() { - let mut server = common::Server::with_uid("$__test"); - let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "invalid_index_uid"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn correct_response_no_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(response["primaryKey"], Value::Null); -} - -#[actix_rt::test] -async fn correct_response_with_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server - .create_index(json!({ "uid": "test", "primaryKey": "test" })) - .await; - assert_eq!(response["primaryKey"], "test"); -} - -#[actix_rt::test] -async fn udpate_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn update_existing_primary_key_is_error() { - let mut server = common::Server::with_uid("test"); - server - .create_index(json!({ "uid": "test", "primaryKey": "key" })) - .await; - let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "primary_key_already_present"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn test_field_distribution_attribute() { - let mut server = common::Server::test_server().await; - - let (response, _status_code) = server.get_index_stats().await; - - let expected = json!({ - "fieldsDistribution": { - "about": 77, - "address": 77, - "age": 77, - "balance": 77, - "color": 77, - "email": 77, - "gender": 77, - "id": 77, - "isActive": 77, - "latitude": 77, - "longitude": 77, - "name": 77, - "phone": 77, - "picture": 77, - "registered": 77, - "tags": 77 - }, - "isIndexing": false, - "numberOfDocuments": 77 - }); - - assert_json_eq!(expected, response, ordered: true); -} diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs new file mode 100644 index 000000000..e65908cb2 --- /dev/null +++ b/meilisearch-http/tests/index/create_index.rs @@ -0,0 +1,74 @@ +use crate::common::Server; +use serde_json::Value; + +#[actix_rt::test] +async fn create_index_no_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], Value::Null); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +#[actix_rt::test] +async fn create_index_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + //assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], "primary"); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn create_existing_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + + let (_response, code) = index.create(Some("primary")).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn create_with_invalid_index_uid() { + let server = Server::new().await; + let index = server.index("test test#!"); + let (_, code) = index.create(None).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn test_create_multiple_indexes() { + let server = Server::new().await; + let index1 = server.index("test1"); + let index2 = server.index("test2"); + let index3 = server.index("test3"); + let index4 = server.index("test4"); + + index1.create(None).await; + index2.create(None).await; + index3.create(None).await; + + assert_eq!(index1.get().await.1, 200); + assert_eq!(index2.get().await.1, 200); + assert_eq!(index3.get().await.1, 200); + assert_eq!(index4.get().await.1, 404); +} diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs new file mode 100644 index 000000000..b0f067b24 --- /dev/null +++ b/meilisearch-http/tests/index/delete_index.rs @@ -0,0 +1,25 @@ +use crate::common::Server; + +#[actix_rt::test] +async fn create_and_delete_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (_response, code) = index.delete().await; + + assert_eq!(code, 204); + + assert_eq!(index.get().await.1, 404); +} + +#[actix_rt::test] +async fn delete_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.delete().await; + + assert_eq!(code, 404); +} diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs new file mode 100644 index 000000000..a6b22509e --- /dev/null +++ b/meilisearch-http/tests/index/get_index.rs @@ -0,0 +1,62 @@ +use crate::common::Server; +use serde_json::Value; + +#[actix_rt::test] +async fn create_and_get_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (response, code) = index.get().await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + assert_eq!(response["createdAt"], response["updatedAt"]); + assert_eq!(response["primaryKey"], Value::Null); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +// TODO: partial test since we are testing error, and error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn get_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, code) = index.get().await; + + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn no_index_return_empty_list() { + let server = Server::new().await; + let (response, code) = server.list_indexes().await; + assert_eq!(code, 200); + assert!(response.is_array()); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn list_multiple_indexes() { + let server = Server::new().await; + server.index("test").create(None).await; + server.index("test1").create(Some("key")).await; + + let (response, code) = server.list_indexes().await; + assert_eq!(code, 200); + assert!(response.is_array()); + let arr = response.as_array().unwrap(); + assert_eq!(arr.len(), 2); + assert!(arr + .iter() + .any(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null)); + assert!(arr + .iter() + .any(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key")); +} diff --git a/meilisearch-http/tests/index/mod.rs b/meilisearch-http/tests/index/mod.rs new file mode 100644 index 000000000..9996df2e7 --- /dev/null +++ b/meilisearch-http/tests/index/mod.rs @@ -0,0 +1,5 @@ +mod create_index; +mod delete_index; +mod get_index; +mod stats; +mod update_index; diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs new file mode 100644 index 000000000..8494bbae3 --- /dev/null +++ b/meilisearch-http/tests/index/stats.rs @@ -0,0 +1,48 @@ +use serde_json::json; + +use crate::common::Server; + +#[actix_rt::test] +async fn stats() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("id")).await; + + assert_eq!(code, 200); + + let (response, code) = index.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["numberOfDocuments"], 0); + assert!(response["isIndexing"] == false); + assert!(response["fieldDistribution"] + .as_object() + .unwrap() + .is_empty()); + + let documents = json!([ + { + "id": 1, + "name": "Alexey", + }, + { + "id": 2, + "age": 45, + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); + + index.wait_update_id(0).await; + + let (response, code) = index.stats().await; + + assert_eq!(code, 200); + assert_eq!(response["numberOfDocuments"], 2); + assert!(response["isIndexing"] == false); + assert_eq!(response["fieldDistribution"]["id"], 2); + assert_eq!(response["fieldDistribution"]["name"], 1); + assert_eq!(response["fieldDistribution"]["age"], 1); +} diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs new file mode 100644 index 000000000..c7d910b59 --- /dev/null +++ b/meilisearch-http/tests/index/update_index.rs @@ -0,0 +1,64 @@ +use crate::common::Server; +use chrono::DateTime; + +#[actix_rt::test] +async fn update_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (response, code) = index.update(Some("primary")).await; + + assert_eq!(code, 200); + assert_eq!(response["uid"], "test"); + assert_eq!(response["name"], "test"); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + + let created_at = DateTime::parse_from_rfc3339(response["createdAt"].as_str().unwrap()).unwrap(); + let updated_at = DateTime::parse_from_rfc3339(response["updatedAt"].as_str().unwrap()).unwrap(); + assert!(created_at < updated_at); + + assert_eq!(response["primaryKey"], "primary"); + assert_eq!(response.as_object().unwrap().len(), 5); +} + +#[actix_rt::test] +async fn update_nothing() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + + assert_eq!(code, 200); + + let (update, code) = index.update(None).await; + + assert_eq!(code, 200); + assert_eq!(response, update); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn update_existing_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.create(Some("primary")).await; + + assert_eq!(code, 200); + + let (_update, code) = index.update(Some("primary2")).await; + + assert_eq!(code, 400); +} + +// TODO: partial test since we are testing error, amd error is not yet fully implemented in +// transplant +#[actix_rt::test] +async fn test_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").update(None).await; + assert_eq!(code, 404); +} diff --git a/meilisearch-http/tests/index_update.rs b/meilisearch-http/tests/index_update.rs deleted file mode 100644 index 4d7e025a6..000000000 --- a/meilisearch-http/tests/index_update.rs +++ /dev/null @@ -1,208 +0,0 @@ -use serde_json::json; -use serde_json::Value; -use assert_json_diff::assert_json_include; - -mod common; - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn return_error_when_get_update_status_of_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - // 1. Fetch the status of unexisting index. - let (_, status_code) = server.get_all_updates_status().await; - - // 2. Verify the fetch returned 404 - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn return_empty_when_get_update_status_of_empty_index() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2. Fetch the status of empty index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and no document are returned - assert_eq!(status_code, 200); - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn return_update_status_of_pushed_documents() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - - let bodies = vec![ - json!([{ - "title": "Test", - "comment": "comment test" - }]), - json!([{ - "title": "Test1", - "comment": "comment test1" - }]), - json!([{ - "title": "Test2", - "comment": "comment test2" - }]), - ]; - - let mut update_ids = Vec::new(); - let mut bodies = bodies.into_iter(); - - let url = "/indexes/test/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, bodies.next().unwrap()).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - update_ids.push(update_id); - server.wait_update_id(update_id).await; - - let url = "/indexes/test/documents"; - for body in bodies { - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - update_ids.push(update_id); - } - - // 2. Fetch the status of index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and updates are returned - - let expected = json!([{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[0] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[1] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[2] - },]); - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} - -#[actix_rt::test] -async fn return_error_if_index_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "index_not_found"); -} - -#[actix_rt::test] -async fn return_error_if_update_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "not_found"); -} - -#[actix_rt::test] -async fn should_return_existing_update() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/test/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - - let update_id = response["updateId"].as_u64().unwrap(); - - let expected = json!({ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_id - }); - - let (response, status_code) = server.get_update_status(update_id).await; - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs new file mode 100644 index 000000000..b414072d4 --- /dev/null +++ b/meilisearch-http/tests/integration.rs @@ -0,0 +1,14 @@ +mod common; +mod documents; +mod index; +mod search; +mod settings; +mod snapshot; +mod stats; +mod updates; + +// Tests are isolated by features in different modules to allow better readability, test +// targetability, and improved incremental compilation times. +// +// All the integration tests live in the same root module so only one test executable is generated, +// thus improving linking time. diff --git a/meilisearch-http/tests/lazy_index_creation.rs b/meilisearch-http/tests/lazy_index_creation.rs deleted file mode 100644 index 6730db82e..000000000 --- a/meilisearch-http/tests/lazy_index_creation.rs +++ /dev/null @@ -1,446 +0,0 @@ -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_and_discover_pk() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "id": 1, - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_with_wrong_name() { - let server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); -} - -#[actix_rt::test] -async fn create_index_lazy_add_documents_failed() { - let mut server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); - - let (_, status_code) = server.get_index().await; - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "other", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "anotherSettings": ["name"], - }); - - let (_, status_code) = server.update_all_settings_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": 123, - }); - - let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!("type"); - - server.update_distinct_attribute(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (resp, status_code) = server.get_all_settings().await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_searchable_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_displayed_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_attributes_for_faceting(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server - .update_attributes_for_faceting_sync(body.clone()) - .await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "road": ["street", "avenue"], - "street": ["avenue"], - }); - - server.update_synonyms(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_synonyms_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["le", "la", "les"]); - - server.update_stop_words(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_stop_words_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} diff --git a/meilisearch-http/tests/placeholder_search.rs b/meilisearch-http/tests/placeholder_search.rs deleted file mode 100644 index 048ab7f8b..000000000 --- a/meilisearch-http/tests/placeholder_search.rs +++ /dev/null @@ -1,629 +0,0 @@ -use std::convert::Into; - -use serde_json::json; -use serde_json::Value; -use std::cell::RefCell; -use std::sync::Mutex; - -#[macro_use] -mod common; - -#[actix_rt::test] -async fn placeholder_search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 6, - }); - - // hack to take a value out of macro (must implement UnwindSafe) - let expected = Mutex::new(RefCell::new(Vec::new())); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - // take results at offset 3 as reference - let lock = expected.lock().unwrap(); - lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); - }); - let expected = expected.into_inner().unwrap().into_inner(); - - let query = json!({ - "limit": 3, - "offset": 3, - }); - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let response = response["hits"].as_array().unwrap(); - assert_eq!(&expected, response); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attribute_to_highlight_wildcard() { - // there should be no highlight in placeholder search - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToHighlight": ["*"] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - for value in result.values() { - assert!(value.to_string().find("").is_none()); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_matches() { - // matches is always empty - let mut server = common::Server::test_server().await; - - let query = json!({ - "matches": true - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) - .all(|m| m.as_object().unwrap().is_empty()); - assert!(result); - }); -} - -#[actix_rt::test] -async fn placeholder_search_witch_crop() { - // placeholder search crop always crop from beggining - let mut server = common::Server::test_server().await; - - let query = json!({ - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - - let hits = response["hits"].as_array().unwrap(); - - for hit in hits { - let hit = hit.as_object().unwrap(); - let formatted = hit["_formatted"].as_object().unwrap(); - - let about = hit["about"].as_str().unwrap(); - let about_formatted = formatted["about"].as_str().unwrap(); - // the formatted about length should be about 20 characters long - assert!(about_formatted.len() < 20 + 10); - // the formatted part should be located at the beginning of the original one - assert_eq!(about.find(&about_formatted).unwrap(), 0); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToRetrieve": ["gender", "about"], - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - assert_eq!(hit.values().count(), 2); - let _ = hit["gender"]; - let _ = hit["about"]; - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "filters": "color='green'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); - }); - - let query = json!({ - "filters": "tags=bug" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let value = Value::String(String::from("bug")); - assert!(hits - .iter() - .all(|v| v["tags"].as_array().unwrap().contains(&value))); - }); - - let query = json!({ - "filters": "color='green' AND (tags='bug' OR tags='wontfix')" - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let bug = Value::String(String::from("bug")); - let wontfix = Value::String(String::from("wontfix")); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" - && v["tags"].as_array().unwrap().contains(&bug) - || v["tags"].as_array().unwrap().contains(&wontfix))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_valid() { - let mut server = common::Server::test_server().await; - - // simple tests on attributes with string value - let body = json!({ - "attributesForFaceting": ["color"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - && value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green"))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "facetFilters": ["color:blue"] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "facetFilters": [] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [[]] - let query = json!({ - "facetFilters": [[]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // ["color:green", []] - let query = json!({ - "facetFilters": ["color:green", []] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - // too much depth - // [[[]]] - let query = json!({ - "facetFilters": [[[]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [["color:green", ["color:blue"]]] - let query = json!({ - "facetFilters": [["color:green", ["color:blue"]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // "color:green" - let query = json!({ - "facetFilters": "color:green" - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); -} - -#[actix_rt::test] -async fn placeholder_test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code| { - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code| { - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 1 - ); - }); - // searching on color and tags - let query = json!({ - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let facets = response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!( - !facets - .get("color") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - assert_ne!( - !facets - .get("tags") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - // wildcard - let query = json!({ - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - // wildcard with other attributes: - let query = json!({ - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - - // empty facet list - let query = json!({ - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - - // attr not set as facet passed: - let query = json!({ - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code| { - assert_eq!(status_code, 400); - }); -} - -#[actix_rt::test] -#[should_panic] -async fn placeholder_test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn placeholder_test_sort() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": ["asc(age)"], - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); - - let query = json!({ - "facetFilters": ["color:green"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_empty_query() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "", - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - eprintln!("{}", response); - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_placeholder() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 3); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/meilisearch-http/tests/search.rs b/meilisearch-http/tests/search.rs deleted file mode 100644 index 13dc4c898..000000000 --- a/meilisearch-http/tests/search.rs +++ /dev/null @@ -1,1976 +0,0 @@ -use std::convert::Into; - -use assert_json_diff::assert_json_eq; -use serde_json::json; -use serde_json::Value; - -#[macro_use] mod common; - -#[actix_rt::test] -async fn search() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let hits: Vec = hits.iter().cloned().take(3).collect(); - assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_no_params() { - let mut server = common::Server::test_server().await; - - let query = json! ({}); - - // an empty search should return the 20 first indexed document - let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); - let expected: Vec = dataset.into_iter().take(20).collect(); - let expected: Value = serde_json::to_value(expected).unwrap(); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_in_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json! ({ - "message": "Index test not found", - "errorCode": "index_not_found", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#index_not_found" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(404, status_code); - assert_json_eq!(expected.clone(), response.clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_unexpected_params() { - - let query = json! ({"lol": "unexpected"}); - - let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; - - let post_query = serde_json::from_str::(&query.to_string()); - assert!(post_query.is_err()); - assert_eq!(expected, post_query.err().unwrap().to_string()); - - let get_query: Result = serde_json::from_str(&query.to_string()); - assert!(get_query.is_err()); - assert_eq!(expected, get_query.err().unwrap().to_string()); -} - -#[actix_rt::test] -async fn search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation", - "limit": 3 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "offset": 1 - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["*"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_wildcard_chinese() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "子孙", - "limit": 1, - "attributesToHighlight": ["*"] - }); - - let expected = json!([ - { - "id": 77, - "isActive": false, - "balance": "$1,274.29", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "Red", - "name": "孫武", - "gender": "male", - "email": "SunTzu@chorizon.com", - "phone": "+1 (810) 407-3258", - "address": "吴國", - "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n", - "registered": "2014-10-20T10:13:32 -02:00", - "latitude": 17.11935, - "longitude": 65.38197, - "tags": [ - "new issue", - "wontfix" - ], - "_formatted": { - "id": 77, - "isActive": false, - "balance": "$1,274.29", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "Red", - "name": "孫武", - "gender": "male", - "email": "SunTzu@chorizon.com", - "phone": "+1 (810) 407-3258", - "address": "吴國", - "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n", - "registered": "2014-10-20T10:13:32 -02:00", - "latitude": 17.11935, - "longitude": 65.38197, - "tags": [ - "new issue", - "wontfix" - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_1() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "matches": true - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_matchesInfo": { - "name": [ - { - "start": 0, - "length": 6 - } - ], - "email": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","color","gender"], - }); - - let expected = json!([ - { - "name": "Cherry Orr", - "age": 27, - "color": "Green", - "gender": "female" - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": [], - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(json!([{}]), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["*"], - }); - - let expected = json!([ - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='male'" - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "name='Lucas Hess'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - }, - { - "id": 75, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [], - "isActive": false - } - ]); - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 30, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 31, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ], - "isActive": false - }, - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "NOT gender = 'female' AND age > 30" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 11, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "filters": "NOT gender = 'female' AND name='Evans Wagner'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name","email"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "email": [ - { - "start": 0, - "length": 6 - } - ], - "name": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches_and_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exerciatation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20, - "attributesToHighlight": ["about"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "about": [ - { - "start": 0, - "length": 12 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_2() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about"], - "cropLength": 20, - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_3() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about:20"], - }); - - let expected = json!( [ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_4() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["name:0","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_5() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr", - "email": "cherryorr", - "age": 27, - "gender": "female" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_6() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:10"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_7() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_8() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender","address"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*","address"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn test_faceted_search_valid() { - // set facetting attributes before adding documents - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - - let body = json!({ - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - let body: Value = serde_json::from_slice(dataset).unwrap(); - server.add_or_update_multiple_documents(body).await; - - // simple tests on attributes with string value - - let query = json!({ - "q": "a", - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "q": "a", - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "q": "a", - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "q": "a", - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("color") - .unwrap() == "blue" - && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green"))); - - }); -} - -#[actix_rt::test] -async fn test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "q": "a", - "facetFilters": ["color:blue"] - }); - - test_post_get_search!(server, query, |response, status_code| { - - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "q": "a", - "facetFilters": [] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - // [[]] - let query = json!({ - "q": "a", - "facetFilters": [[]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // ["color:green", []] - let query = json!({ - "q": "a", - "facetFilters": ["color:green", []] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // too much depth - // [[[]]] - let query = json!({ - "q": "a", - "facetFilters": [[[]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // [["color:green", ["color:blue"]]] - let query = json!({ - "q": "a", - "facetFilters": [["color:green", ["color:blue"]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // "color:green" - let query = json!({ - "q": "a", - "facetFilters": "color:green" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); -} - -#[actix_rt::test] -async fn test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({ - "q": "a", - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "q": "a", - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code|{ - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code|{ - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); - // assert that case is preserved - assert!(response["facetsDistribution"] - .as_object() - .unwrap()["color"] - .as_object() - .unwrap() - .get("Green") - .is_some()); - }); - // searching on color and tags - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); - assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); - }); - // wildcard - let query = json!({ - "q": "a", - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - // wildcard with other attributes: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - - // empty facet list - let query = json!({ - "q": "a", - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); - }); - - // attr not set as facet passed: - let query = json!({ - "q": "a", - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code|{ - assert_eq!(status_code, 400); - }); - -} - -#[actix_rt::test] -#[should_panic] -async fn test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "q": "a", - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn highlight_cropped_text() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let doc = json!([ - { - "id": 1, - "body": r##"well, it may not work like that, try the following: -1. insert your trip -2. google your `searchQuery` -3. find a solution -> say hello"## - } - ]); - server.add_or_replace_multiple_documents(doc).await; - - // tests from #680 - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 30, - }); - let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); - - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 80, - }); - let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); -} - -#[actix_rt::test] -async fn well_formated_error_with_bad_request_params() { - let mut server = common::Server::with_uid("test"); - let query = "foo=bar"; - let (response, _status_code) = server.search_get(query).await; - assert!(response.get("message").is_some()); - assert!(response.get("errorCode").is_some()); - assert!(response.get("errorType").is_some()); - assert!(response.get("errorLink").is_some()); -} - - -#[actix_rt::test] -async fn update_documents_with_facet_distribution() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - }); - server.update_all_settings(settings).await; - let update1 = json!([ - { - "id": "1", - "type": "album", - "title": "Nevermind", - "genre": ["grunge", "alternative"] - }, - { - "id": "2", - "type": "album", - "title": "Mellon Collie and the Infinite Sadness", - "genre": ["alternative", "rock"] - }, - { - "id": "3", - "type": "album", - "title": "The Queen Is Dead", - "genre": ["indie", "rock"] - } - ]); - server.add_or_update_multiple_documents(update1).await; - let search = json!({ - "q": "album", - "facetsDistribution": ["genre"] - }); - let (response1, _) = server.search_post(search.clone()).await; - let expected_facet_distribution = json!({ - "genre": { - "grunge": 1, - "alternative": 2, - "rock": 2, - "indie": 1 - } - }); - assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); - - let update2 = json!([ - { - "id": "3", - "title": "The Queen Is Very Dead" - } - ]); - server.add_or_update_multiple_documents(update2).await; - let (response2, _) = server.search_post(search).await; - assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_normal() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 3); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; - assert_eq!(response["nbHits"], 1); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} - -#[actix_rt::test] -async fn test_max_word_query() { - use meilisearch_core::MAX_QUERY_LEN; - - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - let documents = json!([ - {"id": 1, "value": "1 2 3 4 5 6 7 8 9 10 11"}, - {"id": 2, "value": "1 2 3 4 5 6 7 8 9 10"}] - ); - server.add_or_update_multiple_documents(documents).await; - - // We want to create a request where the 11 will be ignored. We have 2 documents, where a query - // with only one should return both, but a query with 1 and 11 should return only the first. - // This is how we know that outstanding query words have been ignored - let query = (0..MAX_QUERY_LEN) - .map(|_| "1") - .chain(std::iter::once("11")) - .fold(String::new(), |s, w| s + " " + w); - let (response, _) = server.search_post(json!({"q": query})).await; - assert_eq!(response["nbHits"], 2); - let (response, _) = server.search_post(json!({"q": "1 11"})).await; - assert_eq!(response["nbHits"], 1); -} diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs new file mode 100644 index 000000000..56ec6439c --- /dev/null +++ b/meilisearch-http/tests/search/mod.rs @@ -0,0 +1,2 @@ +// This modules contains all the test concerning search. Each particular feture of the search +// should be tested in its own module to isolate tests and keep the tests readable. diff --git a/meilisearch-http/tests/search_settings.rs b/meilisearch-http/tests/search_settings.rs deleted file mode 100644 index 97d27023a..000000000 --- a/meilisearch-http/tests/search_settings.rs +++ /dev/null @@ -1,621 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; - -mod common; - -#[actix_rt::test] -async fn search_with_settings_basic() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - - let expect = json!([ - { - "balance": "$2,467.47", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" - }, - { - "balance": "$3,344.40", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" - }, - { - "balance": "$3,394.96", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_stop_words() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": ["ea"], - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_synonyms() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "Application": [ - "Exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=application&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_normalized_synonyms() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "application": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=application&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_ranking_rules() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exarcitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - println!("{}", response["hits"].clone()); - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "exarcitation": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=Carol&limit=3"; - let expect = json!([ - { - "balance": "$1,440.09", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989" - }, - { - "balance": "$1,977.66", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_displayed_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243" - }, - { - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174" - }, - { - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes_2() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "name": "Harper Carson", - "gender": "male" - }, - { - "age": 27, - "name": "Cherry Orr", - "gender": "female" - }, - { - "age": 28, - "name": "Maureen Dale", - "gender": "female" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -// issue #798 -#[actix_rt::test] -async fn distinct_attributes_returns_name_not_id() { - let mut server = common::Server::test_server().await; - let settings = json!({ - "distinctAttribute": "color", - }); - server.update_all_settings(settings).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["distinctAttribute"], "color"); - let (response, _) = server.get_distinct_attribute().await; - assert_eq!(response, "color"); -} diff --git a/meilisearch-http/tests/settings.rs b/meilisearch-http/tests/settings.rs deleted file mode 100644 index 98973b56f..000000000 --- a/meilisearch-http/tests/settings.rs +++ /dev/null @@ -1,527 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_all_settings().await; - - // 5 - Get all settings and check if they are set to default values - - let (response, _status_code) = server.get_all_settings().await; - - let expect = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - assert_json_eq!(expect, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - "HP": ["Harry Potter"], - "Harry Potter": ["HP"] - }, - "attributesForFaceting": ["title"], - }); - - server.update_all_settings(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_all_settings().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - "hp": ["harry potter"], - "harry potter": ["hp"] - }, - "attributesForFaceting": ["title"], - }); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings_2() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 -#[actix_rt::test] -async fn write_setting_and_update_partial() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 2 - Send the settings - - let body = json!({ - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ] - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn attributes_for_faceting_settings() { - let mut server = common::Server::test_server().await; - // initial attributes array should be empty - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); - // add an attribute and test for its presence - let (_response, _status_code) = server.post_request_async( - "/indexes/test/settings/attributes-for-faceting", - json!(["foobar"])).await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!(["foobar"])); - // remove all attributes and test for emptiness - let (_response, _status_code) = server.delete_request_async( - "/indexes/test/settings/attributes-for-faceting").await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn setting_ranking_rules_dont_mess_with_other_settings() { - let mut server = common::Server::test_server().await; - let body = json!({ - "rankingRules": ["asc(foobar)"] - }); - server.update_all_settings(body).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); - assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); - assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); - assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); -} - -#[actix_rt::test] -async fn displayed_and_searchable_attributes_reset_to_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.delete_searchable_attributes().await; - server.delete_displayed_attributes().await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); - - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn settings_that_contains_wildcard_is_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn test_displayed_attributes_field() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "age", - "email", - "gender", - "name", - "registered", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["avenue", "street"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: true); -} \ No newline at end of file diff --git a/meilisearch-http/tests/settings/distinct.rs b/meilisearch-http/tests/settings/distinct.rs new file mode 100644 index 000000000..818f200fd --- /dev/null +++ b/meilisearch-http/tests/settings/distinct.rs @@ -0,0 +1,48 @@ +use crate::common::Server; +use serde_json::json; + +#[actix_rt::test] +async fn set_and_reset_distinct_attribute() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, _code) = index + .update_settings(json!({ "distinctAttribute": "test"})) + .await; + index.wait_update_id(0).await; + + let (response, _) = index.settings().await; + + assert_eq!(response["distinctAttribute"], "test"); + + index + .update_settings(json!({ "distinctAttribute": null })) + .await; + + index.wait_update_id(1).await; + + let (response, _) = index.settings().await; + + assert_eq!(response["distinctAttribute"], json!(null)); +} + +#[actix_rt::test] +async fn set_and_reset_distinct_attribute_with_dedicated_route() { + let server = Server::new().await; + let index = server.index("test"); + + let (_response, _code) = index.update_distinct_attribute(json!("test")).await; + index.wait_update_id(0).await; + + let (response, _) = index.get_distinct_attribute().await; + + assert_eq!(response, "test"); + + index.update_distinct_attribute(json!(null)).await; + + index.wait_update_id(1).await; + + let (response, _) = index.get_distinct_attribute().await; + + assert_eq!(response, json!(null)); +} diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs new file mode 100644 index 000000000..0b523eef3 --- /dev/null +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -0,0 +1,233 @@ +use std::collections::HashMap; + +use once_cell::sync::Lazy; +use serde_json::{json, Value}; + +use crate::common::Server; + +static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + map.insert("displayed_attributes", json!(["*"])); + map.insert("searchable_attributes", json!(["*"])); + map.insert("filterable_attributes", json!([])); + map.insert("distinct_attribute", json!(Value::Null)); + map.insert( + "ranking_rules", + json!(["words", "typo", "proximity", "attribute", "exactness"]), + ); + map.insert("stop_words", json!([])); + map.insert("synonyms", json!({})); + map +}); + +#[actix_rt::test] +async fn get_settings_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").settings().await; + assert_eq!(code, 404) +} + +#[actix_rt::test] +async fn get_settings() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + let settings = response.as_object().unwrap(); + assert_eq!(settings.keys().len(), 7); + assert_eq!(settings["displayedAttributes"], json!(["*"])); + assert_eq!(settings["searchableAttributes"], json!(["*"])); + assert_eq!(settings["filterableAttributes"], json!([])); + assert_eq!(settings["distinctAttribute"], json!(null)); + assert_eq!( + settings["rankingRules"], + json!(["words", "typo", "proximity", "attribute", "exactness"]) + ); + assert_eq!(settings["stopWords"], json!([])); +} + +#[actix_rt::test] +async fn update_settings_unknown_field() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.update_settings(json!({"foo": 12})).await; + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn test_partial_update() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, _code) = index + .update_settings(json!({"displayedAttributes": ["foo"]})) + .await; + index.wait_update_id(0).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); + + let (_response, _) = index + .update_settings(json!({"searchableAttributes": ["bar"]})) + .await; + index.wait_update_id(1).await; + + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"], json!(["foo"])); + assert_eq!(response["searchableAttributes"], json!(["bar"])); +} + +#[actix_rt::test] +async fn delete_settings_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.delete_settings().await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn reset_all_settings() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "id": 1, + "name": "curqui", + "age": 99 + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + assert_eq!(response["updateId"], 0); + index.wait_update_id(0).await; + + index + .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"], "synonyms": {"puppy": ["dog", "doggo", "potat"] }})) + .await; + index.wait_update_id(1).await; + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"], json!(["name", "age"])); + assert_eq!(response["searchableAttributes"], json!(["name"])); + assert_eq!(response["stopWords"], json!(["the"])); + assert_eq!( + response["synonyms"], + json!({"puppy": ["dog", "doggo", "potat"] }) + ); + assert_eq!(response["filterableAttributes"], json!(["age"])); + + index.delete_settings().await; + index.wait_update_id(2).await; + + let (response, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!(response["displayedAttributes"], json!(["*"])); + assert_eq!(response["searchableAttributes"], json!(["*"])); + assert_eq!(response["stopWords"], json!([])); + assert_eq!(response["filterableAttributes"], json!([])); + assert_eq!(response["synonyms"], json!({})); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert!(response.as_object().unwrap().get("age").is_some()); +} + +#[actix_rt::test] +async fn update_setting_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + let (_response, code) = index.update_settings(json!({})).await; + assert_eq!(code, 202); + let (_response, code) = index.get().await; + assert_eq!(code, 200); + let (_response, code) = index.delete_settings().await; + assert_eq!(code, 202); +} + +#[actix_rt::test] +async fn update_setting_unexisting_index_invalid_uid() { + let server = Server::new().await; + let index = server.index("test##! "); + let (_response, code) = index.update_settings(json!({})).await; + assert_eq!(code, 400); +} + +macro_rules! test_setting_routes { + ($($setting:ident), *) => { + $( + mod $setting { + use crate::common::Server; + use super::DEFAULT_SETTINGS_VALUES; + + #[actix_rt::test] + async fn get_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (_response, code) = server.service.get(url).await; + assert_eq!(code, 404); + } + + #[actix_rt::test] + async fn update_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (response, code) = server.service.post(url, serde_json::Value::Null).await; + assert_eq!(code, 202, "{}", response); + let (response, code) = server.index("test").get().await; + assert_eq!(code, 200, "{}", response); + } + + #[actix_rt::test] + async fn delete_unexisting_index() { + let server = Server::new().await; + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (response, code) = server.service.delete(url).await; + assert_eq!(code, 404, "{}", response); + } + + #[actix_rt::test] + async fn get_default() { + let server = Server::new().await; + let index = server.index("test"); + let (response, code) = index.create(None).await; + assert_eq!(code, 200, "{}", response); + let url = format!("/indexes/test/settings/{}", + stringify!($setting) + .chars() + .map(|c| if c == '_' { '-' } else { c }) + .collect::()); + let (response, code) = server.service.get(url).await; + assert_eq!(code, 200, "{}", response); + let expected = DEFAULT_SETTINGS_VALUES.get(stringify!($setting)).unwrap(); + assert_eq!(expected, &response); + } + } + )* + }; +} + +test_setting_routes!( + filterable_attributes, + displayed_attributes, + searchable_attributes, + distinct_attribute, + stop_words, + ranking_rules, + synonyms +); diff --git a/meilisearch-http/tests/settings/mod.rs b/meilisearch-http/tests/settings/mod.rs new file mode 100644 index 000000000..05339cb37 --- /dev/null +++ b/meilisearch-http/tests/settings/mod.rs @@ -0,0 +1,2 @@ +mod distinct; +mod get_settings; diff --git a/meilisearch-http/tests/settings_ranking_rules.rs b/meilisearch-http/tests/settings_ranking_rules.rs deleted file mode 100644 index ac9a1e00c..000000000 --- a/meilisearch-http/tests/settings_ranking_rules.rs +++ /dev/null @@ -1,182 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_ranking_rules().await; - - // 5 - Get all settings and check if they are empty - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - server.update_ranking_rules(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn send_undefined_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["typos",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn send_malformed_custom_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["dsc(truc)",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 -#[actix_rt::test] -async fn write_custom_ranking_and_index_documents() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Add ranking rules with one custom ranking on a string - - let body = json!(["asc(name)", "typo"]); - - server.update_ranking_rules(body).await; - - // 2 - Add documents - - let body = json!([ - { - "id": 1, - "name": "Cherry Orr", - "color": "green" - }, - { - "id": 2, - "name": "Lucas Hess", - "color": "yellow" - } - ]); - - server.add_or_replace_multiple_documents(body).await; - - // 3 - Get the first document and compare - - let expected = json!({ - "id": 1, - "name": "Cherry Orr", - "color": "green" - }); - - let (response, status_code) = server.get_document(1).await; - assert_eq!(status_code, 200); - - assert_json_eq!(response, expected, ordered: false); -} diff --git a/meilisearch-http/tests/settings_stop_words.rs b/meilisearch-http/tests/settings_stop_words.rs deleted file mode 100644 index 3ff2e8bb7..000000000 --- a/meilisearch-http/tests/settings_stop_words.rs +++ /dev/null @@ -1,61 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn update_stop_words() { - let mut server = common::Server::test_server().await; - - // 1 - Get stop words - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); - - // 2 - Update stop words - - let body = json!(["ut", "ea"]); - server.update_stop_words(body.clone()).await; - - // 3 - Get all stop words and compare to the previous one - - let (response, _status_code) = server.get_stop_words().await; - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all stop words - - server.delete_stop_words().await; - - // 5 - Get all stop words and check if they are empty - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); -} - -#[actix_rt::test] -async fn add_documents_and_stop_words() { - let mut server = common::Server::test_server().await; - - // 2 - Update stop words - - let body = json!(["ad", "in"]); - server.update_stop_words(body.clone()).await; - - // 3 - Search for a document with stop words - - let (response, _status_code) = server.search_get("q=in%20exercitation").await; - assert!(!response["hits"].as_array().unwrap().is_empty()); - - // 4 - Search for documents with *only* stop words - - let (response, _status_code) = server.search_get("q=ad%20in").await; - assert!(response["hits"].as_array().unwrap().is_empty()); - - // 5 - Delete all stop words - - // server.delete_stop_words(); - - // // 6 - Search for a document with one stop word - - // assert!(!response["hits"].as_array().unwrap().is_empty()); -} diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs new file mode 100644 index 000000000..b5602c508 --- /dev/null +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use crate::common::server::default_settings; +use crate::common::GetAllDocumentsOptions; +use crate::common::Server; +use tokio::time::sleep; + +use meilisearch_http::Opt; + +#[actix_rt::test] +async fn perform_snapshot() { + let temp = tempfile::tempdir_in(".").unwrap(); + let snapshot_dir = tempfile::tempdir_in(".").unwrap(); + + let options = Opt { + snapshot_dir: snapshot_dir.path().to_owned(), + snapshot_interval_sec: 1, + schedule_snapshot: true, + ..default_settings(temp.path()) + }; + + let server = Server::new_with_options(options).await; + let index = server.index("test"); + index.load_test_set().await; + + let (response, _) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + + sleep(Duration::from_secs(2)).await; + + let temp = tempfile::tempdir_in(".").unwrap(); + + let snapshot_path = snapshot_dir + .path() + .to_owned() + .join("db.snapshot".to_string()); + + let options = Opt { + import_snapshot: Some(snapshot_path), + ..default_settings(temp.path()) + }; + + let server = Server::new_with_options(options).await; + let index = server.index("test"); + + let (response_from_snapshot, _) = index + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + + assert_eq!(response, response_from_snapshot); +} diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs new file mode 100644 index 000000000..aba860256 --- /dev/null +++ b/meilisearch-http/tests/stats/mod.rs @@ -0,0 +1,70 @@ +use serde_json::json; + +use crate::common::Server; + +#[actix_rt::test] +async fn get_settings_unexisting_index() { + let server = Server::new().await; + let (response, code) = server.version().await; + assert_eq!(code, 200); + let version = response.as_object().unwrap(); + assert!(version.get("commitSha").is_some()); + assert!(version.get("commitDate").is_some()); + assert!(version.get("pkgVersion").is_some()); +} + +#[actix_rt::test] +async fn test_healthyness() { + let server = Server::new().await; + + let (response, status_code) = server.service.get("/health").await; + assert_eq!(status_code, 200); + assert_eq!(response["status"], "available"); +} + +#[actix_rt::test] +async fn stats() { + let server = Server::new().await; + let index = server.index("test"); + let (_, code) = index.create(Some("id")).await; + + assert_eq!(code, 200); + + let (response, code) = server.stats().await; + + assert_eq!(code, 200); + assert!(response.get("databaseSize").is_some()); + assert!(response.get("lastUpdate").is_some()); + assert!(response["indexes"].get("test").is_some()); + assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 0); + assert!(response["indexes"]["test"]["isIndexing"] == false); + + let documents = json!([ + { + "id": 1, + "name": "Alexey", + }, + { + "id": 2, + "age": 45, + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202, "{}", response); + assert_eq!(response["updateId"], 0); + + let response = index.wait_update_id(0).await; + println!("response: {}", response); + + let (response, code) = server.stats().await; + + assert_eq!(code, 200); + assert!(response["databaseSize"].as_u64().unwrap() > 0); + assert!(response.get("lastUpdate").is_some()); + assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); + assert!(response["indexes"]["test"]["isIndexing"] == false); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["id"], 2); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["name"], 1); + assert_eq!(response["indexes"]["test"]["fieldDistribution"]["age"], 1); +} diff --git a/meilisearch-http/tests/updates/mod.rs b/meilisearch-http/tests/updates/mod.rs new file mode 100644 index 000000000..00bbf32a8 --- /dev/null +++ b/meilisearch-http/tests/updates/mod.rs @@ -0,0 +1,69 @@ +use crate::common::Server; + +#[actix_rt::test] +async fn get_update_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").get_update(0).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn get_unexisting_update_status() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (_response, code) = index.get_update(0).await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn get_update_status() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index + .add_documents( + serde_json::json!([{ + "id": 1, + "content": "foobar", + }]), + None, + ) + .await; + let (_response, code) = index.get_update(0).await; + assert_eq!(code, 200); + // TODO check resonse format, as per #48 +} + +#[actix_rt::test] +async fn list_updates_unexisting_index() { + let server = Server::new().await; + let (_response, code) = server.index("test").list_updates().await; + assert_eq!(code, 404); +} + +#[actix_rt::test] +async fn list_no_updates() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let (response, code) = index.list_updates().await; + assert_eq!(code, 200); + assert!(response.as_array().unwrap().is_empty()); +} + +#[actix_rt::test] +async fn list_updates() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; + let (response, code) = index.list_updates().await; + assert_eq!(code, 200); + assert_eq!(response.as_array().unwrap().len(), 1); +} diff --git a/meilisearch-http/tests/url_normalizer.rs b/meilisearch-http/tests/url_normalizer.rs deleted file mode 100644 index c2c9187ee..000000000 --- a/meilisearch-http/tests/url_normalizer.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn url_normalizer() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/version/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version/").await; - assert_eq!(status_code, 200); -} diff --git a/meilisearch-schema/Cargo.toml b/meilisearch-schema/Cargo.toml deleted file mode 100644 index 7fcc62380..000000000 --- a/meilisearch-schema/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "meilisearch-schema" -version = "0.20.0" -license = "MIT" -authors = ["Kerollmops "] -edition = "2018" - -[dependencies] -indexmap = { version = "1.6.1", features = ["serde-1"] } -meilisearch-error = { path = "../meilisearch-error", version = "0.20.0" } -serde = { version = "1.0.118", features = ["derive"] } -serde_json = { version = "1.0.61", features = ["preserve_order"] } -zerocopy = "0.3.0" diff --git a/meilisearch-schema/src/error.rs b/meilisearch-schema/src/error.rs deleted file mode 100644 index 331721e24..000000000 --- a/meilisearch-schema/src/error.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::{error, fmt}; - -use meilisearch_error::{ErrorCode, Code}; - -pub type SResult = Result; - -#[derive(Debug)] -pub enum Error { - FieldNameNotFound(String), - PrimaryKeyAlreadyPresent, - MaxFieldsLimitExceeded, -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::Error::*; - match self { - FieldNameNotFound(field) => write!(f, "The field {:?} doesn't exist", field), - PrimaryKeyAlreadyPresent => write!(f, "A primary key is already present. It's impossible to update it"), - MaxFieldsLimitExceeded => write!(f, "The maximum of possible reattributed field id has been reached"), - } - } -} - -impl error::Error for Error {} - -impl ErrorCode for Error { - fn error_code(&self) -> Code { - use Error::*; - - match self { - FieldNameNotFound(_) => Code::Internal, - MaxFieldsLimitExceeded => Code::MaxFieldsLimitExceeded, - PrimaryKeyAlreadyPresent => Code::PrimaryKeyAlreadyPresent, - } - } -} diff --git a/meilisearch-schema/src/fields_map.rs b/meilisearch-schema/src/fields_map.rs deleted file mode 100644 index b182c9c25..000000000 --- a/meilisearch-schema/src/fields_map.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::collections::HashMap; -use std::collections::hash_map::Iter; - -use serde::{Deserialize, Serialize}; - -use crate::{SResult, FieldId}; - -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct FieldsMap { - name_map: HashMap, - id_map: HashMap, - next_id: FieldId -} - -impl FieldsMap { - pub(crate) fn insert(&mut self, name: &str) -> SResult { - if let Some(id) = self.name_map.get(name) { - return Ok(*id) - } - let id = self.next_id; - self.next_id = self.next_id.next()?; - self.name_map.insert(name.to_string(), id); - self.id_map.insert(id, name.to_string()); - Ok(id) - } - - pub(crate) fn id(&self, name: &str) -> Option { - self.name_map.get(name).copied() - } - - pub(crate) fn name>(&self, id: I) -> Option<&str> { - self.id_map.get(&id.into()).map(|s| s.as_str()) - } - - pub(crate) fn iter(&self) -> Iter<'_, String, FieldId> { - self.name_map.iter() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn fields_map() { - let mut fields_map = FieldsMap::default(); - assert_eq!(fields_map.insert("id").unwrap(), 0.into()); - assert_eq!(fields_map.insert("title").unwrap(), 1.into()); - assert_eq!(fields_map.insert("descritpion").unwrap(), 2.into()); - assert_eq!(fields_map.insert("id").unwrap(), 0.into()); - assert_eq!(fields_map.insert("title").unwrap(), 1.into()); - assert_eq!(fields_map.insert("descritpion").unwrap(), 2.into()); - assert_eq!(fields_map.id("id"), Some(0.into())); - assert_eq!(fields_map.id("title"), Some(1.into())); - assert_eq!(fields_map.id("descritpion"), Some(2.into())); - assert_eq!(fields_map.id("date"), None); - assert_eq!(fields_map.name(0), Some("id")); - assert_eq!(fields_map.name(1), Some("title")); - assert_eq!(fields_map.name(2), Some("descritpion")); - assert_eq!(fields_map.name(4), None); - assert_eq!(fields_map.insert("title").unwrap(), 1.into()); - } -} diff --git a/meilisearch-schema/src/lib.rs b/meilisearch-schema/src/lib.rs deleted file mode 100644 index dd2e7c2fb..000000000 --- a/meilisearch-schema/src/lib.rs +++ /dev/null @@ -1,75 +0,0 @@ -mod error; -mod fields_map; -mod schema; -mod position_map; - -pub use error::{Error, SResult}; -use fields_map::FieldsMap; -pub use schema::Schema; -use serde::{Deserialize, Serialize}; -use zerocopy::{AsBytes, FromBytes}; - -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Default, PartialOrd, Ord, PartialEq, Eq, Hash)] -pub struct IndexedPos(pub u16); - -impl IndexedPos { - pub const fn new(value: u16) -> IndexedPos { - IndexedPos(value) - } - - pub const fn min() -> IndexedPos { - IndexedPos(u16::min_value()) - } - - pub const fn max() -> IndexedPos { - IndexedPos(u16::max_value()) - } -} - -impl From for IndexedPos { - fn from(value: u16) -> IndexedPos { - IndexedPos(value) - } -} - -impl Into for IndexedPos { - fn into(self) -> u16 { - self.0 - } -} - -#[derive(Debug, Copy, Clone, Default, PartialOrd, Ord, PartialEq, Eq, Hash)] -#[derive(Serialize, Deserialize)] -#[derive(AsBytes, FromBytes)] -#[repr(C)] -pub struct FieldId(pub u16); - -impl FieldId { - pub const fn new(value: u16) -> FieldId { - FieldId(value) - } - - pub const fn min() -> FieldId { - FieldId(u16::min_value()) - } - - pub const fn max() -> FieldId { - FieldId(u16::max_value()) - } - - pub fn next(self) -> SResult { - self.0.checked_add(1).map(FieldId).ok_or(Error::MaxFieldsLimitExceeded) - } -} - -impl From for FieldId { - fn from(value: u16) -> FieldId { - FieldId(value) - } -} - -impl From for u16 { - fn from(other: FieldId) -> u16 { - other.0 - } -} diff --git a/meilisearch-schema/src/position_map.rs b/meilisearch-schema/src/position_map.rs deleted file mode 100644 index 9da578771..000000000 --- a/meilisearch-schema/src/position_map.rs +++ /dev/null @@ -1,161 +0,0 @@ -use std::collections::BTreeMap; - -use crate::{FieldId, IndexedPos}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct PositionMap { - pos_to_field: Vec, - field_to_pos: BTreeMap, -} - -impl PositionMap { - /// insert `id` at the specified `position` updating the other position if a shift is caused by - /// the operation. If `id` is already present in the position map, it is moved to the requested - /// `position`, potentially causing shifts. - pub fn insert(&mut self, id: FieldId, position: IndexedPos) -> IndexedPos { - let mut upos = position.0 as usize; - let mut must_rebuild_map = false; - - if let Some(old_pos) = self.field_to_pos.get(&id) { - let uold_pos = old_pos.0 as usize; - self.pos_to_field.remove(uold_pos); - must_rebuild_map = true; - } - - if upos < self.pos_to_field.len() { - self.pos_to_field.insert(upos, id); - must_rebuild_map = true; - } else { - upos = self.pos_to_field.len(); - self.pos_to_field.push(id); - } - - // we only need to update all the positions if there have been a shift a some point. In - // most cases we only did a push, so we don't need to rebuild the `field_to_pos` map. - if must_rebuild_map { - self.field_to_pos.clear(); - self.field_to_pos.extend( - self.pos_to_field - .iter() - .enumerate() - .map(|(p, f)| (*f, IndexedPos(p as u16))), - ); - } else { - self.field_to_pos.insert(id, IndexedPos(upos as u16)); - } - IndexedPos(upos as u16) - } - - /// Pushes `id` in last position - pub fn push(&mut self, id: FieldId) -> IndexedPos { - let pos = self.len(); - self.insert(id, IndexedPos(pos as u16)) - } - - pub fn len(&self) -> usize { - self.pos_to_field.len() - } - - pub fn field_to_pos(&self, id: FieldId) -> Option { - self.field_to_pos.get(&id).cloned() - } - - pub fn pos_to_field(&self, pos: IndexedPos) -> Option { - let pos = pos.0 as usize; - self.pos_to_field.get(pos).cloned() - } - - pub fn field_pos(&self) -> impl Iterator + '_ { - self.pos_to_field - .iter() - .enumerate() - .map(|(i, f)| (*f, IndexedPos(i as u16))) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_default() { - assert_eq!( - format!("{:?}", PositionMap::default()), - r##"PositionMap { pos_to_field: [], field_to_pos: {} }"## - ); - } - - #[test] - fn test_insert() { - let mut map = PositionMap::default(); - // changing position removes from old position - map.insert(0.into(), 0.into()); - map.insert(1.into(), 1.into()); - assert_eq!( - format!("{:?}", map), - r##"PositionMap { pos_to_field: [FieldId(0), FieldId(1)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(1): IndexedPos(1)} }"## - ); - map.insert(0.into(), 1.into()); - assert_eq!( - format!("{:?}", map), - r##"PositionMap { pos_to_field: [FieldId(1), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(1), FieldId(1): IndexedPos(0)} }"## - ); - map.insert(2.into(), 1.into()); - assert_eq!( - format!("{:?}", map), - r##"PositionMap { pos_to_field: [FieldId(1), FieldId(2), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(2), FieldId(1): IndexedPos(0), FieldId(2): IndexedPos(1)} }"## - ); - } - - #[test] - fn test_push() { - let mut map = PositionMap::default(); - map.push(0.into()); - map.push(2.into()); - assert_eq!(map.len(), 2); - assert_eq!( - format!("{:?}", map), - r##"PositionMap { pos_to_field: [FieldId(0), FieldId(2)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(2): IndexedPos(1)} }"## - ); - } - - #[test] - fn test_field_to_pos() { - let mut map = PositionMap::default(); - map.push(0.into()); - map.push(2.into()); - assert_eq!(map.field_to_pos(2.into()), Some(1.into())); - assert_eq!(map.field_to_pos(0.into()), Some(0.into())); - assert_eq!(map.field_to_pos(4.into()), None); - } - - #[test] - fn test_pos_to_field() { - let mut map = PositionMap::default(); - map.push(0.into()); - map.push(2.into()); - map.push(3.into()); - map.push(4.into()); - assert_eq!( - format!("{:?}", map), - r##"PositionMap { pos_to_field: [FieldId(0), FieldId(2), FieldId(3), FieldId(4)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(2): IndexedPos(1), FieldId(3): IndexedPos(2), FieldId(4): IndexedPos(3)} }"## - ); - assert_eq!(map.pos_to_field(0.into()), Some(0.into())); - assert_eq!(map.pos_to_field(1.into()), Some(2.into())); - assert_eq!(map.pos_to_field(2.into()), Some(3.into())); - assert_eq!(map.pos_to_field(3.into()), Some(4.into())); - assert_eq!(map.pos_to_field(4.into()), None); - } - - #[test] - fn test_field_pos() { - let mut map = PositionMap::default(); - map.push(0.into()); - map.push(2.into()); - let mut iter = map.field_pos(); - assert_eq!(iter.next(), Some((0.into(), 0.into()))); - assert_eq!(iter.next(), Some((2.into(), 1.into()))); - assert_eq!(iter.next(), None); - } -} diff --git a/meilisearch-schema/src/schema.rs b/meilisearch-schema/src/schema.rs deleted file mode 100644 index 17377cedd..000000000 --- a/meilisearch-schema/src/schema.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::borrow::Cow; -use std::collections::{BTreeSet, HashSet}; - -use serde::{Deserialize, Serialize}; - -use crate::position_map::PositionMap; -use crate::{Error, FieldId, FieldsMap, IndexedPos, SResult}; - -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub struct Schema { - fields_map: FieldsMap, - - primary_key: Option, - ranked: HashSet, - displayed: Option>, - - searchable: Option>, - pub indexed_position: PositionMap, -} - -impl Schema { - pub fn with_primary_key(name: &str) -> Schema { - let mut fields_map = FieldsMap::default(); - let field_id = fields_map.insert(name).unwrap(); - let mut indexed_position = PositionMap::default(); - indexed_position.push(field_id); - - Schema { - fields_map, - primary_key: Some(field_id), - ranked: HashSet::new(), - displayed: None, - searchable: None, - indexed_position, - } - } - - pub fn primary_key(&self) -> Option<&str> { - self.primary_key.map(|id| self.fields_map.name(id).unwrap()) - } - - pub fn set_primary_key(&mut self, name: &str) -> SResult { - if self.primary_key.is_some() { - return Err(Error::PrimaryKeyAlreadyPresent); - } - - let id = self.insert(name)?; - self.primary_key = Some(id); - - Ok(id) - } - - pub fn id(&self, name: &str) -> Option { - self.fields_map.id(name) - } - - pub fn name>(&self, id: I) -> Option<&str> { - self.fields_map.name(id) - } - - pub fn names(&self) -> impl Iterator { - self.fields_map.iter().map(|(k, _)| k.as_ref()) - } - - /// add `name` to the list of known fields - pub fn insert(&mut self, name: &str) -> SResult { - self.fields_map.insert(name) - } - - /// Adds `name` to the list of known fields, and in the last position of the indexed_position map. This - /// field is taken into acccount when `searchableAttribute` or `displayedAttributes` is set to `"*"` - pub fn insert_with_position(&mut self, name: &str) -> SResult<(FieldId, IndexedPos)> { - let field_id = self.fields_map.insert(name)?; - let position = self - .is_searchable(field_id) - .unwrap_or_else(|| self.indexed_position.push(field_id)); - Ok((field_id, position)) - } - - pub fn ranked(&self) -> &HashSet { - &self.ranked - } - - fn displayed(&self) -> Cow> { - match &self.displayed { - Some(displayed) => Cow::Borrowed(displayed), - None => Cow::Owned(self.indexed_position.field_pos().map(|(f, _)| f).collect()), - } - } - - pub fn is_displayed_all(&self) -> bool { - self.displayed.is_none() - } - - pub fn displayed_names(&self) -> BTreeSet<&str> { - self.displayed() - .iter() - .filter_map(|&f| self.name(f)) - .collect() - } - - fn searchable(&self) -> Cow<[FieldId]> { - match &self.searchable { - Some(searchable) => Cow::Borrowed(&searchable), - None => Cow::Owned(self.indexed_position.field_pos().map(|(f, _)| f).collect()), - } - } - - pub fn searchable_names(&self) -> Vec<&str> { - self.searchable() - .iter() - .filter_map(|a| self.name(*a)) - .collect() - } - - pub(crate) fn set_ranked(&mut self, name: &str) -> SResult { - let id = self.fields_map.insert(name)?; - self.ranked.insert(id); - Ok(id) - } - - pub fn clear_ranked(&mut self) { - self.ranked.clear(); - } - - pub fn is_ranked(&self, id: FieldId) -> bool { - self.ranked.get(&id).is_some() - } - - pub fn is_displayed(&self, id: FieldId) -> bool { - match &self.displayed { - Some(displayed) => displayed.contains(&id), - None => true, - } - } - - pub fn is_searchable(&self, id: FieldId) -> Option { - match &self.searchable { - Some(searchable) if searchable.contains(&id) => self.indexed_position.field_to_pos(id), - None => self.indexed_position.field_to_pos(id), - _ => None, - } - } - - pub fn is_searchable_all(&self) -> bool { - self.searchable.is_none() - } - - pub fn indexed_pos_to_field_id>(&self, pos: I) -> Option { - self.indexed_position.pos_to_field(pos.into()) - } - - pub fn update_ranked>( - &mut self, - data: impl IntoIterator, - ) -> SResult<()> { - self.ranked.clear(); - for name in data { - self.set_ranked(name.as_ref())?; - } - Ok(()) - } - - pub fn update_displayed>( - &mut self, - data: impl IntoIterator, - ) -> SResult<()> { - let mut displayed = BTreeSet::new(); - for name in data { - let id = self.fields_map.insert(name.as_ref())?; - displayed.insert(id); - } - self.displayed.replace(displayed); - Ok(()) - } - - pub fn update_searchable>(&mut self, data: Vec) -> SResult<()> { - let mut searchable = Vec::with_capacity(data.len()); - for (pos, name) in data.iter().enumerate() { - let id = self.insert(name.as_ref())?; - self.indexed_position.insert(id, IndexedPos(pos as u16)); - searchable.push(id); - } - self.searchable.replace(searchable); - Ok(()) - } - - pub fn set_all_searchable(&mut self) { - self.searchable.take(); - } - - pub fn set_all_displayed(&mut self) { - self.displayed.take(); - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_with_primary_key() { - let schema = Schema::with_primary_key("test"); - assert_eq!( - format!("{:?}", schema), - r##"Schema { fields_map: FieldsMap { name_map: {"test": FieldId(0)}, id_map: {FieldId(0): "test"}, next_id: FieldId(1) }, primary_key: Some(FieldId(0)), ranked: {}, displayed: None, searchable: None, indexed_position: PositionMap { pos_to_field: [FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(0)} } }"## - ); - } - - #[test] - fn primary_key() { - let schema = Schema::with_primary_key("test"); - assert_eq!(schema.primary_key(), Some("test")); - } - - #[test] - fn test_insert_with_position_base() { - let mut schema = Schema::default(); - let (id, position) = schema.insert_with_position("foo").unwrap(); - assert!(schema.searchable.is_none()); - assert!(schema.displayed.is_none()); - assert_eq!(id, 0.into()); - assert_eq!(position, 0.into()); - let (id, position) = schema.insert_with_position("bar").unwrap(); - assert_eq!(id, 1.into()); - assert_eq!(position, 1.into()); - } - - #[test] - fn test_insert_with_position_primary_key() { - let mut schema = Schema::with_primary_key("test"); - let (id, position) = schema.insert_with_position("foo").unwrap(); - assert!(schema.searchable.is_none()); - assert!(schema.displayed.is_none()); - assert_eq!(id, 1.into()); - assert_eq!(position, 1.into()); - let (id, position) = schema.insert_with_position("test").unwrap(); - assert_eq!(id, 0.into()); - assert_eq!(position, 0.into()); - } - - #[test] - fn test_insert() { - let mut schema = Schema::default(); - let field_id = schema.insert("foo").unwrap(); - assert!(schema.fields_map.name(field_id).is_some()); - assert!(schema.searchable.is_none()); - assert!(schema.displayed.is_none()); - } - - #[test] - fn test_update_searchable() { - let mut schema = Schema::default(); - - schema.update_searchable(vec!["foo", "bar"]).unwrap(); - assert_eq!( - format!("{:?}", schema.indexed_position), - r##"PositionMap { pos_to_field: [FieldId(0), FieldId(1)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(1): IndexedPos(1)} }"## - ); - assert_eq!( - format!("{:?}", schema.searchable), - r##"Some([FieldId(0), FieldId(1)])"## - ); - schema.update_searchable(vec!["bar"]).unwrap(); - assert_eq!( - format!("{:?}", schema.searchable), - r##"Some([FieldId(1)])"## - ); - assert_eq!( - format!("{:?}", schema.indexed_position), - r##"PositionMap { pos_to_field: [FieldId(1), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(1), FieldId(1): IndexedPos(0)} }"## - ); - } - - #[test] - fn test_update_displayed() { - let mut schema = Schema::default(); - schema.update_displayed(vec!["foobar"]).unwrap(); - assert_eq!( - format!("{:?}", schema.displayed), - r##"Some({FieldId(0)})"## - ); - assert_eq!( - format!("{:?}", schema.indexed_position), - r##"PositionMap { pos_to_field: [], field_to_pos: {} }"## - ); - } - - #[test] - fn test_is_searchable_all() { - let mut schema = Schema::default(); - assert!(schema.is_searchable_all()); - schema.update_searchable(vec!["foo"]).unwrap(); - assert!(!schema.is_searchable_all()); - } - - #[test] - fn test_is_displayed_all() { - let mut schema = Schema::default(); - assert!(schema.is_displayed_all()); - schema.update_displayed(vec!["foo"]).unwrap(); - assert!(!schema.is_displayed_all()); - } - - #[test] - fn test_searchable_names() { - let mut schema = Schema::default(); - assert_eq!(format!("{:?}", schema.searchable_names()), r##"[]"##); - schema.insert_with_position("foo").unwrap(); - schema.insert_with_position("bar").unwrap(); - assert_eq!( - format!("{:?}", schema.searchable_names()), - r##"["foo", "bar"]"## - ); - schema.update_searchable(vec!["hello", "world"]).unwrap(); - assert_eq!( - format!("{:?}", schema.searchable_names()), - r##"["hello", "world"]"## - ); - schema.set_all_searchable(); - assert_eq!( - format!("{:?}", schema.searchable_names()), - r##"["hello", "world", "foo", "bar"]"## - ); - } - - #[test] - fn test_displayed_names() { - let mut schema = Schema::default(); - assert_eq!(format!("{:?}", schema.displayed_names()), r##"{}"##); - schema.insert_with_position("foo").unwrap(); - schema.insert_with_position("bar").unwrap(); - assert_eq!( - format!("{:?}", schema.displayed_names()), - r##"{"bar", "foo"}"## - ); - schema.update_displayed(vec!["hello", "world"]).unwrap(); - assert_eq!( - format!("{:?}", schema.displayed_names()), - r##"{"hello", "world"}"## - ); - schema.set_all_displayed(); - assert_eq!( - format!("{:?}", schema.displayed_names()), - r##"{"bar", "foo"}"## - ); - } - - #[test] - fn test_set_all_searchable() { - let mut schema = Schema::default(); - assert!(schema.is_searchable_all()); - schema.update_searchable(vec!["foobar"]).unwrap(); - assert!(!schema.is_searchable_all()); - schema.set_all_searchable(); - assert!(schema.is_searchable_all()); - } - - #[test] - fn test_set_all_displayed() { - let mut schema = Schema::default(); - assert!(schema.is_displayed_all()); - schema.update_displayed(vec!["foobar"]).unwrap(); - assert!(!schema.is_displayed_all()); - schema.set_all_displayed(); - assert!(schema.is_displayed_all()); - } -} diff --git a/meilisearch-tokenizer/Cargo.toml b/meilisearch-tokenizer/Cargo.toml deleted file mode 100644 index c7a6264cb..000000000 --- a/meilisearch-tokenizer/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "meilisearch-tokenizer" -version = "0.20.0" -license = "MIT" -authors = ["Kerollmops "] -edition = "2018" - -[dependencies] -deunicode = "1.1.1" -slice-group-by = "0.2.6" diff --git a/meilisearch-tokenizer/src/lib.rs b/meilisearch-tokenizer/src/lib.rs deleted file mode 100644 index 13874498b..000000000 --- a/meilisearch-tokenizer/src/lib.rs +++ /dev/null @@ -1,548 +0,0 @@ -use self::SeparatorCategory::*; -use deunicode::deunicode_char; -use slice_group_by::StrGroupBy; -use std::iter::Peekable; - -pub fn is_cjk(c: char) -> bool { - ('\u{1100}'..='\u{11ff}').contains(&c) - || ('\u{2e80}'..='\u{2eff}').contains(&c) // CJK Radicals Supplement - || ('\u{2f00}'..='\u{2fdf}').contains(&c) // Kangxi radical - || ('\u{3000}'..='\u{303f}').contains(&c) // Japanese-style punctuation - || ('\u{3040}'..='\u{309f}').contains(&c) // Japanese Hiragana - || ('\u{30a0}'..='\u{30ff}').contains(&c) // Japanese Katakana - || ('\u{3100}'..='\u{312f}').contains(&c) - || ('\u{3130}'..='\u{318F}').contains(&c) // Hangul Compatibility Jamo - || ('\u{3200}'..='\u{32ff}').contains(&c) // Enclosed CJK Letters and Months - || ('\u{3400}'..='\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A - || ('\u{4e00}'..='\u{9fff}').contains(&c) // CJK Unified Ideographs - || ('\u{a960}'..='\u{a97f}').contains(&c) // Hangul Jamo Extended-A - || ('\u{ac00}'..='\u{d7a3}').contains(&c) // Hangul Syllables - || ('\u{d7b0}'..='\u{d7ff}').contains(&c) // Hangul Jamo Extended-B - || ('\u{f900}'..='\u{faff}').contains(&c) // CJK Compatibility Ideographs - || ('\u{ff00}'..='\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum SeparatorCategory { - Soft, - Hard, -} - -impl SeparatorCategory { - fn merge(self, other: SeparatorCategory) -> SeparatorCategory { - if let (Soft, Soft) = (self, other) { - Soft - } else { - Hard - } - } - - fn to_usize(self) -> usize { - match self { - Soft => 1, - Hard => 8, - } - } -} - -fn is_separator(c: char) -> bool { - classify_separator(c).is_some() -} - -fn classify_separator(c: char) -> Option { - match c { - c if c.is_whitespace() => Some(Soft), // whitespaces - c if deunicode_char(c) == Some("'") => Some(Soft), // quotes - c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes - '-' | '_' | '\'' | ':' | '/' | '\\' | '@' => Some(Soft), - '.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard), - _ => None, - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum CharCategory { - Separator(SeparatorCategory), - Cjk, - Other, -} - -fn classify_char(c: char) -> CharCategory { - if let Some(category) = classify_separator(c) { - CharCategory::Separator(category) - } else if is_cjk(c) { - CharCategory::Cjk - } else { - CharCategory::Other - } -} - -fn is_str_word(s: &str) -> bool { - !s.chars().any(is_separator) -} - -fn same_group_category(a: char, b: char) -> bool { - match (classify_char(a), classify_char(b)) { - (CharCategory::Cjk, _) | (_, CharCategory::Cjk) => false, - (CharCategory::Separator(_), CharCategory::Separator(_)) => true, - (a, b) => a == b, - } -} - -// fold the number of chars along with the index position -fn chars_count_index((n, _): (usize, usize), (i, c): (usize, char)) -> (usize, usize) { - (n + 1, i + c.len_utf8()) -} - -pub fn split_query_string(query: &str) -> impl Iterator { - Tokenizer::new(query).map(|t| t.word) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct Token<'a> { - pub word: &'a str, - /// index of the token in the token sequence - pub index: usize, - pub word_index: usize, - pub char_index: usize, -} - -pub struct Tokenizer<'a> { - count: usize, - inner: &'a str, - word_index: usize, - char_index: usize, -} - -impl<'a> Tokenizer<'a> { - pub fn new(string: &str) -> Tokenizer { - // skip every separator and set `char_index` - // to the number of char trimmed - let (count, index) = string - .char_indices() - .take_while(|(_, c)| is_separator(*c)) - .fold((0, 0), chars_count_index); - - Tokenizer { - count: 0, - inner: &string[index..], - word_index: 0, - char_index: count, - } - } -} - -impl<'a> Iterator for Tokenizer<'a> { - type Item = Token<'a>; - - fn next(&mut self) -> Option { - let mut iter = self.inner.linear_group_by(same_group_category).peekable(); - - while let (Some(string), next_string) = (iter.next(), iter.peek()) { - let (count, index) = string.char_indices().fold((0, 0), chars_count_index); - - if !is_str_word(string) { - self.word_index += string - .chars() - .filter_map(classify_separator) - .fold(Soft, |a, x| a.merge(x)) - .to_usize(); - self.char_index += count; - self.inner = &self.inner[index..]; - continue; - } - - let token = Token { - word: string, - index: self.count, - word_index: self.word_index, - char_index: self.char_index, - }; - - if next_string.filter(|s| is_str_word(s)).is_some() { - self.word_index += 1; - } - - self.count += 1; - self.char_index += count; - self.inner = &self.inner[index..]; - - return Some(token); - } - - self.inner = ""; - None - } -} - -pub struct SeqTokenizer<'a, I> -where - I: Iterator, -{ - inner: I, - current: Option>>, - count: usize, - word_offset: usize, - char_offset: usize, -} - -impl<'a, I> SeqTokenizer<'a, I> -where - I: Iterator, -{ - pub fn new(mut iter: I) -> SeqTokenizer<'a, I> { - let current = iter.next().map(|s| Tokenizer::new(s).peekable()); - SeqTokenizer { - inner: iter, - current, - count: 0, - word_offset: 0, - char_offset: 0, - } - } -} - -impl<'a, I> Iterator for SeqTokenizer<'a, I> -where - I: Iterator, -{ - type Item = Token<'a>; - - fn next(&mut self) -> Option { - match &mut self.current { - Some(current) => { - match current.next() { - Some(token) => { - // we must apply the word and char offsets - // to the token before returning it - let token = Token { - word: token.word, - index: self.count, - word_index: token.word_index + self.word_offset, - char_index: token.char_index + self.char_offset, - }; - - // if this is the last iteration on this text - // we must save the offsets for next texts - if current.peek().is_none() { - let hard_space = SeparatorCategory::Hard.to_usize(); - self.word_offset = token.word_index + hard_space; - self.char_offset = token.char_index + hard_space; - } - - Some(token) - } - None => { - // no more words in this text we must - // start tokenizing the next text - self.current = self.inner.next().map(|s| Tokenizer::new(s).peekable()); - self.next() - } - } - } - // no more texts available - None => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn easy() { - let mut tokenizer = Tokenizer::new("salut"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "salut", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!(tokenizer.next(), None); - - let mut tokenizer = Tokenizer::new("yo "); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "yo", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!(tokenizer.next(), None); - } - - #[test] - fn hard() { - let mut tokenizer = Tokenizer::new(" .? yo lolo. aïe (ouch)"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "yo", - index: 0, - word_index: 0, - char_index: 4 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lolo", - index: 1, - word_index: 1, - char_index: 7 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "aïe", - index: 2, - word_index: 9, - char_index: 13 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "ouch", - index: 3, - word_index: 17, - char_index: 18 - }) - ); - assert_eq!(tokenizer.next(), None); - - let mut tokenizer = Tokenizer::new("yo ! lolo ? wtf - lol . aïe ,"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "yo", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lolo", - index: 1, - word_index: 8, - char_index: 5 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "wtf", - index: 2, - word_index: 16, - char_index: 12 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lol", - index: 3, - word_index: 17, - char_index: 18 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "aïe", - index: 4, - word_index: 25, - char_index: 24 - }) - ); - assert_eq!(tokenizer.next(), None); - } - - #[test] - fn hard_long_chars() { - let mut tokenizer = Tokenizer::new(" .? yo 😂. aïe"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "yo", - index: 0, - word_index: 0, - char_index: 4 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "😂", - index: 1, - word_index: 1, - char_index: 7 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "aïe", - index: 2, - word_index: 9, - char_index: 10 - }) - ); - assert_eq!(tokenizer.next(), None); - - let mut tokenizer = Tokenizer::new("yo ! lolo ? 😱 - lol . 😣 ,"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "yo", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lolo", - index: 1, - word_index: 8, - char_index: 5 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "😱", - index: 2, - word_index: 16, - char_index: 12 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lol", - index: 3, - word_index: 17, - char_index: 16 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "😣", - index: 4, - word_index: 25, - char_index: 22 - }) - ); - assert_eq!(tokenizer.next(), None); - } - - #[test] - fn hard_kanjis() { - let mut tokenizer = Tokenizer::new("\u{2ec4}lolilol\u{2ec7}"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ec4}", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lolilol", - index: 1, - word_index: 1, - char_index: 1 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ec7}", - index: 2, - word_index: 2, - char_index: 8 - }) - ); - assert_eq!(tokenizer.next(), None); - - let mut tokenizer = Tokenizer::new("\u{2ec4}\u{2ed3}\u{2ef2} lolilol - hello \u{2ec7}"); - - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ec4}", - index: 0, - word_index: 0, - char_index: 0 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ed3}", - index: 1, - word_index: 1, - char_index: 1 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ef2}", - index: 2, - word_index: 2, - char_index: 2 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "lolilol", - index: 3, - word_index: 3, - char_index: 4 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "hello", - index: 4, - word_index: 4, - char_index: 14 - }) - ); - assert_eq!( - tokenizer.next(), - Some(Token { - word: "\u{2ec7}", - index: 5, - word_index: 5, - char_index: 23 - }) - ); - assert_eq!(tokenizer.next(), None); - } -} diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml deleted file mode 100644 index b3c42775c..000000000 --- a/meilisearch-types/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "meilisearch-types" -version = "0.20.0" -license = "MIT" -authors = ["Clément Renault "] -edition = "2018" - -[dependencies.zerocopy] -version = "0.3.0" -optional = true - -[dependencies.serde] -version = "1.0.118" -features = ["derive"] -optional = true - -[features] -default = ["serde", "zerocopy"] diff --git a/meilisearch-types/src/lib.rs b/meilisearch-types/src/lib.rs deleted file mode 100644 index 3e14521e0..000000000 --- a/meilisearch-types/src/lib.rs +++ /dev/null @@ -1,68 +0,0 @@ -#[cfg(feature = "zerocopy")] -use zerocopy::{AsBytes, FromBytes}; - -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; - -/// Represent an internally generated document unique identifier. -/// -/// It is used to inform the database the document you want to deserialize. -/// Helpful for custom ranking. -#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] -#[cfg_attr(feature = "zerocopy", derive(AsBytes, FromBytes))] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[repr(C)] -pub struct DocumentId(pub u32); - -/// This structure represent the position of a word -/// in a document and its attributes. -/// -/// This is stored in the map, generated at index time, -/// extracted and interpreted at search time. -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg_attr(feature = "zerocopy", derive(AsBytes, FromBytes))] -#[repr(C)] -pub struct DocIndex { - /// The document identifier where the word was found. - pub document_id: DocumentId, - - /// The attribute in the document where the word was found - /// along with the index in it. - /// This is an IndexedPos and not a FieldId. Must be converted each time. - pub attribute: u16, - pub word_index: u16, - - /// The position in bytes where the word was found - /// along with the length of it. - /// - /// It informs on the original word area in the text indexed - /// without needing to run the tokenizer again. - pub char_index: u16, - pub char_length: u16, -} - -/// This structure represent a matching word with informations -/// on the location of the word in the document. -/// -/// The order of the field is important because it defines -/// the way these structures are ordered between themselves. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg_attr(feature = "zerocopy", derive(AsBytes, FromBytes))] -#[repr(C)] -pub struct Highlight { - /// The attribute in the document where the word was found - /// along with the index in it. - pub attribute: u16, - - /// The position in bytes where the word was found. - /// - /// It informs on the original word area in the text indexed - /// without needing to run the tokenizer again. - pub char_index: u16, - - /// The length in bytes of the found word. - /// - /// It informs on the original word area in the text indexed - /// without needing to run the tokenizer again. - pub char_length: u16, -}