diff --git a/.github/scripts/is-latest-release.sh b/.github/scripts/is-latest-release.sh deleted file mode 100644 index 54f0a9d3a..000000000 --- a/.github/scripts/is-latest-release.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh - -# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions. -# Returns "true" or "false" (as a string) to be used in the `if` in GHA - -# Checks if the current tag should be the latest (in terms of semver and not of release date). -# Ex: previous tag -> v2.1.1 -# new tag -> v1.20.3 -# The new tag (v1.20.3) should NOT be the latest -# So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 - -# GLOBAL -GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] - -# FUNCTIONS - -# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh - -# usage: semverParseInto version major minor patch special -# version: the string version -# major, minor, patch, special: will be assigned by the function -semverParseInto() { - local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' - #MAJOR - eval $2=`echo $1 | sed -e "s#$RE#\1#"` - #MINOR - eval $3=`echo $1 | sed -e "s#$RE#\2#"` - #MINOR - eval $4=`echo $1 | sed -e "s#$RE#\3#"` - #SPECIAL - eval $5=`echo $1 | sed -e "s#$RE#\4#"` -} - -# usage: semverLT version1 version2 -semverLT() { - local MAJOR_A=0 - local MINOR_A=0 - local PATCH_A=0 - local SPECIAL_A=0 - - local MAJOR_B=0 - local MINOR_B=0 - local PATCH_B=0 - local SPECIAL_B=0 - - semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A - semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B - - if [ $MAJOR_A -lt $MAJOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then - return 0 - fi - if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" == "_" ] ; then - return 1 - fi - if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" != "_" ] ; then - return 1 - fi - if [ "_$SPECIAL_A" != "_" ] && [ "_$SPECIAL_B" == "_" ] ; then - return 0 - fi - if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then - return 0 - fi - - return 1 -} - -# Returns the tag of the latest stable release (in terms of semver and not of release date) -get_latest() { - temp_file='temp_file' # temp_file needed because the grep would start before the download is over - curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" - releases=$(cat "$temp_file" | \ - grep -E "tag_name|draft|prerelease" \ - | tr -d ',"' | cut -d ':' -f2 | tr -d ' ') - # Returns a list of [tag_name draft_boolean prerelease_boolean ...] - # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false... - - i=0 - latest="" - current_tag="" - for release_info in $releases; do - if [ $i -eq 0 ]; then # Checking tag_name - if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release - current_tag=$release_info - else - current_tag="" - fi - i=1 - elif [ $i -eq 1 ]; then # Checking draft boolean - if [ "$release_info" = "true" ]; then - current_tag="" - fi - i=2 - elif [ $i -eq 2 ]; then # Checking prerelease boolean - if [ "$release_info" = "true" ]; then - current_tag="" - fi - i=0 - if [ "$current_tag" != "" ]; then # If the current_tag is valid - if [ "$latest" = "" ]; then # If there is no latest yet - latest="$current_tag" - else - semverLT $current_tag $latest # Comparing latest and the current tag - if [ $? -eq 1 ]; then - latest="$current_tag" - fi - fi - fi - fi - done - - rm -f "$temp_file" - echo $latest -} - -# MAIN -current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')" -latest="$(get_latest)" - -if [ "$current_tag" != "$latest" ]; then - # The current release tag is not the latest - echo "false" -else - # The current release tag is the latest - echo "true" -fi diff --git a/.github/workflows/latest-git-tag.yml b/.github/workflows/latest-git-tag.yml index 9d9339ca8..e8cbc50f3 100644 --- a/.github/workflows/latest-git-tag.yml +++ b/.github/workflows/latest-git-tag.yml @@ -1,4 +1,4 @@ -# Create or update a latest git tag when releasing a stable vesrsin of Meilisearch +# Create or update a latest git tag when releasing a stable version of Meilisearch name: Update latest git tag on: workflow_dispatch: @@ -6,6 +6,15 @@ on: types: [released] jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check release validity + if: github.event_name == 'release' + run: bash .github/scripts/check-release.sh + update-latest-tag: runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index ecfc94666..89f55aae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a621d5d6d6c8d086dbaf1fe659981da41a1b63c6bdbba30b4dbb592c6d3bd49" +checksum = "aa0e3586af56b3bfa51fca452bd56e8dbbbd5d8d81cbf0b7e4e35b695b537eb8" dependencies = [ "serde", "toml", @@ -1101,7 +1101,7 @@ dependencies = [ [[package]] name = "dump" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "file-store" -version = "0.30.0" +version = "0.30.1" dependencies = [ "faux", "tempfile", @@ -1332,8 +1332,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "nom", "nom_locate", @@ -1351,8 +1351,8 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -1625,7 +1625,7 @@ dependencies = [ "libc", "lmdb-rkv-sys", "once_cell", - "page_size", + "page_size 0.4.2", "synchronoise", "url", "zerocopy", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -1783,6 +1783,7 @@ dependencies = [ "meili-snap", "meilisearch-types", "nelson", + "page_size 0.5.0", "roaring", "serde", "serde_json", @@ -1897,8 +1898,8 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -2257,7 +2258,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" dependencies = [ "insta", "md5", @@ -2266,7 +2267,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" dependencies = [ "enum-iterator", "hmac", @@ -2283,7 +2284,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-cors", "actix-http", @@ -2366,7 +2367,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-web", "anyhow", @@ -2417,8 +2418,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "bimap", "bincode", @@ -2664,6 +2665,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "page_size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2748,7 +2759,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" dependencies = [ "big_s", "serde_json", diff --git a/dump/Cargo.toml b/dump/Cargo.toml index c5dc10949..f6850f71c 100644 --- a/dump/Cargo.toml +++ b/dump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dump" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/dump/src/writer.rs b/dump/src/writer.rs index c11cf6b9a..81695f55c 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -292,10 +292,10 @@ pub(crate) mod test { // ==== checking the top level infos let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap(); let metadata: Metadata = serde_json::from_str(&metadata).unwrap(); - insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" + insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]", ".dbVersion" => "[version]" }, @r###" { "dumpVersion": "V6", - "dbVersion": "0.30.0", + "dbVersion": "[version]", "dumpDate": "[date]" } "###); diff --git a/file-store/Cargo.toml b/file-store/Cargo.toml index 0110a00b1..3adef3315 100644 --- a/file-store/Cargo.toml +++ b/file-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-store" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index e46712703..fa324d98b 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] @@ -13,6 +13,7 @@ enum-iterator = "1.1.3" file-store = { path = "../file-store" } log = "0.4.14" meilisearch-types = { path = "../meilisearch-types" } +page_size = "0.5.0" roaring = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.85", features = ["preserve_order"] } diff --git a/index-scheduler/src/index_mapper.rs b/index-scheduler/src/index_mapper.rs index 8869591f7..c65d76837 100644 --- a/index-scheduler/src/index_mapper.rs +++ b/index-scheduler/src/index_mapper.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use self::IndexStatus::{Available, BeingDeleted}; use crate::uuid_codec::UuidCodec; -use crate::{Error, Result}; +use crate::{clamp_to_page_size, Error, Result}; const INDEX_MAPPING: &str = "index-mapping"; @@ -68,7 +68,7 @@ impl IndexMapper { /// The path *must* exists or an error will be thrown. fn create_or_open_index(&self, path: &Path) -> Result { let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(clamp_to_page_size(self.index_size)); options.max_readers(1024); Ok(Index::new(options, path)?) } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index a7dab68d1..3ad546eb4 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m use uuid::Uuid; use crate::index_mapper::IndexMapper; -use crate::utils::check_index_swap_validity; +use crate::utils::{check_index_swap_validity, clamp_to_page_size}; pub(crate) type BEI128 = meilisearch_types::heed::zerocopy::I128; @@ -362,7 +362,7 @@ impl IndexScheduler { let env = heed::EnvOpenOptions::new() .max_dbs(10) - .map_size(options.task_db_size) + .map_size(clamp_to_page_size(options.task_db_size)) .open(options.tasks_path)?; let file_store = FileStore::new(&options.update_file_path)?; @@ -1114,8 +1114,8 @@ mod tests { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), - task_db_size: 1024 * 1024, // 1 MiB - index_size: 1024 * 1024, // 1 MiB + task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. + index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. indexer_config: IndexerConfig::default(), autobatching_enabled, }; diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index a193c2bec..0018ae1d0 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { Ok(()) } +/// Clamp the provided value to be a multiple of system page size. +pub fn clamp_to_page_size(size: usize) -> usize { + size / page_size::get() * page_size::get() +} + #[cfg(test)] impl IndexScheduler { /// Asserts that the index scheduler's content is internally consistent. diff --git a/meili-snap/Cargo.toml b/meili-snap/Cargo.toml index 6706bf693..1682ddc11 100644 --- a/meili-snap/Cargo.toml +++ b/meili-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index f0b73b539..1df43411e 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 1d451f9d0..7ffbd483b 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" [[bin]] name = "meilisearch" @@ -89,7 +89,7 @@ yaup = "0.2.1" [build-dependencies] anyhow = { version = "1.0.65", optional = true } -cargo_toml = { version = "0.12.4", optional = true } +cargo_toml = { version = "0.13.0", optional = true } hex = { version = "0.4.3", optional = true } reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true } sha-1 = { version = "0.10.0", optional = true } diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 7bc66a37e..f499aeaae 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" authors = ["marin "] edition = "2021" @@ -13,7 +13,7 @@ enum-iterator = "1.1.3" flate2 = "1.0.24" fst = "0.4.7" memmap2 = "0.5.7" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false } proptest = { version = "1.0.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } roaring = { version = "0.10.0", features = ["serde"] } diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index b036a0c88..3343ee41f 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" edition = "2021" description = "A permissive json pointer" readme = "README.md"