Merge branch 'fix-3037' of github.com:jiangbo212/meilisearch into fix-3037

This commit is contained in:
jiangbo212 2022-12-07 22:54:16 +08:00
commit 717dd36547
15 changed files with 64 additions and 170 deletions

View File

@ -1,132 +0,0 @@
#!/bin/sh
# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions.
# Returns "true" or "false" (as a string) to be used in the `if` in GHA
# Checks if the current tag should be the latest (in terms of semver and not of release date).
# Ex: previous tag -> v2.1.1
# new tag -> v1.20.3
# The new tag (v1.20.3) should NOT be the latest
# So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1
# GLOBAL
GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
# FUNCTIONS
# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
# usage: semverParseInto version major minor patch special
# version: the string version
# major, minor, patch, special: will be assigned by the function
semverParseInto() {
local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
#MAJOR
eval $2=`echo $1 | sed -e "s#$RE#\1#"`
#MINOR
eval $3=`echo $1 | sed -e "s#$RE#\2#"`
#MINOR
eval $4=`echo $1 | sed -e "s#$RE#\3#"`
#SPECIAL
eval $5=`echo $1 | sed -e "s#$RE#\4#"`
}
# usage: semverLT version1 version2
semverLT() {
local MAJOR_A=0
local MINOR_A=0
local PATCH_A=0
local SPECIAL_A=0
local MAJOR_B=0
local MINOR_B=0
local PATCH_B=0
local SPECIAL_B=0
semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
if [ $MAJOR_A -lt $MAJOR_B ]; then
return 0
fi
if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
return 0
fi
if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
return 0
fi
if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" == "_" ] ; then
return 1
fi
if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" != "_" ] ; then
return 1
fi
if [ "_$SPECIAL_A" != "_" ] && [ "_$SPECIAL_B" == "_" ] ; then
return 0
fi
if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
return 0
fi
return 1
}
# Returns the tag of the latest stable release (in terms of semver and not of release date)
get_latest() {
temp_file='temp_file' # temp_file needed because the grep would start before the download is over
curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file"
releases=$(cat "$temp_file" | \
grep -E "tag_name|draft|prerelease" \
| tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
# Returns a list of [tag_name draft_boolean prerelease_boolean ...]
# Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
i=0
latest=""
current_tag=""
for release_info in $releases; do
if [ $i -eq 0 ]; then # Checking tag_name
if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
current_tag=$release_info
else
current_tag=""
fi
i=1
elif [ $i -eq 1 ]; then # Checking draft boolean
if [ "$release_info" = "true" ]; then
current_tag=""
fi
i=2
elif [ $i -eq 2 ]; then # Checking prerelease boolean
if [ "$release_info" = "true" ]; then
current_tag=""
fi
i=0
if [ "$current_tag" != "" ]; then # If the current_tag is valid
if [ "$latest" = "" ]; then # If there is no latest yet
latest="$current_tag"
else
semverLT $current_tag $latest # Comparing latest and the current tag
if [ $? -eq 1 ]; then
latest="$current_tag"
fi
fi
fi
fi
done
rm -f "$temp_file"
echo $latest
}
# MAIN
current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
latest="$(get_latest)"
if [ "$current_tag" != "$latest" ]; then
# The current release tag is not the latest
echo "false"
else
# The current release tag is the latest
echo "true"
fi

View File

@ -1,4 +1,4 @@
# Create or update a latest git tag when releasing a stable vesrsin of Meilisearch # Create or update a latest git tag when releasing a stable version of Meilisearch
name: Update latest git tag name: Update latest git tag
on: on:
workflow_dispatch: workflow_dispatch:
@ -6,6 +6,15 @@ on:
types: [released] types: [released]
jobs: jobs:
check-version:
name: Check the version validity
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Check release validity
if: github.event_name == 'release'
run: bash .github/scripts/check-release.sh
update-latest-tag: update-latest-tag:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

49
Cargo.lock generated
View File

@ -617,9 +617,9 @@ dependencies = [
[[package]] [[package]]
name = "cargo_toml" name = "cargo_toml"
version = "0.12.4" version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a621d5d6d6c8d086dbaf1fe659981da41a1b63c6bdbba30b4dbb592c6d3bd49" checksum = "aa0e3586af56b3bfa51fca452bd56e8dbbbd5d8d81cbf0b7e4e35b695b537eb8"
dependencies = [ dependencies = [
"serde", "serde",
"toml", "toml",
@ -1101,7 +1101,7 @@ dependencies = [
[[package]] [[package]]
name = "dump" name = "dump"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@ -1310,7 +1310,7 @@ dependencies = [
[[package]] [[package]]
name = "file-store" name = "file-store"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"faux", "faux",
"tempfile", "tempfile",
@ -1332,8 +1332,8 @@ dependencies = [
[[package]] [[package]]
name = "filter-parser" name = "filter-parser"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"nom", "nom",
"nom_locate", "nom_locate",
@ -1351,8 +1351,8 @@ dependencies = [
[[package]] [[package]]
name = "flatten-serde-json" name = "flatten-serde-json"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -1625,7 +1625,7 @@ dependencies = [
"libc", "libc",
"lmdb-rkv-sys", "lmdb-rkv-sys",
"once_cell", "once_cell",
"page_size", "page_size 0.4.2",
"synchronoise", "synchronoise",
"url", "url",
"zerocopy", "zerocopy",
@ -1767,7 +1767,7 @@ dependencies = [
[[package]] [[package]]
name = "index-scheduler" name = "index-scheduler"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@ -1783,6 +1783,7 @@ dependencies = [
"meili-snap", "meili-snap",
"meilisearch-types", "meilisearch-types",
"nelson", "nelson",
"page_size 0.5.0",
"roaring", "roaring",
"serde", "serde",
"serde_json", "serde_json",
@ -1897,8 +1898,8 @@ dependencies = [
[[package]] [[package]]
name = "json-depth-checker" name = "json-depth-checker"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -2257,7 +2258,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]] [[package]]
name = "meili-snap" name = "meili-snap"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"insta", "insta",
"md5", "md5",
@ -2266,7 +2267,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-auth" name = "meilisearch-auth"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"enum-iterator", "enum-iterator",
"hmac", "hmac",
@ -2283,7 +2284,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-http" name = "meilisearch-http"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-http", "actix-http",
@ -2366,7 +2367,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-types" name = "meilisearch-types"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"actix-web", "actix-web",
"anyhow", "anyhow",
@ -2417,8 +2418,8 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.37.0" version = "0.37.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
dependencies = [ dependencies = [
"bimap", "bimap",
"bincode", "bincode",
@ -2664,6 +2665,16 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "page_size"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"
@ -2748,7 +2759,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]] [[package]]
name = "permissive-json-pointer" name = "permissive-json-pointer"
version = "0.30.0" version = "0.30.1"
dependencies = [ dependencies = [
"big_s", "big_s",
"serde_json", "serde_json",

View File

@ -1,6 +1,6 @@
[package] [package]
name = "dump" name = "dump"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

View File

@ -292,10 +292,10 @@ pub(crate) mod test {
// ==== checking the top level infos // ==== checking the top level infos
let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap(); let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap();
let metadata: Metadata = serde_json::from_str(&metadata).unwrap(); let metadata: Metadata = serde_json::from_str(&metadata).unwrap();
insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]", ".dbVersion" => "[version]" }, @r###"
{ {
"dumpVersion": "V6", "dumpVersion": "V6",
"dbVersion": "0.30.0", "dbVersion": "[version]",
"dumpDate": "[date]" "dumpDate": "[date]"
} }
"###); "###);

View File

@ -1,6 +1,6 @@
[package] [package]
name = "file-store" name = "file-store"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

View File

@ -1,6 +1,6 @@
[package] [package]
name = "index-scheduler" name = "index-scheduler"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
@ -13,6 +13,7 @@ enum-iterator = "1.1.3"
file-store = { path = "../file-store" } file-store = { path = "../file-store" }
log = "0.4.14" log = "0.4.14"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
page_size = "0.5.0"
roaring = { version = "0.10.0", features = ["serde"] } roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] } serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] } serde_json = { version = "1.0.85", features = ["preserve_order"] }

View File

@ -13,7 +13,7 @@ use uuid::Uuid;
use self::IndexStatus::{Available, BeingDeleted}; use self::IndexStatus::{Available, BeingDeleted};
use crate::uuid_codec::UuidCodec; use crate::uuid_codec::UuidCodec;
use crate::{Error, Result}; use crate::{clamp_to_page_size, Error, Result};
const INDEX_MAPPING: &str = "index-mapping"; const INDEX_MAPPING: &str = "index-mapping";
@ -68,7 +68,7 @@ impl IndexMapper {
/// The path *must* exists or an error will be thrown. /// The path *must* exists or an error will be thrown.
fn create_or_open_index(&self, path: &Path) -> Result<Index> { fn create_or_open_index(&self, path: &Path) -> Result<Index> {
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(self.index_size); options.map_size(clamp_to_page_size(self.index_size));
options.max_readers(1024); options.max_readers(1024);
Ok(Index::new(options, path)?) Ok(Index::new(options, path)?)
} }

View File

@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m
use uuid::Uuid; use uuid::Uuid;
use crate::index_mapper::IndexMapper; use crate::index_mapper::IndexMapper;
use crate::utils::check_index_swap_validity; use crate::utils::{check_index_swap_validity, clamp_to_page_size};
pub(crate) type BEI128 = pub(crate) type BEI128 =
meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>; meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
@ -362,7 +362,7 @@ impl IndexScheduler {
let env = heed::EnvOpenOptions::new() let env = heed::EnvOpenOptions::new()
.max_dbs(10) .max_dbs(10)
.map_size(options.task_db_size) .map_size(clamp_to_page_size(options.task_db_size))
.open(options.tasks_path)?; .open(options.tasks_path)?;
let file_store = FileStore::new(&options.update_file_path)?; let file_store = FileStore::new(&options.update_file_path)?;
@ -1114,8 +1114,8 @@ mod tests {
indexes_path: tempdir.path().join("indexes"), indexes_path: tempdir.path().join("indexes"),
snapshots_path: tempdir.path().join("snapshots"), snapshots_path: tempdir.path().join("snapshots"),
dumps_path: tempdir.path().join("dumps"), dumps_path: tempdir.path().join("dumps"),
task_db_size: 1024 * 1024, // 1 MiB task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
index_size: 1024 * 1024, // 1 MiB index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
indexer_config: IndexerConfig::default(), indexer_config: IndexerConfig::default(),
autobatching_enabled, autobatching_enabled,
}; };

View File

@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> {
Ok(()) Ok(())
} }
/// Clamp the provided value to be a multiple of system page size.
pub fn clamp_to_page_size(size: usize) -> usize {
size / page_size::get() * page_size::get()
}
#[cfg(test)] #[cfg(test)]
impl IndexScheduler { impl IndexScheduler {
/// Asserts that the index scheduler's content is internally consistent. /// Asserts that the index scheduler's content is internally consistent.

View File

@ -1,6 +1,6 @@
[package] [package]
name = "meili-snap" name = "meili-snap"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

View File

@ -1,6 +1,6 @@
[package] [package]
name = "meilisearch-auth" name = "meilisearch-auth"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
[dependencies] [dependencies]

View File

@ -4,7 +4,7 @@ description = "Meilisearch HTTP server"
edition = "2021" edition = "2021"
license = "MIT" license = "MIT"
name = "meilisearch-http" name = "meilisearch-http"
version = "0.30.0" version = "0.30.1"
[[bin]] [[bin]]
name = "meilisearch" name = "meilisearch"
@ -89,7 +89,7 @@ yaup = "0.2.1"
[build-dependencies] [build-dependencies]
anyhow = { version = "1.0.65", optional = true } anyhow = { version = "1.0.65", optional = true }
cargo_toml = { version = "0.12.4", optional = true } cargo_toml = { version = "0.13.0", optional = true }
hex = { version = "0.4.3", optional = true } hex = { version = "0.4.3", optional = true }
reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true } reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
sha-1 = { version = "0.10.0", optional = true } sha-1 = { version = "0.10.0", optional = true }

View File

@ -1,6 +1,6 @@
[package] [package]
name = "meilisearch-types" name = "meilisearch-types"
version = "0.30.0" version = "0.30.1"
authors = ["marin <postma.marin@protonmail.com>"] authors = ["marin <postma.marin@protonmail.com>"]
edition = "2021" edition = "2021"
@ -13,7 +13,7 @@ enum-iterator = "1.1.3"
flate2 = "1.0.24" flate2 = "1.0.24"
fst = "0.4.7" fst = "0.4.7"
memmap2 = "0.5.7" memmap2 = "0.5.7"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false } milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false }
proptest = { version = "1.0.0", optional = true } proptest = { version = "1.0.0", optional = true }
proptest-derive = { version = "0.3.0", optional = true } proptest-derive = { version = "0.3.0", optional = true }
roaring = { version = "0.10.0", features = ["serde"] } roaring = { version = "0.10.0", features = ["serde"] }

View File

@ -1,6 +1,6 @@
[package] [package]
name = "permissive-json-pointer" name = "permissive-json-pointer"
version = "0.30.0" version = "0.30.1"
edition = "2021" edition = "2021"
description = "A permissive json pointer" description = "A permissive json pointer"
readme = "README.md" readme = "README.md"