searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-03-12 19:11:42 +01:00
Code Issues Releases Wiki Activity

Use the new arroy upgrade method to move from 0.4 to 0.5

Browse Source
This commit is contained in:
Kerollmops 2025-03-11 14:52:46 +01:00
parent c63dae8267
commit c1fa3e81a0
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
7 changed files with 30 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
Cargo.lock generated
View File

@ -395,30 +395,11 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arroy"
version = "0.5.0"
source = "git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05#053807bf38dc079f25b003f19fc30fbf3613f6e7"
source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#3b307a0e042c7527396ac36e6e88fc14372482da"
dependencies = [
"bytemuck",
"byteorder",
"heed 0.20.5",
"log",
"memmap2",
"nohash",
"ordered-float",
"rand 0.8.5",
"rayon",
"roaring",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "arroy"
version = "0.5.0"
source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#010f9f9e43327af6e4f7088147730f00f4901574"
dependencies = [
"bytemuck",
"byteorder",
"heed 0.22.0",
"heed",
"log",
"memmap2",
"nohash",
@ -1898,7 +1879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@ -2567,35 +2548,17 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heed"
version = "0.20.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb"
dependencies = [
"bitflags 2.9.0",
"byteorder",
"heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"heed-types 0.20.1",
"libc",
"lmdb-master-sys 0.2.4",
"once_cell",
"page_size",
"synchronoise",
"url",
]
[[package]]
name = "heed"
version = "0.22.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a"
source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [
"bitflags 2.9.0",
"byteorder",
"heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)",
"heed-types 0.21.0",
"heed-traits",
"heed-types",
"libc",
"lmdb-master-sys 0.2.5",
"lmdb-master-sys",
"once_cell",
"page_size",
"synchronoise",
@ -2605,32 +2568,16 @@ dependencies = [
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a"
[[package]]
name = "heed-types"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114"
dependencies = [
"byteorder",
"heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
[[package]]
name = "heed-types"
version = "0.21.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a"
source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [
"bincode",
"byteorder",
"heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)",
"heed-traits",
"serde",
"serde_json",
]
@ -2972,7 +2919,7 @@ name = "index-scheduler"
version = "1.13.3"
dependencies = [
"anyhow",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)",
"arroy",
"big_s",
"bincode",
"bumpalo",
@ -3087,7 +3034,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi 0.5.0",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@ -3749,21 +3696,10 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
[[package]]
name = "lmdb-master-sys"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9"
dependencies = [
"cc",
"doxygen-rs",
"libc",
]
[[package]]
name = "lmdb-master-sys"
version = "0.2.5"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a"
source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [
"cc",
"doxygen-rs",
@ -4018,7 +3954,6 @@ name = "meilitool"
version = "1.13.3"
dependencies = [
"anyhow",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05)",
"clap",
"dump",
"file-store",
@ -4053,7 +3988,7 @@ name = "milli"
version = "1.13.3"
dependencies = [
"allocator-api2",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)",
"arroy",
"bbqueue",
"big_s",
"bimap",
@ -4082,7 +4017,7 @@ dependencies = [
"geoutils",
"grenad",
"hashbrown 0.15.2",
"heed 0.22.0",
"heed",
"hf-hub",
"indexmap",
"insta",
@ -4906,7 +4841,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@ -5320,7 +5255,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@ -5333,7 +5268,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.9.2",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@ -5898,7 +5833,7 @@ dependencies = [
"getrandom 0.3.1",
"once_cell",
"rustix 1.0.2",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]

9
crates/index-scheduler/src/lib.rs
View File

@ -240,9 +240,12 @@ impl IndexScheduler {
};
let env = unsafe {
let options = heed::EnvOpenOptions::new();
let mut options = options.read_txn_without_tls();
options.max_dbs(Self::nb_db()).map_size(budget.task_db_size).open(&options.tasks_path)
let env_options = heed::EnvOpenOptions::new();
let mut env_options = env_options.read_txn_without_tls();
env_options
.max_dbs(Self::nb_db())
.map_size(budget.task_db_size)
.open(&options.tasks_path)
}?;
// We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them

2
crates/index-scheduler/src/upgrade/mod.rs
View File

@ -1,5 +1,5 @@
use anyhow::bail;
use meilisearch_types::heed::{Env, RwTxn, WithTls, WithoutTls};
use meilisearch_types::heed::{Env, RwTxn, WithoutTls};
use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
use time::OffsetDateTime;

1
crates/meilitool/Cargo.toml
View File

@ -10,7 +10,6 @@ license.workspace = true
[dependencies]
anyhow = "1.0.95"
arroy_v04_to_v05 = { package = "arroy", git = "https://github.com/meilisearch/arroy/", tag = "DO-NOT-DELETE-upgrade-v04-to-v05" }
clap = { version = "4.5.24", features = ["derive"] }
dump = { path = "../dump" }
file-store = { path = "../file-store" }

2
crates/meilitool/src/upgrade/v1_11.rs
View File

@ -76,7 +76,7 @@ pub fn v1_10_to_v1_11(
try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY)
.with_context(|| format!("while updating date format for index `{uid}`"))?;
arroy_v04_to_v05::ugrade_from_prev_version(
meilisearch_types::milli::arroy::upgrade::cosine_from_0_4_to_0_5(
&index_rtxn,
index_read_database,
&mut index_wtxn,

2
crates/milli/Cargo.toml
View File

@ -32,7 +32,7 @@ grenad = { version = "0.5.0", default-features = false, features = [
"rayon",
"tempfile",
] }
heed = { version = "0.22.0", branch = "bump-versions", git = "https://github.com/meilisearch/heed", default-features = false, features = [
heed = { version = "0.22.0", branch = "main", git = "https://github.com/meilisearch/heed", default-features = false, features = [
"serde-json",
"serde-bincode",
] }

2
crates/milli/src/lib.rs
View File

@ -76,6 +76,8 @@ pub use self::search::{
};
pub use self::update::ChannelCongestion;
pub use arroy;
pub type Result<T> = std::result::Result<T, error::Error>;
pub type Attribute = u32;