From c1fa3e81a0c32db70e236c97dbebf3e13a355991 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 11 Mar 2025 14:52:46 +0100 Subject: [PATCH] Use the new arroy upgrade method to move from 0.4 to 0.5 --- Cargo.lock | 103 ++++------------------ crates/index-scheduler/src/lib.rs | 9 +- crates/index-scheduler/src/upgrade/mod.rs | 2 +- crates/meilitool/Cargo.toml | 1 - crates/meilitool/src/upgrade/v1_11.rs | 2 +- crates/milli/Cargo.toml | 2 +- crates/milli/src/lib.rs | 2 + 7 files changed, 30 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00e647072..9d27c6ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -395,30 +395,11 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arroy" version = "0.5.0" -source = "git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05#053807bf38dc079f25b003f19fc30fbf3613f6e7" +source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#3b307a0e042c7527396ac36e6e88fc14372482da" dependencies = [ "bytemuck", "byteorder", - "heed 0.20.5", - "log", - "memmap2", - "nohash", - "ordered-float", - "rand 0.8.5", - "rayon", - "roaring", - "tempfile", - "thiserror 1.0.69", -] - -[[package]] -name = "arroy" -version = "0.5.0" -source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#010f9f9e43327af6e4f7088147730f00f4901574" -dependencies = [ - "bytemuck", - "byteorder", - "heed 0.22.0", + "heed", "log", "memmap2", "nohash", @@ -1898,7 +1879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2567,35 +2548,17 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "heed" -version = "0.20.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" -dependencies = [ - "bitflags 2.9.0", - "byteorder", - "heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heed-types 0.20.1", - "libc", - "lmdb-master-sys 0.2.4", - "once_cell", - "page_size", - "synchronoise", - "url", -] - [[package]] name = "heed" version = "0.22.0" -source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" +source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5" dependencies = [ "bitflags 2.9.0", "byteorder", - "heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)", - "heed-types 0.21.0", + "heed-traits", + "heed-types", "libc", - "lmdb-master-sys 0.2.5", + "lmdb-master-sys", "once_cell", "page_size", "synchronoise", @@ -2605,32 +2568,16 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" - -[[package]] -name = "heed-traits" -version = "0.20.0" -source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" - -[[package]] -name = "heed-types" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" -dependencies = [ - "byteorder", - "heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", -] +source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5" [[package]] name = "heed-types" version = "0.21.0" -source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" +source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5" dependencies = [ "bincode", "byteorder", - "heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)", + "heed-traits", "serde", "serde_json", ] @@ -2972,7 +2919,7 @@ name = "index-scheduler" version = "1.13.3" dependencies = [ "anyhow", - "arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)", + "arroy", "big_s", "bincode", "bumpalo", @@ -3087,7 +3034,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi 0.5.0", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3749,21 +3696,10 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" -[[package]] -name = "lmdb-master-sys" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9" -dependencies = [ - "cc", - "doxygen-rs", - "libc", -] - [[package]] name = "lmdb-master-sys" version = "0.2.5" -source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" +source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5" dependencies = [ "cc", "doxygen-rs", @@ -4018,7 +3954,6 @@ name = "meilitool" version = "1.13.3" dependencies = [ "anyhow", - "arroy 0.5.0 (git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05)", "clap", "dump", "file-store", @@ -4053,7 +3988,7 @@ name = "milli" version = "1.13.3" dependencies = [ "allocator-api2", - "arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)", + "arroy", "bbqueue", "big_s", "bimap", @@ -4082,7 +4017,7 @@ dependencies = [ "geoutils", "grenad", "hashbrown 0.15.2", - "heed 0.22.0", + "heed", "hf-hub", "indexmap", "insta", @@ -4906,7 +4841,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5320,7 +5255,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5333,7 +5268,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.2", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5898,7 +5833,7 @@ dependencies = [ "getrandom 0.3.1", "once_cell", "rustix 1.0.2", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 41c407494..4605be6eb 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -240,9 +240,12 @@ impl IndexScheduler { }; let env = unsafe { - let options = heed::EnvOpenOptions::new(); - let mut options = options.read_txn_without_tls(); - options.max_dbs(Self::nb_db()).map_size(budget.task_db_size).open(&options.tasks_path) + let env_options = heed::EnvOpenOptions::new(); + let mut env_options = env_options.read_txn_without_tls(); + env_options + .max_dbs(Self::nb_db()) + .map_size(budget.task_db_size) + .open(&options.tasks_path) }?; // We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index 596d9a0c0..86ca755b8 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -1,5 +1,5 @@ use anyhow::bail; -use meilisearch_types::heed::{Env, RwTxn, WithTls, WithoutTls}; +use meilisearch_types::heed::{Env, RwTxn, WithoutTls}; use meilisearch_types::tasks::{Details, KindWithContent, Status, Task}; use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use time::OffsetDateTime; diff --git a/crates/meilitool/Cargo.toml b/crates/meilitool/Cargo.toml index ffd13da34..485177838 100644 --- a/crates/meilitool/Cargo.toml +++ b/crates/meilitool/Cargo.toml @@ -10,7 +10,6 @@ license.workspace = true [dependencies] anyhow = "1.0.95" -arroy_v04_to_v05 = { package = "arroy", git = "https://github.com/meilisearch/arroy/", tag = "DO-NOT-DELETE-upgrade-v04-to-v05" } clap = { version = "4.5.24", features = ["derive"] } dump = { path = "../dump" } file-store = { path = "../file-store" } diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index 44aeb125f..dea768dd5 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -76,7 +76,7 @@ pub fn v1_10_to_v1_11( try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY) .with_context(|| format!("while updating date format for index `{uid}`"))?; - arroy_v04_to_v05::ugrade_from_prev_version( + meilisearch_types::milli::arroy::upgrade::cosine_from_0_4_to_0_5( &index_rtxn, index_read_database, &mut index_wtxn, diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 5f4da149b..8985f2bb7 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -32,7 +32,7 @@ grenad = { version = "0.5.0", default-features = false, features = [ "rayon", "tempfile", ] } -heed = { version = "0.22.0", branch = "bump-versions", git = "https://github.com/meilisearch/heed", default-features = false, features = [ +heed = { version = "0.22.0", branch = "main", git = "https://github.com/meilisearch/heed", default-features = false, features = [ "serde-json", "serde-bincode", ] } diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index 1d6d04fc7..a6219bcd9 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -76,6 +76,8 @@ pub use self::search::{ }; pub use self::update::ChannelCongestion; +pub use arroy; + pub type Result = std::result::Result; pub type Attribute = u32;