Use the new arroy upgrade method to move from 0.4 to 0.5

This commit is contained in:
Kerollmops 2025-03-11 14:52:46 +01:00
parent c63dae8267
commit c1fa3e81a0
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
7 changed files with 30 additions and 91 deletions

103
Cargo.lock generated
View File

@ -395,30 +395,11 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]] [[package]]
name = "arroy" name = "arroy"
version = "0.5.0" version = "0.5.0"
source = "git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05#053807bf38dc079f25b003f19fc30fbf3613f6e7" source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#3b307a0e042c7527396ac36e6e88fc14372482da"
dependencies = [ dependencies = [
"bytemuck", "bytemuck",
"byteorder", "byteorder",
"heed 0.20.5", "heed",
"log",
"memmap2",
"nohash",
"ordered-float",
"rand 0.8.5",
"rayon",
"roaring",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "arroy"
version = "0.5.0"
source = "git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22#010f9f9e43327af6e4f7088147730f00f4901574"
dependencies = [
"bytemuck",
"byteorder",
"heed 0.22.0",
"log", "log",
"memmap2", "memmap2",
"nohash", "nohash",
@ -1898,7 +1879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -2567,35 +2548,17 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "heed"
version = "0.20.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb"
dependencies = [
"bitflags 2.9.0",
"byteorder",
"heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"heed-types 0.20.1",
"libc",
"lmdb-master-sys 0.2.4",
"once_cell",
"page_size",
"synchronoise",
"url",
]
[[package]] [[package]]
name = "heed" name = "heed"
version = "0.22.0" version = "0.22.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [ dependencies = [
"bitflags 2.9.0", "bitflags 2.9.0",
"byteorder", "byteorder",
"heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)", "heed-traits",
"heed-types 0.21.0", "heed-types",
"libc", "libc",
"lmdb-master-sys 0.2.5", "lmdb-master-sys",
"once_cell", "once_cell",
"page_size", "page_size",
"synchronoise", "synchronoise",
@ -2605,32 +2568,16 @@ dependencies = [
[[package]] [[package]]
name = "heed-traits" name = "heed-traits"
version = "0.20.0" version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-traits"
version = "0.20.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a"
[[package]]
name = "heed-types"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114"
dependencies = [
"byteorder",
"heed-traits 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "heed-types" name = "heed-types"
version = "0.21.0" version = "0.21.0"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [ dependencies = [
"bincode", "bincode",
"byteorder", "byteorder",
"heed-traits 0.20.0 (git+https://github.com/meilisearch/heed?branch=bump-versions)", "heed-traits",
"serde", "serde",
"serde_json", "serde_json",
] ]
@ -2972,7 +2919,7 @@ name = "index-scheduler"
version = "1.13.3" version = "1.13.3"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)", "arroy",
"big_s", "big_s",
"bincode", "bincode",
"bumpalo", "bumpalo",
@ -3087,7 +3034,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [ dependencies = [
"hermit-abi 0.5.0", "hermit-abi 0.5.0",
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -3749,21 +3696,10 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
[[package]]
name = "lmdb-master-sys"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9"
dependencies = [
"cc",
"doxygen-rs",
"libc",
]
[[package]] [[package]]
name = "lmdb-master-sys" name = "lmdb-master-sys"
version = "0.2.5" version = "0.2.5"
source = "git+https://github.com/meilisearch/heed?branch=bump-versions#a778043eb1768c7885828f6eaa9a668a64c5950a" source = "git+https://github.com/meilisearch/heed?branch=main#a3c6f002925630d4e5c808e58e62a121887094f5"
dependencies = [ dependencies = [
"cc", "cc",
"doxygen-rs", "doxygen-rs",
@ -4018,7 +3954,6 @@ name = "meilitool"
version = "1.13.3" version = "1.13.3"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05)",
"clap", "clap",
"dump", "dump",
"file-store", "file-store",
@ -4053,7 +3988,7 @@ name = "milli"
version = "1.13.3" version = "1.13.3"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
"arroy 0.5.0 (git+https://github.com/meilisearch/arroy?branch=update-heed-to-0-22)", "arroy",
"bbqueue", "bbqueue",
"big_s", "big_s",
"bimap", "bimap",
@ -4082,7 +4017,7 @@ dependencies = [
"geoutils", "geoutils",
"grenad", "grenad",
"hashbrown 0.15.2", "hashbrown 0.15.2",
"heed 0.22.0", "heed",
"hf-hub", "hf-hub",
"indexmap", "indexmap",
"insta", "insta",
@ -4906,7 +4841,7 @@ dependencies = [
"once_cell", "once_cell",
"socket2", "socket2",
"tracing", "tracing",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -5320,7 +5255,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.4.15", "linux-raw-sys 0.4.15",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -5333,7 +5268,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.9.2", "linux-raw-sys 0.9.2",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -5898,7 +5833,7 @@ dependencies = [
"getrandom 0.3.1", "getrandom 0.3.1",
"once_cell", "once_cell",
"rustix 1.0.2", "rustix 1.0.2",
"windows-sys 0.59.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]

View File

@ -240,9 +240,12 @@ impl IndexScheduler {
}; };
let env = unsafe { let env = unsafe {
let options = heed::EnvOpenOptions::new(); let env_options = heed::EnvOpenOptions::new();
let mut options = options.read_txn_without_tls(); let mut env_options = env_options.read_txn_without_tls();
options.max_dbs(Self::nb_db()).map_size(budget.task_db_size).open(&options.tasks_path) env_options
.max_dbs(Self::nb_db())
.map_size(budget.task_db_size)
.open(&options.tasks_path)
}?; }?;
// We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them // We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them

View File

@ -1,5 +1,5 @@
use anyhow::bail; use anyhow::bail;
use meilisearch_types::heed::{Env, RwTxn, WithTls, WithoutTls}; use meilisearch_types::heed::{Env, RwTxn, WithoutTls};
use meilisearch_types::tasks::{Details, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
use time::OffsetDateTime; use time::OffsetDateTime;

View File

@ -10,7 +10,6 @@ license.workspace = true
[dependencies] [dependencies]
anyhow = "1.0.95" anyhow = "1.0.95"
arroy_v04_to_v05 = { package = "arroy", git = "https://github.com/meilisearch/arroy/", tag = "DO-NOT-DELETE-upgrade-v04-to-v05" }
clap = { version = "4.5.24", features = ["derive"] } clap = { version = "4.5.24", features = ["derive"] }
dump = { path = "../dump" } dump = { path = "../dump" }
file-store = { path = "../file-store" } file-store = { path = "../file-store" }

View File

@ -76,7 +76,7 @@ pub fn v1_10_to_v1_11(
try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY) try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY)
.with_context(|| format!("while updating date format for index `{uid}`"))?; .with_context(|| format!("while updating date format for index `{uid}`"))?;
arroy_v04_to_v05::ugrade_from_prev_version( meilisearch_types::milli::arroy::upgrade::cosine_from_0_4_to_0_5(
&index_rtxn, &index_rtxn,
index_read_database, index_read_database,
&mut index_wtxn, &mut index_wtxn,

View File

@ -32,7 +32,7 @@ grenad = { version = "0.5.0", default-features = false, features = [
"rayon", "rayon",
"tempfile", "tempfile",
] } ] }
heed = { version = "0.22.0", branch = "bump-versions", git = "https://github.com/meilisearch/heed", default-features = false, features = [ heed = { version = "0.22.0", branch = "main", git = "https://github.com/meilisearch/heed", default-features = false, features = [
"serde-json", "serde-json",
"serde-bincode", "serde-bincode",
] } ] }

View File

@ -76,6 +76,8 @@ pub use self::search::{
}; };
pub use self::update::ChannelCongestion; pub use self::update::ChannelCongestion;
pub use arroy;
pub type Result<T> = std::result::Result<T, error::Error>; pub type Result<T> = std::result::Result<T, error::Error>;
pub type Attribute = u32; pub type Attribute = u32;