From cfc1e193b6ad2c4ff7cffdde27bbda30af463c14 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 20 Jan 2025 15:15:45 +0100 Subject: [PATCH] update the test with the stats --- .../src/scheduler/process_upgrade/mod.rs | 12 ++++- .../upgrade/v1_12/v1_12_0.ms/auth/lock.mdb | Bin 8192 -> 8192 bytes .../upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb | Bin 8192 -> 8192 bytes .../tests/upgrade/v1_12/v1_12_0.rs | 45 ++++++++++++++---- crates/milli/src/update/upgrade/mod.rs | 16 ++++--- 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs index f3038d343..1471723ef 100644 --- a/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs +++ b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs @@ -19,8 +19,18 @@ impl IndexScheduler { indexes.len() as u32, )); let index = self.index(uid)?; - milli::update::upgrade::upgrade(&index, progress.clone()) + let mut wtxn = index.write_txn()?; + let regenerate = milli::update::upgrade::upgrade(&mut wtxn, &index, progress.clone()) .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; + if regenerate { + let stats = crate::index_mapper::IndexStats::new(&index, &wtxn) + .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; + // Release wtxn as soon as possible because it stops us from registering tasks + let mut index_schd_wtxn = self.env.write_txn()?; + self.index_mapper.store_stats_of(&mut index_schd_wtxn, uid, &stats)?; + index_schd_wtxn.commit()?; + } + wtxn.commit()?; } Ok(()) diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb index 987a95d1bc69c5786a55929782921ddbf1de04e9..9154fcee23c0f8e6e815de8fce3f313de6ae1433 100644 GIT binary patch delta 20 bcmZp0XmFS?f$czBut!|w#)a|n6BD=rSjh-0 delta 20 bcmZp0XmFS?fvq9C`Mz1n#)a|n6BD=rTB`_Z diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb index 13319457beb92c94ef07cb823aec8b6c0577342c..2fcbf7530a8cdca920fa033fd01171dcf5bc8b70 100644 GIT binary patch delta 19 acmZp0XmFS?k+m(@Bd&5{pw`9%4)Op?LkD{R delta 19 bcmZp0XmFS?kyUN}T(gpifm$06ILHG4OUVb1 diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs index e47926262..fd2d363f2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -206,6 +206,42 @@ async fn check_the_index_scheduler(server: &Server) { let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16:47:41"); + let (stats, _) = server.stats().await; + snapshot!(stats, @r#" + { + "databaseSize": 425984, + "lastUpdate": "2025-01-16T17:18:43.296777845Z", + "indexes": { + "kefir": { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "age": 1, + "description": 1, + "id": 1, + "name": 1, + "surname": 1 + } + } + } + } + "#); + let index = server.index("kefir"); + let (stats, _) = index.stats().await; + snapshot!(stats, @r#" + { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "age": 1, + "description": 1, + "id": 1, + "name": 1, + "surname": 1 + } + } + "#); + // Delete all the tasks of a specific batch let (task, _) = server.delete_tasks("batchUids=10").await; server.wait_task(task.uid()).await.succeeded(); @@ -218,15 +254,6 @@ async fn check_the_index_scheduler(server: &Server) { let index = server.index("kefirausaurus"); let (task, _) = index.create(Some("kefid")).await; server.wait_task(task.uid()).await.succeeded(); - - let (stats, _) = index.stats().await; - snapshot!(stats, @r#" - { - "numberOfDocuments": 0, - "isIndexing": false, - "fieldDistribution": {} - } - "#); } /// Ensuring the index roughly works with filter and sort. diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 6b6e551a3..6e533177a 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -1,17 +1,19 @@ +use heed::RwTxn; + use crate::progress::{Progress, VariableNameStep}; use crate::{Index, InternalError, Result}; -pub fn upgrade(index: &Index, progress: Progress) -> Result<()> { - let wtxn = index.env.write_txn()?; - let from = index.get_version(&wtxn)?; +/// Return true if the cached stats of the index must be regenerated +pub fn upgrade(wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result { + let from = index.get_version(wtxn)?; let upgrade_functions = [(v1_12_to_v1_13 as fn(&Index, Progress) -> Result<()>, "Upgrading from v1.12 to v1.13")]; - let start = match from { + let (start, regenerate_stats) = match from { // If there was no version it means we're coming from the v1.12 - None | Some((1, 12, _)) => 0, + None | Some((1, 12, _)) => (0, false), // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. - Some((1, 13, _)) => return Ok(()), + Some((1, 13, _)) => return Ok(false), Some((major, minor, patch)) => { return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) } @@ -29,7 +31,7 @@ pub fn upgrade(index: &Index, progress: Progress) -> Result<()> { (upgrade_function)(index, progress.clone())?; } - Ok(()) + Ok(regenerate_stats) } fn v1_12_to_v1_13(_index: &Index, _progress: Progress) -> Result<()> {