From 5820d822c8f18846c45f45bc4787a33400a623a3 Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Tue, 25 Mar 2025 16:51:18 +0100
Subject: [PATCH] Add more details about the finalizing progress step
---
crates/index-scheduler/src/insta_snapshot.rs | 1 +
crates/index-scheduler/src/processing.rs | 7 +
.../src/scheduler/process_batch.rs | 6 +-
.../src/scheduler/process_index_operation.rs | 10 +-
crates/meilisearch/tests/batches/mod.rs | 126 ++++++++++++------
crates/meilisearch/tests/dumps/mod.rs | 1 +
.../batches.snap | 1 +
...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 3 +-
...rFinishedAt_equal_2025-01-16T16_47_41.snap | 3 +-
...erStartedAt_equal_2025-01-16T16_47_41.snap | 3 +-
...ue_once_everything_has_been_processed.snap | 3 +-
.../tests/upgrade/v1_12/v1_12_0.rs | 22 +--
12 files changed, 122 insertions(+), 64 deletions(-)
diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs
index bcc295afd..949edf369 100644
--- a/crates/index-scheduler/src/insta_snapshot.rs
+++ b/crates/index-scheduler/src/insta_snapshot.rs
@@ -344,6 +344,7 @@ pub fn snapshot_batch(batch: &Batch) -> String {
let Batch { uid, details, stats, started_at, finished_at, progress: _, enqueued_at } = batch;
let stats = BatchStats {
progress_trace: Default::default(),
+ internal_database_sizes: Default::default(),
write_channel_congestion: None,
..stats.clone()
};
diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs
index fed26aeb7..09ce46884 100644
--- a/crates/index-scheduler/src/processing.rs
+++ b/crates/index-scheduler/src/processing.rs
@@ -64,6 +64,13 @@ make_enum_progress! {
}
}
+make_enum_progress! {
+ pub enum FinalizingIndexStep {
+ Committing,
+ ComputingStats,
+ }
+}
+
make_enum_progress! {
pub enum TaskCancelationProgress {
RetrievingTasks,
diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs
index 996b548c2..42de1d137 100644
--- a/crates/index-scheduler/src/scheduler/process_batch.rs
+++ b/crates/index-scheduler/src/scheduler/process_batch.rs
@@ -12,7 +12,7 @@ use roaring::RoaringBitmap;
use super::create_batch::Batch;
use crate::processing::{
- AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress,
+ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep,
InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress,
UpdateIndexProgress,
};
@@ -151,9 +151,10 @@ impl IndexScheduler {
let mut index_wtxn = index.write_txn()?;
let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?;
let (tasks, congestion) =
- self.apply_index_operation(&mut index_wtxn, &index, op, progress)?;
+ self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?;
{
+ progress.update_progress(FinalizingIndexStep::Committing);
let span = tracing::trace_span!(target: "indexing::scheduler", "commit");
let _entered = span.enter();
@@ -166,6 +167,7 @@ impl IndexScheduler {
// the entire batch.
let mut post_commit_dabases_sizes = None;
let res = || -> Result<()> {
+ progress.update_progress(FinalizingIndexStep::ComputingStats);
let index_rtxn = index.read_txn()?;
let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
.map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;
diff --git a/crates/index-scheduler/src/scheduler/process_index_operation.rs b/crates/index-scheduler/src/scheduler/process_index_operation.rs
index 690fe2efd..9b12d61cf 100644
--- a/crates/index-scheduler/src/scheduler/process_index_operation.rs
+++ b/crates/index-scheduler/src/scheduler/process_index_operation.rs
@@ -32,7 +32,7 @@ impl IndexScheduler {
index_wtxn: &mut RwTxn<'i>,
index: &'i Index,
operation: IndexOperation,
- progress: Progress,
+ progress: &Progress,
) -> Result<(Vec<Task>, Option<ChannelCongestion>)> {
let indexer_alloc = Bump::new();
let started_processing_at = std::time::Instant::now();
@@ -186,7 +186,7 @@ impl IndexScheduler {
&document_changes,
embedders,
&|| must_stop_processing.get(),
- &progress,
+ progress,
)
.map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?,
);
@@ -307,7 +307,7 @@ impl IndexScheduler {
&document_changes,
embedders,
&|| must_stop_processing.get(),
- &progress,
+ progress,
)
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
);
@@ -465,7 +465,7 @@ impl IndexScheduler {
&document_changes,
embedders,
&|| must_stop_processing.get(),
- &progress,
+ progress,
)
.map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
);
@@ -520,7 +520,7 @@ impl IndexScheduler {
index_uid: index_uid.clone(),
tasks: cleared_tasks,
},
- progress.clone(),
+ progress,
)?;
let (settings_tasks, _congestion) = self.apply_index_operation(
diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs
index 468963631..e955c6883 100644
--- a/crates/meilisearch/tests/batches/mod.rs
+++ b/crates/meilisearch/tests/batches/mod.rs
@@ -281,7 +281,8 @@ async fn test_summarized_document_addition_or_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -303,7 +304,8 @@ async fn test_summarized_document_addition_or_update() {
"test": 1
},
"progressTrace": "[progressTrace]",
- "writeChannelCongestion": "[writeChannelCongestion]"
+ "writeChannelCongestion": "[writeChannelCongestion]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -322,7 +324,8 @@ async fn test_summarized_document_addition_or_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -344,7 +347,8 @@ async fn test_summarized_document_addition_or_update() {
"test": 1
},
"progressTrace": "[progressTrace]",
- "writeChannelCongestion": "[writeChannelCongestion]"
+ "writeChannelCongestion": "[writeChannelCongestion]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -367,7 +371,8 @@ async fn test_summarized_delete_documents_by_batch() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -388,7 +393,8 @@ async fn test_summarized_delete_documents_by_batch() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -407,7 +413,8 @@ async fn test_summarized_delete_documents_by_batch() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -428,7 +435,8 @@ async fn test_summarized_delete_documents_by_batch() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -453,7 +461,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -475,7 +484,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -495,7 +505,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -517,7 +528,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -537,7 +549,8 @@ async fn test_summarized_delete_documents_by_filter() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r#"
{
@@ -559,7 +572,8 @@ async fn test_summarized_delete_documents_by_filter() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -583,7 +597,8 @@ async fn test_summarized_delete_document_by_id() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r#"
{
@@ -604,7 +619,8 @@ async fn test_summarized_delete_document_by_id() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -623,7 +639,8 @@ async fn test_summarized_delete_document_by_id() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r#"
{
@@ -644,7 +661,8 @@ async fn test_summarized_delete_document_by_id() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -679,7 +697,8 @@ async fn test_summarized_settings_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -709,7 +728,8 @@ async fn test_summarized_settings_update() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -732,7 +752,8 @@ async fn test_summarized_index_creation() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -750,7 +771,8 @@ async fn test_summarized_index_creation() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -768,7 +790,8 @@ async fn test_summarized_index_creation() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -788,7 +811,8 @@ async fn test_summarized_index_creation() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -921,7 +945,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -939,7 +964,8 @@ async fn test_summarized_index_update() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -957,7 +983,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -977,7 +1004,8 @@ async fn test_summarized_index_update() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -998,7 +1026,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r#"
{
@@ -1016,7 +1045,8 @@ async fn test_summarized_index_update() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1034,7 +1064,8 @@ async fn test_summarized_index_update() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1054,7 +1085,8 @@ async fn test_summarized_index_update() {
"indexUids": {
"test": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1080,7 +1112,8 @@ async fn test_summarized_index_swap() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1105,7 +1138,8 @@ async fn test_summarized_index_swap() {
"indexSwap": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1129,7 +1163,8 @@ async fn test_summarized_index_swap() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1147,7 +1182,8 @@ async fn test_summarized_index_swap() {
"indexUids": {
"doggos": 1
},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1173,7 +1209,8 @@ async fn test_summarized_batch_cancelation() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1193,7 +1230,8 @@ async fn test_summarized_batch_cancelation() {
"taskCancelation": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1219,7 +1257,8 @@ async fn test_summarized_batch_deletion() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1239,7 +1278,8 @@ async fn test_summarized_batch_deletion() {
"taskDeletion": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
@@ -1262,7 +1302,8 @@ async fn test_summarized_dump_creation() {
".startedAt" => "[date]",
".finishedAt" => "[date]",
".stats.progressTrace" => "[progressTrace]",
- ".stats.writeChannelCongestion" => "[writeChannelCongestion]"
+ ".stats.writeChannelCongestion" => "[writeChannelCongestion]",
+ ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]"
},
@r###"
{
@@ -1280,7 +1321,8 @@ async fn test_summarized_dump_creation() {
"dumpCreation": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
diff --git a/crates/meilisearch/tests/dumps/mod.rs b/crates/meilisearch/tests/dumps/mod.rs
index ff0b027cb..fa05d9ec9 100644
--- a/crates/meilisearch/tests/dumps/mod.rs
+++ b/crates/meilisearch/tests/dumps/mod.rs
@@ -2236,6 +2236,7 @@ async fn import_dump_v6_containing_batches_and_enqueued_tasks() {
".results[0].finishedAt" => "[date]",
".results[0].duration" => "[date]",
".results[0].stats.progressTrace" => "[progressTrace]",
+ ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]",
".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]",
}), name: "batches");
diff --git a/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap
index b38340ef6..b2dea1f06 100644
--- a/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap
+++ b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap
@@ -22,6 +22,7 @@ source: crates/meilisearch/tests/dumps/mod.rs
"kefir": 1
},
"progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]",
"writeChannelCongestion": "[writeChannelCongestion]"
},
"duration": "[date]",
diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
index 99caeaf96..b79f55351 100644
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
index 99caeaf96..b79f55351 100644
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
index 99caeaf96..b79f55351 100644
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
index 623c1f778..3cfed9f74 100644
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
@@ -19,7 +19,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"upgradeDatabase": 1
},
"indexUids": {},
- "progressTrace": "[progressTrace]"
+ "progressTrace": "[progressTrace]",
+ "internalDatabaseSizes": "[internalDatabaseSizes]"
},
"duration": "[duration]",
"startedAt": "[date]",
diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
index 11ba2882a..8157f0923 100644
--- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
+++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
@@ -166,7 +166,7 @@ async fn check_the_index_scheduler(server: &Server) {
let (tasks, _) = server.tasks_filter("limit=1000").await;
snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed");
let (batches, _) = server.batches_filter("limit=1000").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed");
// Tests all the tasks query parameters
let (tasks, _) = server.tasks_filter("uids=10").await;
@@ -193,26 +193,26 @@ async fn check_the_index_scheduler(server: &Server) {
// Tests all the batches query parameters
let (batches, _) = server.batches_filter("uids=10").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10");
let (batches, _) = server.batches_filter("batchUids=10").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10");
let (batches, _) = server.batches_filter("statuses=canceled").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled");
// types has already been tested above to retrieve the upgrade database
let (batches, _) = server.batches_filter("canceledBy=19").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19");
let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await;
- snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
+ snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
let (stats, _) = server.stats().await;
assert_json_snapshot!(stats, {