diff --git a/Cargo.lock b/Cargo.lock index e251776c3..6a42ffa26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3776,6 +3776,7 @@ dependencies = [ "candle-transformers", "charabia", "concat-arrays", + "convert_case 0.6.0", "crossbeam-channel", "csv", "deserr", diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 4acd7b22a..7c1783a1a 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -38,7 +38,7 @@ fn setup_index() -> Index { let mut options = EnvOpenOptions::new(); options.map_size(100 * 1024 * 1024 * 1024); // 100 GB options.max_readers(100); - Index::new(options, path).unwrap() + Index::new(options, path, true).unwrap() } fn setup_settings<'t>( diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 889478d40..b472b4f6b 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -68,7 +68,7 @@ pub fn base_setup(conf: &Conf) -> Index { let mut options = EnvOpenOptions::new(); options.map_size(100 * 1024 * 1024 * 1024); // 100 GB options.max_readers(100); - let index = Index::new(options, conf.database_name).unwrap(); + let index = Index::new(options, conf.database_name, true).unwrap(); let config = IndexerConfig::default(); let mut wtxn = index.write_txn().unwrap(); diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index f822421cf..0fb5570b0 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -141,6 +141,9 @@ pub enum KindDump { instance_uid: Option, }, SnapshotCreation, + UpgradeDatabase { + from: (u32, u32, u32), + }, } impl From for TaskDump { @@ -210,6 +213,9 @@ impl From for KindDump { KindDump::DumpCreation { keys, instance_uid } } KindWithContent::SnapshotCreation => KindDump::SnapshotCreation, + KindWithContent::UpgradeDatabase { from: version } => { + KindDump::UpgradeDatabase { from: version } + } } } } diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index 08711e5e3..1216083ca 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -63,7 +63,7 @@ fn main() { Some(path) => TempDir::new_in(path).unwrap(), None => TempDir::new().unwrap(), }; - let index = Index::new(options, tempdir.path()).unwrap(); + let index = Index::new(options, tempdir.path(), true).unwrap(); let indexer_config = IndexerConfig::default(); std::thread::scope(|s| { diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 2dde6d623..7e0341fcb 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -144,6 +144,7 @@ impl<'a> Dump<'a> { KindWithContent::DumpCreation { keys, instance_uid } } KindDump::SnapshotCreation => KindWithContent::SnapshotCreation, + KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from }, }, }; diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index f6ee1f685..d3feecd73 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -147,7 +147,9 @@ pub enum Error { #[error("Corrupted task queue.")] CorruptedTaskQueue, #[error(transparent)] - TaskDatabaseUpdate(Box), + DatabaseUpgrade(Box), + #[error(transparent)] + UnrecoverableError(Box), #[error(transparent)] HeedTransaction(heed::Error), @@ -202,7 +204,8 @@ impl Error { | Error::Anyhow(_) => true, Error::CreateBatch(_) | Error::CorruptedTaskQueue - | Error::TaskDatabaseUpdate(_) + | Error::DatabaseUpgrade(_) + | Error::UnrecoverableError(_) | Error::HeedTransaction(_) => false, #[cfg(test)] Error::PlannedFailure => false, @@ -266,7 +269,8 @@ impl ErrorCode for Error { Error::Anyhow(_) => Code::Internal, Error::CorruptedTaskQueue => Code::Internal, Error::CorruptedDump => Code::Internal, - Error::TaskDatabaseUpdate(_) => Code::Internal, + Error::DatabaseUpgrade(_) => Code::Internal, + Error::UnrecoverableError(_) => Code::Internal, Error::CreateBatch(_) => Code::Internal, // This one should never be seen by the end user diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index e29e52d44..c6c17b2d5 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -7,7 +7,12 @@ use meilisearch_types::heed::{Database, Env, RwTxn}; use crate::error::FeatureNotEnabledError; use crate::Result; -const EXPERIMENTAL_FEATURES: &str = "experimental-features"; +/// The number of database used by features +const NUMBER_OF_DATABASES: u32 = 1; +/// Database const names for the `FeatureData`. +mod db_name { + pub const EXPERIMENTAL_FEATURES: &str = "experimental-features"; +} #[derive(Clone)] pub(crate) struct FeatureData { @@ -84,14 +89,20 @@ impl RoFeatures { } impl FeatureData { - pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result { - let mut wtxn = env.write_txn()?; - let runtime_features_db = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?; - wtxn.commit()?; + pub(crate) const fn nb_db() -> u32 { + NUMBER_OF_DATABASES + } + + pub fn new( + env: &Env, + wtxn: &mut RwTxn, + instance_features: InstanceTogglableFeatures, + ) -> Result { + let runtime_features_db = + env.create_database(wtxn, Some(db_name::EXPERIMENTAL_FEATURES))?; - let txn = env.read_txn()?; let persisted_features: RuntimeTogglableFeatures = - runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default(); + runtime_features_db.get(wtxn, db_name::EXPERIMENTAL_FEATURES)?.unwrap_or_default(); let InstanceTogglableFeatures { metrics, logs_route, contains_filter } = instance_features; let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures { metrics: metrics || persisted_features.metrics, @@ -108,7 +119,7 @@ impl FeatureData { mut wtxn: RwTxn, features: RuntimeTogglableFeatures, ) -> Result<()> { - self.persisted.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?; + self.persisted.put(&mut wtxn, db_name::EXPERIMENTAL_FEATURES, &features)?; wtxn.commit()?; // safe to unwrap, the lock will only fail if: diff --git a/crates/index-scheduler/src/index_mapper/index_map.rs b/crates/index-scheduler/src/index_mapper/index_map.rs index 947f558aa..3031043a9 100644 --- a/crates/index-scheduler/src/index_mapper/index_map.rs +++ b/crates/index-scheduler/src/index_mapper/index_map.rs @@ -102,7 +102,7 @@ impl ReopenableIndex { return Ok(()); } map.unavailable.remove(&self.uuid); - map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?; + map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size, false)?; } Ok(()) } @@ -171,11 +171,12 @@ impl IndexMap { date: Option<(OffsetDateTime, OffsetDateTime)>, enable_mdb_writemap: bool, map_size: usize, + creation: bool, ) -> Result { if !matches!(self.get_unavailable(uuid), Missing) { panic!("Attempt to open an index that was unavailable"); } - let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?; + let index = create_or_open_index(path, date, enable_mdb_writemap, map_size, creation)?; match self.available.insert(*uuid, index.clone()) { InsertionOutcome::InsertedNew => (), InsertionOutcome::Evicted(evicted_uuid, evicted_index) => { @@ -299,6 +300,7 @@ fn create_or_open_index( date: Option<(OffsetDateTime, OffsetDateTime)>, enable_mdb_writemap: bool, map_size: usize, + creation: bool, ) -> Result { let mut options = EnvOpenOptions::new(); options.map_size(clamp_to_page_size(map_size)); @@ -308,9 +310,9 @@ fn create_or_open_index( } if let Some((created, updated)) = date { - Ok(Index::new_with_creation_dates(options, path, created, updated)?) + Ok(Index::new_with_creation_dates(options, path, created, updated, creation)?) } else { - Ok(Index::new(options, path)?) + Ok(Index::new(options, path, creation)?) } } diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index 77cccf9b1..dad73d4c6 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -20,8 +20,13 @@ use crate::{Error, IndexBudget, IndexSchedulerOptions, Result}; mod index_map; -const INDEX_MAPPING: &str = "index-mapping"; -const INDEX_STATS: &str = "index-stats"; +/// The number of database used by index mapper +const NUMBER_OF_DATABASES: u32 = 2; +/// Database const names for the `IndexMapper`. +mod db_name { + pub const INDEX_MAPPING: &str = "index-mapping"; + pub const INDEX_STATS: &str = "index-stats"; +} /// Structure managing meilisearch's indexes. /// @@ -138,6 +143,10 @@ impl IndexStats { } impl IndexMapper { + pub(crate) const fn nb_db() -> u32 { + NUMBER_OF_DATABASES + } + pub fn new( env: &Env, wtxn: &mut RwTxn, @@ -146,8 +155,8 @@ impl IndexMapper { ) -> Result { Ok(Self { index_map: Arc::new(RwLock::new(IndexMap::new(budget.index_count))), - index_mapping: env.create_database(wtxn, Some(INDEX_MAPPING))?, - index_stats: env.create_database(wtxn, Some(INDEX_STATS))?, + index_mapping: env.create_database(wtxn, Some(db_name::INDEX_MAPPING))?, + index_stats: env.create_database(wtxn, Some(db_name::INDEX_STATS))?, base_path: options.indexes_path.clone(), index_base_map_size: budget.map_size, index_growth_amount: options.index_growth_amount, @@ -189,6 +198,7 @@ impl IndexMapper { date, self.enable_mdb_writemap, self.index_base_map_size, + true, ) .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?; let index_rtxn = index.read_txn()?; @@ -387,6 +397,7 @@ impl IndexMapper { None, self.enable_mdb_writemap, self.index_base_map_size, + false, ) .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?; } diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index de79cd7c0..4bc2beb05 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -6,6 +6,7 @@ use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str}; use meilisearch_types::heed::{Database, RoTxn}; use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32}; use meilisearch_types::tasks::{Details, Kind, Status, Task}; +use meilisearch_types::versioning; use roaring::RoaringBitmap; use crate::index_mapper::IndexMapper; @@ -21,6 +22,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { cleanup_enabled: _, processing_tasks, env, + version, queue, scheduler, @@ -38,6 +40,16 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { let mut snap = String::new(); + let indx_sched_version = version.get_version(&rtxn).unwrap(); + let latest_version = ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ); + if indx_sched_version != Some(latest_version) { + snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n")); + } + let processing = processing_tasks.read().unwrap().clone(); snap.push_str(&format!("### Autobatching Enabled = {}\n", scheduler.autobatching_enabled)); snap.push_str(&format!( @@ -279,6 +291,9 @@ fn snapshot_details(d: &Details) -> String { Details::IndexSwap { swaps } => { format!("{{ swaps: {swaps:?} }}") } + Details::UpgradeDatabase { from, to } => { + format!("{{ from: {from:?}, to: {to:?} }}") + } } } diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 3c50283d9..530b7bedc 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -30,8 +30,10 @@ mod queue; mod scheduler; #[cfg(test)] mod test_utils; +pub mod upgrade; mod utils; pub mod uuid_codec; +mod versioning; pub type Result = std::result::Result; pub type TaskId = u32; @@ -65,6 +67,7 @@ use queue::Queue; use roaring::RoaringBitmap; use scheduler::Scheduler; use time::OffsetDateTime; +use versioning::Versioning; use crate::index_mapper::IndexMapper; use crate::utils::clamp_to_page_size; @@ -120,6 +123,8 @@ pub struct IndexSchedulerOptions { pub batched_tasks_size_limit: u64, /// The experimental features enabled for this instance. pub instance_features: InstanceTogglableFeatures, + /// The experimental features enabled for this instance. + pub auto_upgrade: bool, } /// Structure which holds meilisearch's indexes and schedules the tasks @@ -131,17 +136,18 @@ pub struct IndexScheduler { /// The list of tasks currently processing pub(crate) processing_tasks: Arc>, + /// A database containing only the version of the index-scheduler + pub version: versioning::Versioning, /// The queue containing both the tasks and the batches. pub queue: queue::Queue, - - pub scheduler: scheduler::Scheduler, - /// In charge of creating, opening, storing and returning indexes. pub(crate) index_mapper: IndexMapper, - /// In charge of fetching and setting the status of experimental features. features: features::FeatureData, + /// Everything related to the processing of the tasks + pub scheduler: scheduler::Scheduler, + /// Whether we should automatically cleanup the task queue or not. pub(crate) cleanup_enabled: bool, @@ -176,6 +182,7 @@ impl IndexScheduler { IndexScheduler { env: self.env.clone(), processing_tasks: self.processing_tasks.clone(), + version: self.version.clone(), queue: self.queue.private_clone(), scheduler: self.scheduler.private_clone(), @@ -194,10 +201,15 @@ impl IndexScheduler { } } + pub(crate) const fn nb_db() -> u32 { + Versioning::nb_db() + Queue::nb_db() + IndexMapper::nb_db() + features::FeatureData::nb_db() + } + /// Create an index scheduler and start its run loop. #[allow(private_interfaces)] // because test_utils is private pub fn new( options: IndexSchedulerOptions, + from_db_version: (u32, u32, u32), #[cfg(test)] test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>, #[cfg(test)] planned_failures: Vec<(usize, test_utils::FailureLocation)>, ) -> Result { @@ -229,14 +241,16 @@ impl IndexScheduler { let env = unsafe { heed::EnvOpenOptions::new() - .max_dbs(19) + .max_dbs(Self::nb_db()) .map_size(budget.task_db_size) .open(&options.tasks_path) }?; - let features = features::FeatureData::new(&env, options.instance_features)?; + // We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them + let version = versioning::Versioning::new(&env, from_db_version)?; let mut wtxn = env.write_txn()?; + let features = features::FeatureData::new(&env, &mut wtxn, options.instance_features)?; let queue = Queue::new(&env, &mut wtxn, &options)?; let index_mapper = IndexMapper::new(&env, &mut wtxn, &options, budget)?; wtxn.commit()?; @@ -244,6 +258,7 @@ impl IndexScheduler { // allow unreachable_code to get rids of the warning in the case of a test build. let this = Self { processing_tasks: Arc::new(RwLock::new(ProcessingTasks::new())), + version, queue, scheduler: Scheduler::new(&options), @@ -366,6 +381,7 @@ impl IndexScheduler { match ret { Ok(Ok(TickOutcome::TickAgain(_))) => (), Ok(Ok(TickOutcome::WaitForSignal)) => run.scheduler.wake_up.wait(), + Ok(Ok(TickOutcome::StopProcessingForever)) => break, Ok(Err(e)) => { tracing::error!("{e}"); // Wait one second when an irrecoverable error occurs. @@ -813,6 +829,8 @@ pub enum TickOutcome { TickAgain(u64), /// The scheduler should wait for an external signal before attempting another `tick`. WaitForSignal, + /// The scheduler exits the run-loop and will never process tasks again + StopProcessingForever, } /// How many indexes we can afford to have open simultaneously. diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index d0382a81b..58f01c770 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -1,8 +1,6 @@ -use std::borrow::Cow; use std::sync::Arc; -use enum_iterator::Sequence; -use meilisearch_types::milli::progress::{AtomicSubStep, NamedStep, Progress, ProgressView, Step}; +use meilisearch_types::milli::progress::{AtomicSubStep, NamedStep, Progress, ProgressView}; use meilisearch_types::milli::{make_atomic_progress, make_enum_progress}; use roaring::RoaringBitmap; @@ -173,32 +171,6 @@ make_atomic_progress!(Document alias AtomicDocumentStep => "document" ); make_atomic_progress!(Batch alias AtomicBatchStep => "batch" ); make_atomic_progress!(UpdateFile alias AtomicUpdateFileStep => "update file" ); -pub struct VariableNameStep { - name: String, - current: u32, - total: u32, -} - -impl VariableNameStep { - pub fn new(name: impl Into, current: u32, total: u32) -> Self { - Self { name: name.into(), current, total } - } -} - -impl Step for VariableNameStep { - fn name(&self) -> Cow<'static, str> { - self.name.clone().into() - } - - fn current(&self) -> u32 { - self.current - } - - fn total(&self) -> u32 { - self.total - } -} - #[cfg(test)] mod test { use std::sync::atomic::Ordering; diff --git a/crates/index-scheduler/src/queue/batches.rs b/crates/index-scheduler/src/queue/batches.rs index a31653387..5c8a573ab 100644 --- a/crates/index-scheduler/src/queue/batches.rs +++ b/crates/index-scheduler/src/queue/batches.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::ops::{Bound, RangeBounds}; use meilisearch_types::batches::{Batch, BatchId}; @@ -10,9 +11,14 @@ use time::OffsetDateTime; use super::{Query, Queue}; use crate::processing::ProcessingTasks; -use crate::utils::{insert_task_datetime, keep_ids_within_datetimes, map_bound, ProcessingBatch}; +use crate::utils::{ + insert_task_datetime, keep_ids_within_datetimes, map_bound, remove_task_datetime, + ProcessingBatch, +}; use crate::{Error, Result, BEI128}; +/// The number of database used by the batch queue +const NUMBER_OF_DATABASES: u32 = 7; /// Database const names for the `IndexScheduler`. mod db_name { pub const ALL_BATCHES: &str = "all-batches"; @@ -56,6 +62,10 @@ impl BatchQueue { } } + pub(crate) const fn nb_db() -> u32 { + NUMBER_OF_DATABASES + } + pub(super) fn new(env: &Env, wtxn: &mut RwTxn) -> Result { Ok(Self { all_batches: env.create_database(wtxn, Some(db_name::ALL_BATCHES))?, @@ -159,6 +169,8 @@ impl BatchQueue { } pub(crate) fn write_batch(&self, wtxn: &mut RwTxn, batch: ProcessingBatch) -> Result<()> { + let old_batch = self.all_batches.get(wtxn, &batch.uid)?; + self.all_batches.put( wtxn, &batch.uid, @@ -172,30 +184,92 @@ impl BatchQueue { }, )?; + // Update the statuses + if let Some(ref old_batch) = old_batch { + for status in old_batch.stats.status.keys() { + self.update_status(wtxn, *status, |bitmap| { + bitmap.remove(batch.uid); + })?; + } + } for status in batch.statuses { self.update_status(wtxn, status, |bitmap| { bitmap.insert(batch.uid); })?; } + // Update the kinds / types + if let Some(ref old_batch) = old_batch { + let kinds: HashSet<_> = old_batch.stats.types.keys().cloned().collect(); + for kind in kinds.difference(&batch.kinds) { + self.update_kind(wtxn, *kind, |bitmap| { + bitmap.remove(batch.uid); + })?; + } + } for kind in batch.kinds { self.update_kind(wtxn, kind, |bitmap| { bitmap.insert(batch.uid); })?; } + // Update the indexes + if let Some(ref old_batch) = old_batch { + let indexes: HashSet<_> = old_batch.stats.index_uids.keys().cloned().collect(); + for index in indexes.difference(&batch.indexes) { + self.update_index(wtxn, index, |bitmap| { + bitmap.remove(batch.uid); + })?; + } + } for index in batch.indexes { self.update_index(wtxn, &index, |bitmap| { bitmap.insert(batch.uid); })?; } + // Update the enqueued_at: we cannot retrieve the previous enqueued at from the previous batch, and + // must instead go through the db looking for it. We cannot look at the task contained in this batch either + // because they may have been removed. + // What we know, though, is that the task date is from before the enqueued_at, and max two timestamps have been written + // to the DB per batches. + if let Some(ref old_batch) = old_batch { + let started_at = old_batch.started_at.unix_timestamp_nanos(); + + // We have either one or two enqueued at to remove + let mut exit = old_batch.stats.total_nb_tasks.clamp(0, 2); + let mut iterator = self.enqueued_at.rev_iter_mut(wtxn)?; + while let Some(entry) = iterator.next() { + let (key, mut value) = entry?; + if key > started_at { + continue; + } + if value.remove(old_batch.uid) { + exit = exit.saturating_sub(1); + // Safe because the key and value are owned + unsafe { + iterator.put_current(&key, &value)?; + } + if exit == 0 { + break; + } + } + } + } if let Some(enqueued_at) = batch.oldest_enqueued_at { insert_task_datetime(wtxn, self.enqueued_at, enqueued_at, batch.uid)?; } if let Some(enqueued_at) = batch.earliest_enqueued_at { insert_task_datetime(wtxn, self.enqueued_at, enqueued_at, batch.uid)?; } + + // Update the started at and finished at + if let Some(ref old_batch) = old_batch { + remove_task_datetime(wtxn, self.started_at, old_batch.started_at, old_batch.uid)?; + if let Some(finished_at) = old_batch.finished_at { + remove_task_datetime(wtxn, self.finished_at, finished_at, old_batch.uid)?; + } + } insert_task_datetime(wtxn, self.started_at, batch.started_at, batch.uid)?; insert_task_datetime(wtxn, self.finished_at, batch.finished_at.unwrap(), batch.uid)?; diff --git a/crates/index-scheduler/src/queue/mod.rs b/crates/index-scheduler/src/queue/mod.rs index 4921d05e6..c6a79fbb2 100644 --- a/crates/index-scheduler/src/queue/mod.rs +++ b/crates/index-scheduler/src/queue/mod.rs @@ -20,14 +20,16 @@ use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; use uuid::Uuid; -use self::batches::BatchQueue; -use self::tasks::TaskQueue; +pub(crate) use self::batches::BatchQueue; +pub(crate) use self::tasks::TaskQueue; use crate::processing::ProcessingTasks; use crate::utils::{ check_index_swap_validity, filter_out_references_to_newer_tasks, ProcessingBatch, }; use crate::{Error, IndexSchedulerOptions, Result, TaskId}; +/// The number of database used by queue itself +const NUMBER_OF_DATABASES: u32 = 1; /// Database const names for the `IndexScheduler`. mod db_name { pub const BATCH_TO_TASKS_MAPPING: &str = "batch-to-tasks-mapping"; @@ -148,6 +150,10 @@ impl Queue { } } + pub(crate) const fn nb_db() -> u32 { + tasks::TaskQueue::nb_db() + batches::BatchQueue::nb_db() + NUMBER_OF_DATABASES + } + /// Create an index scheduler and start its run loop. pub(crate) fn new( env: &Env, diff --git a/crates/index-scheduler/src/queue/tasks.rs b/crates/index-scheduler/src/queue/tasks.rs index c88192e17..913ebcb30 100644 --- a/crates/index-scheduler/src/queue/tasks.rs +++ b/crates/index-scheduler/src/queue/tasks.rs @@ -9,12 +9,17 @@ use time::OffsetDateTime; use super::{Query, Queue}; use crate::processing::ProcessingTasks; -use crate::utils::{self, insert_task_datetime, keep_ids_within_datetimes, map_bound}; +use crate::utils::{ + self, insert_task_datetime, keep_ids_within_datetimes, map_bound, remove_task_datetime, +}; use crate::{Error, Result, TaskId, BEI128}; +/// The number of database used by the task queue +const NUMBER_OF_DATABASES: u32 = 8; /// Database const names for the `IndexScheduler`. mod db_name { pub const ALL_TASKS: &str = "all-tasks"; + pub const STATUS: &str = "status"; pub const KIND: &str = "kind"; pub const INDEX_TASKS: &str = "index-tasks"; @@ -59,7 +64,11 @@ impl TaskQueue { } } - pub(super) fn new(env: &Env, wtxn: &mut RwTxn) -> Result { + pub(crate) const fn nb_db() -> u32 { + NUMBER_OF_DATABASES + } + + pub(crate) fn new(env: &Env, wtxn: &mut RwTxn) -> Result { Ok(Self { all_tasks: env.create_database(wtxn, Some(db_name::ALL_TASKS))?, status: env.create_database(wtxn, Some(db_name::STATUS))?, @@ -90,12 +99,14 @@ impl TaskQueue { pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> { let old_task = self.get_task(wtxn, task.uid)?.ok_or(Error::CorruptedTaskQueue)?; + let reprocessing = old_task.status != Status::Enqueued; debug_assert!(old_task != *task); debug_assert_eq!(old_task.uid, task.uid); - debug_assert!(old_task.batch_uid.is_none() && task.batch_uid.is_some()); + + // If we're processing a task that failed it may already contains a batch_uid debug_assert!( - old_task.batch_uid.is_none() && task.batch_uid.is_some(), + reprocessing || (old_task.batch_uid.is_none() && task.batch_uid.is_some()), "\n==> old: {old_task:?}\n==> new: {task:?}" ); @@ -122,13 +133,25 @@ impl TaskQueue { "Cannot update a task's enqueued_at time" ); if old_task.started_at != task.started_at { - assert!(old_task.started_at.is_none(), "Cannot update a task's started_at time"); + assert!( + reprocessing || old_task.started_at.is_none(), + "Cannot update a task's started_at time" + ); + if let Some(started_at) = old_task.started_at { + remove_task_datetime(wtxn, self.started_at, started_at, task.uid)?; + } if let Some(started_at) = task.started_at { insert_task_datetime(wtxn, self.started_at, started_at, task.uid)?; } } if old_task.finished_at != task.finished_at { - assert!(old_task.finished_at.is_none(), "Cannot update a task's finished_at time"); + assert!( + reprocessing || old_task.finished_at.is_none(), + "Cannot update a task's finished_at time" + ); + if let Some(finished_at) = old_task.finished_at { + remove_task_datetime(wtxn, self.finished_at, finished_at, task.uid)?; + } if let Some(finished_at) = task.finished_at { insert_task_datetime(wtxn, self.finished_at, finished_at, task.uid)?; } diff --git a/crates/index-scheduler/src/queue/test.rs b/crates/index-scheduler/src/queue/test.rs index 5a886b088..eb3314496 100644 --- a/crates/index-scheduler/src/queue/test.rs +++ b/crates/index-scheduler/src/queue/test.rs @@ -165,6 +165,7 @@ fn test_disable_auto_deletion_of_tasks() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { config.cleanup_enabled = false; config.max_number_of_tasks = 2; + None }); index_scheduler @@ -228,6 +229,7 @@ fn test_disable_auto_deletion_of_tasks() { fn test_auto_deletion_of_tasks() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { config.max_number_of_tasks = 2; + None }); index_scheduler @@ -325,6 +327,7 @@ fn test_task_queue_is_full() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { // that's the minimum map size possible config.task_db_size = 1048576; + None }); index_scheduler diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index 3363b2c8f..7f55a9254 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -85,6 +85,7 @@ impl From for AutobatchKind { KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } | KindWithContent::DumpCreation { .. } + | KindWithContent::UpgradeDatabase { .. } | KindWithContent::SnapshotCreation => { panic!("The autobatcher should never be called with tasks that don't apply to an index.") } diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index b224ee6eb..2fc3025d7 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -47,6 +47,9 @@ pub(crate) enum Batch { IndexSwap { task: Task, }, + UpgradeDatabase { + tasks: Vec, + }, } #[derive(Debug)] @@ -105,6 +108,7 @@ impl Batch { } Batch::SnapshotCreation(tasks) | Batch::TaskDeletions(tasks) + | Batch::UpgradeDatabase { tasks } | Batch::IndexDeletion { tasks, .. } => { RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid)) } @@ -138,6 +142,7 @@ impl Batch { | TaskDeletions(_) | SnapshotCreation(_) | Dump(_) + | UpgradeDatabase { .. } | IndexSwap { .. } => None, IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } @@ -162,6 +167,7 @@ impl fmt::Display for Batch { Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?, Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?, Batch::IndexSwap { .. } => f.write_str("IndexSwap")?, + Batch::UpgradeDatabase { .. } => f.write_str("UpgradeDatabase")?, }; match index_uid { Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")), @@ -427,9 +433,24 @@ impl IndexScheduler { let mut current_batch = ProcessingBatch::new(batch_id); let enqueued = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?; - let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued; + let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?; + + // 0. The priority over everything is to upgrade the instance + // There shouldn't be multiple upgrade tasks but just in case we're going to batch all of them at the same time + let upgrade = self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)? & (enqueued | failed); + if !upgrade.is_empty() { + let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, upgrade)?; + // In the case of an upgrade database batch, we want to find back the original batch that tried processing it + // and re-use its id + if let Some(batch_uid) = tasks.last().unwrap().batch_uid { + current_batch.uid = batch_uid; + } + current_batch.processing(&mut tasks); + return Ok(Some((Batch::UpgradeDatabase { tasks }, current_batch))); + } // 1. we get the last task to cancel. + let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued; if let Some(task_id) = to_cancel.max() { let mut task = self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?; diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index 2d20c4d55..eddf8fba7 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -6,6 +6,7 @@ mod process_batch; mod process_dump_creation; mod process_index_operation; mod process_snapshot_creation; +mod process_upgrade; #[cfg(test)] mod test; #[cfg(test)] @@ -183,6 +184,7 @@ impl IndexScheduler { progress.update_progress(BatchProgress::WritingTasksToDisk); processing_batch.finished(); + let mut stop_scheduler_forever = false; let mut wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?; let mut canceled = RoaringBitmap::new(); @@ -221,7 +223,7 @@ impl IndexScheduler { self.queue .tasks .update_task(&mut wtxn, &task) - .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))?; + .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } if let Some(canceled_by) = canceled_by { self.queue.tasks.canceled_by.put(&mut wtxn, &canceled_by, &canceled)?; @@ -272,6 +274,12 @@ impl IndexScheduler { let (task_progress, task_progress_obj) = AtomicTaskStep::new(ids.len() as u32); progress.update_progress(task_progress_obj); + if matches!(err, Error::DatabaseUpgrade(_)) { + tracing::error!( + "Upgrade task failed, tasks won't be processed until the following issue is fixed: {err}" + ); + stop_scheduler_forever = true; + } let error: ResponseError = err.into(); for id in ids.iter() { task_progress.fetch_add(1, Ordering::Relaxed); @@ -279,7 +287,7 @@ impl IndexScheduler { .queue .tasks .get_task(&wtxn, id) - .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))? + .map_err(|e| Error::UnrecoverableError(Box::new(e)))? .ok_or(Error::CorruptedTaskQueue)?; task.status = Status::Failed; task.error = Some(error.clone()); @@ -296,7 +304,7 @@ impl IndexScheduler { self.queue .tasks .update_task(&mut wtxn, &task) - .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))?; + .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } } } @@ -326,7 +334,7 @@ impl IndexScheduler { .queue .tasks .get_task(&rtxn, id) - .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))? + .map_err(|e| Error::UnrecoverableError(Box::new(e)))? .ok_or(Error::CorruptedTaskQueue)?; if let Err(e) = self.queue.delete_persisted_task_data(&task) { tracing::error!( @@ -344,6 +352,10 @@ impl IndexScheduler { #[cfg(test)] self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing); - Ok(TickOutcome::TickAgain(processed_tasks)) + if stop_scheduler_forever { + Ok(TickOutcome::StopProcessingForever) + } else { + Ok(TickOutcome::TickAgain(processed_tasks)) + } } } diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 9a86939a4..7eda1d56f 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -1,9 +1,10 @@ use std::collections::{BTreeSet, HashMap, HashSet}; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::atomic::Ordering; use meilisearch_types::batches::BatchId; use meilisearch_types::heed::{RoTxn, RwTxn}; -use meilisearch_types::milli::progress::Progress; +use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::{self}; use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task}; use milli::update::Settings as MilliSettings; @@ -13,7 +14,7 @@ use super::create_batch::Batch; use crate::processing::{ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, - UpdateIndexProgress, VariableNameStep, + UpdateIndexProgress, }; use crate::utils::{self, swap_index_uid_in_task, ProcessingBatch}; use crate::{Error, IndexScheduler, Result, TaskId}; @@ -297,7 +298,7 @@ impl IndexScheduler { } progress.update_progress(SwappingTheIndexes::SwappingTheIndexes); for (step, swap) in swaps.iter().enumerate() { - progress.update_progress(VariableNameStep::new( + progress.update_progress(VariableNameStep::::new( format!("swapping index {} and {}", swap.indexes.0, swap.indexes.1), step as u32, swaps.len() as u32, @@ -314,6 +315,27 @@ impl IndexScheduler { task.status = Status::Succeeded; Ok(vec![task]) } + Batch::UpgradeDatabase { mut tasks } => { + let KindWithContent::UpgradeDatabase { from } = tasks.last().unwrap().kind else { + unreachable!(); + }; + let ret = catch_unwind(AssertUnwindSafe(|| self.process_upgrade(from, progress))); + match ret { + Ok(Ok(())) => (), + Ok(Err(e)) => return Err(Error::DatabaseUpgrade(Box::new(e))), + Err(_e) => { + return Err(Error::DatabaseUpgrade(Box::new(Error::ProcessBatchPanicked))); + } + } + + for task in tasks.iter_mut() { + task.status = Status::Succeeded; + // Since this task can be retried we must reset its error status + task.error = None; + } + + Ok(tasks) + } } } diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index 9691bdbef..09c1020ac 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -4,16 +4,14 @@ use std::sync::atomic::Ordering; use dump::IndexMetadata; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; -use meilisearch_types::milli::progress::Progress; +use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::{self}; use meilisearch_types::tasks::{Details, KindWithContent, Status, Task}; use time::macros::format_description; use time::OffsetDateTime; -use crate::processing::{ - AtomicDocumentStep, AtomicTaskStep, DumpCreationProgress, VariableNameStep, -}; +use crate::processing::{AtomicDocumentStep, AtomicTaskStep, DumpCreationProgress}; use crate::{Error, IndexScheduler, Result}; impl IndexScheduler { @@ -109,7 +107,11 @@ impl IndexScheduler { let nb_indexes = self.index_mapper.index_mapping.len(&rtxn)? as u32; let mut count = 0; let () = self.index_mapper.try_for_each_index(&rtxn, |uid, index| -> Result<()> { - progress.update_progress(VariableNameStep::new(uid.to_string(), count, nb_indexes)); + progress.update_progress(VariableNameStep::::new( + uid.to_string(), + count, + nb_indexes, + )); count += 1; let rtxn = index.read_txn()?; diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index c6d6e2dc8..3e1a63ce3 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -3,12 +3,12 @@ use std::fs; use std::sync::atomic::Ordering; use meilisearch_types::heed::CompactionOption; -use meilisearch_types::milli::progress::Progress; +use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::{self}; use meilisearch_types::tasks::{Status, Task}; use meilisearch_types::{compression, VERSION_FILE_NAME}; -use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress, VariableNameStep}; +use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress}; use crate::{Error, IndexScheduler, Result}; impl IndexScheduler { @@ -74,7 +74,9 @@ impl IndexScheduler { for (i, result) in index_mapping.iter(&rtxn)?.enumerate() { let (name, uuid) = result?; - progress.update_progress(VariableNameStep::new(name, i as u32, nb_indexes)); + progress.update_progress(VariableNameStep::::new( + name, i as u32, nb_indexes, + )); let index = self.index_mapper.index(&rtxn, name)?; let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string()); fs::create_dir_all(&dst)?; diff --git a/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs new file mode 100644 index 000000000..4feebabc4 --- /dev/null +++ b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs @@ -0,0 +1,49 @@ +use meilisearch_types::milli; +use meilisearch_types::milli::progress::{Progress, VariableNameStep}; + +use crate::{Error, IndexScheduler, Result}; + +impl IndexScheduler { + pub(super) fn process_upgrade( + &self, + db_version: (u32, u32, u32), + progress: Progress, + ) -> Result<()> { + #[cfg(test)] + self.maybe_fail(crate::test_utils::FailureLocation::ProcessUpgrade)?; + + enum UpgradeIndex {} + let indexes = self.index_names()?; + + for (i, uid) in indexes.iter().enumerate() { + progress.update_progress(VariableNameStep::::new( + format!("Upgrading index `{uid}`"), + i as u32, + indexes.len() as u32, + )); + let index = self.index(uid)?; + let mut index_wtxn = index.write_txn()?; + let regen_stats = milli::update::upgrade::upgrade( + &mut index_wtxn, + &index, + db_version, + progress.clone(), + ) + .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; + if regen_stats { + let stats = crate::index_mapper::IndexStats::new(&index, &index_wtxn) + .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; + index_wtxn.commit()?; + + // Release wtxn as soon as possible because it stops us from registering tasks + let mut index_schd_wtxn = self.env.write_txn()?; + self.index_mapper.store_stats_of(&mut index_schd_wtxn, uid, &stats)?; + index_schd_wtxn.commit()?; + } else { + index_wtxn.commit()?; + } + } + + Ok(()) + } +} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap new file mode 100644 index 000000000..3ad2076c8 --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -0,0 +1,110 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [0,1,2,4,] +failed [3,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,2,3,4,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +doggo [2,3,] +girafo [4,] +---------------------------------------------------------------------- +### Index Mapper: +catto: { number_of_documents: 0, field_distribution: {} } +doggo: { number_of_documents: 0, field_distribution: {} } +girafo: { number_of_documents: 0, field_distribution: {} } + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### All Batches: +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } +2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } +3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } +4 {uid: 4, details: {"primaryKey":"leaves"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"girafo":1}}, } +---------------------------------------------------------------------- +### Batch to tasks mapping: +0 [0,] +1 [1,] +2 [2,] +3 [3,] +4 [4,] +---------------------------------------------------------------------- +### Batches Status: +succeeded [0,1,2,4,] +failed [3,] +---------------------------------------------------------------------- +### Batches Kind: +"indexCreation" [1,2,3,4,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Batches Index Tasks: +catto [1,] +doggo [2,3,] +girafo [4,] +---------------------------------------------------------------------- +### Batches Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### Batches Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### Batches Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap new file mode 100644 index 000000000..9e490843e --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -0,0 +1,115 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [1,2,4,5,] +failed [3,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,2,3,4,] +"taskDeletion" [5,] +"upgradeDatabase" [] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +doggo [2,3,] +girafo [4,] +---------------------------------------------------------------------- +### Index Mapper: +catto: { number_of_documents: 0, field_distribution: {} } +doggo: { number_of_documents: 0, field_distribution: {} } +girafo: { number_of_documents: 0, field_distribution: {} } + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### All Batches: +1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } +2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } +3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } +4 {uid: 4, details: {"primaryKey":"leaves"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"girafo":1}}, } +5 {uid: 5, details: {"matchedTasks":1,"deletedTasks":1,"originalFilter":"types=upgradeDatabase"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"taskDeletion":1},"indexUids":{}}, } +---------------------------------------------------------------------- +### Batch to tasks mapping: +1 [1,] +2 [2,] +3 [3,] +4 [4,] +5 [5,] +---------------------------------------------------------------------- +### Batches Status: +succeeded [1,2,4,5,] +failed [3,] +---------------------------------------------------------------------- +### Batches Kind: +"indexCreation" [1,2,3,4,] +"taskDeletion" [5,] +"upgradeDatabase" [] +---------------------------------------------------------------------- +### Batches Index Tasks: +catto [1,] +doggo [2,3,] +girafo [4,] +---------------------------------------------------------------------- +### Batches Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Batches Started At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### Batches Finished At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +[timestamp] [4,] +[timestamp] [5,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap new file mode 100644 index 000000000..eead6e773 --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -0,0 +1,51 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### All Batches: +---------------------------------------------------------------------- +### Batch to tasks mapping: +---------------------------------------------------------------------- +### Batches Status: +---------------------------------------------------------------------- +### Batches Kind: +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +---------------------------------------------------------------------- +### Batches Started At: +---------------------------------------------------------------------- +### Batches Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap new file mode 100644 index 000000000..52f0b61a7 --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -0,0 +1,55 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +---------------------------------------------------------------------- +### Status: +enqueued [0,1,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### All Batches: +---------------------------------------------------------------------- +### Batch to tasks mapping: +---------------------------------------------------------------------- +### Batches Status: +---------------------------------------------------------------------- +### Batches Kind: +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +---------------------------------------------------------------------- +### Batches Started At: +---------------------------------------------------------------------- +### Batches Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap new file mode 100644 index 000000000..96efafc9e --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -0,0 +1,65 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +---------------------------------------------------------------------- +### Status: +enqueued [1,] +failed [0,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### All Batches: +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +---------------------------------------------------------------------- +### Batch to tasks mapping: +0 [0,] +---------------------------------------------------------------------- +### Batches Status: +failed [0,] +---------------------------------------------------------------------- +### Batches Kind: +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap new file mode 100644 index 000000000..bd223298d --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -0,0 +1,68 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +---------------------------------------------------------------------- +### Status: +enqueued [1,2,] +failed [0,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,2,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +doggo [2,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### All Batches: +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +---------------------------------------------------------------------- +### Batch to tasks mapping: +0 [0,] +---------------------------------------------------------------------- +### Batches Status: +failed [0,] +---------------------------------------------------------------------- +### Batches Kind: +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap new file mode 100644 index 000000000..5bb2d57cf --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -0,0 +1,72 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +---------------------------------------------------------------------- +### Status: +enqueued [1,2,3,] +succeeded [0,] +failed [] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [1,2,3,] +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +catto [1,] +doggo [2,3,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +[timestamp] [2,] +[timestamp] [3,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### All Batches: +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +---------------------------------------------------------------------- +### Batch to tasks mapping: +0 [0,] +---------------------------------------------------------------------- +### Batches Status: +succeeded [0,] +failed [] +---------------------------------------------------------------------- +### Batches Kind: +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Batches Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index de12cb25d..a8ef88d56 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -713,68 +713,70 @@ fn basic_get_stats() { let kind = index_creation_task("whalo", "fish"); let _task = index_scheduler.register(kind, None, false).unwrap(); - snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###" - { - "indexes": { - "catto": 1, - "doggo": 1, - "whalo": 1 - }, - "statuses": { - "canceled": 0, - "enqueued": 3, - "failed": 0, - "processing": 0, - "succeeded": 0 - }, - "types": { - "documentAdditionOrUpdate": 0, - "documentDeletion": 0, - "documentEdition": 0, - "dumpCreation": 0, - "indexCreation": 3, - "indexDeletion": 0, - "indexSwap": 0, - "indexUpdate": 0, - "settingsUpdate": 0, - "snapshotCreation": 0, - "taskCancelation": 0, - "taskDeletion": 0 - } - } - "###); + snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#" + { + "indexes": { + "catto": 1, + "doggo": 1, + "whalo": 1 + }, + "statuses": { + "canceled": 0, + "enqueued": 3, + "failed": 0, + "processing": 0, + "succeeded": 0 + }, + "types": { + "documentAdditionOrUpdate": 0, + "documentDeletion": 0, + "documentEdition": 0, + "dumpCreation": 0, + "indexCreation": 3, + "indexDeletion": 0, + "indexSwap": 0, + "indexUpdate": 0, + "settingsUpdate": 0, + "snapshotCreation": 0, + "taskCancelation": 0, + "taskDeletion": 0, + "upgradeDatabase": 0 + } + } + "#); handle.advance_till([Start, BatchCreated]); - snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###" - { - "indexes": { - "catto": 1, - "doggo": 1, - "whalo": 1 - }, - "statuses": { - "canceled": 0, - "enqueued": 2, - "failed": 0, - "processing": 1, - "succeeded": 0 - }, - "types": { - "documentAdditionOrUpdate": 0, - "documentDeletion": 0, - "documentEdition": 0, - "dumpCreation": 0, - "indexCreation": 3, - "indexDeletion": 0, - "indexSwap": 0, - "indexUpdate": 0, - "settingsUpdate": 0, - "snapshotCreation": 0, - "taskCancelation": 0, - "taskDeletion": 0 - } - } - "###); + snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#" + { + "indexes": { + "catto": 1, + "doggo": 1, + "whalo": 1 + }, + "statuses": { + "canceled": 0, + "enqueued": 2, + "failed": 0, + "processing": 1, + "succeeded": 0 + }, + "types": { + "documentAdditionOrUpdate": 0, + "documentDeletion": 0, + "documentEdition": 0, + "dumpCreation": 0, + "indexCreation": 3, + "indexDeletion": 0, + "indexSwap": 0, + "indexUpdate": 0, + "settingsUpdate": 0, + "snapshotCreation": 0, + "taskCancelation": 0, + "taskDeletion": 0, + "upgradeDatabase": 0 + } + } + "#); handle.advance_till([ InsideProcessBatch, @@ -784,36 +786,37 @@ fn basic_get_stats() { Start, BatchCreated, ]); - snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###" - { - "indexes": { - "catto": 1, - "doggo": 1, - "whalo": 1 - }, - "statuses": { - "canceled": 0, - "enqueued": 1, - "failed": 0, - "processing": 1, - "succeeded": 1 - }, - "types": { - "documentAdditionOrUpdate": 0, - "documentDeletion": 0, - "documentEdition": 0, - "dumpCreation": 0, - "indexCreation": 3, - "indexDeletion": 0, - "indexSwap": 0, - "indexUpdate": 0, - "settingsUpdate": 0, - "snapshotCreation": 0, - "taskCancelation": 0, - "taskDeletion": 0 - } - } - "###); + snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#" + { + "indexes": { + "catto": 1, + "doggo": 1, + "whalo": 1 + }, + "statuses": { + "canceled": 0, + "enqueued": 1, + "failed": 0, + "processing": 1, + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 0, + "documentDeletion": 0, + "documentEdition": 0, + "dumpCreation": 0, + "indexCreation": 3, + "indexDeletion": 0, + "indexSwap": 0, + "indexUpdate": 0, + "settingsUpdate": 0, + "snapshotCreation": 0, + "taskCancelation": 0, + "taskDeletion": 0, + "upgradeDatabase": 0 + } + } + "#); // now we make one more batch, the started_at field of the new tasks will be past `second_start_time` handle.advance_till([ @@ -824,36 +827,37 @@ fn basic_get_stats() { Start, BatchCreated, ]); - snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r###" - { - "indexes": { - "catto": 1, - "doggo": 1, - "whalo": 1 - }, - "statuses": { - "canceled": 0, - "enqueued": 0, - "failed": 0, - "processing": 1, - "succeeded": 2 - }, - "types": { - "documentAdditionOrUpdate": 0, - "documentDeletion": 0, - "documentEdition": 0, - "dumpCreation": 0, - "indexCreation": 3, - "indexDeletion": 0, - "indexSwap": 0, - "indexUpdate": 0, - "settingsUpdate": 0, - "snapshotCreation": 0, - "taskCancelation": 0, - "taskDeletion": 0 - } - } - "###); + snapshot!(json_string!(index_scheduler.get_stats().unwrap()), @r#" + { + "indexes": { + "catto": 1, + "doggo": 1, + "whalo": 1 + }, + "statuses": { + "canceled": 0, + "enqueued": 0, + "failed": 0, + "processing": 1, + "succeeded": 2 + }, + "types": { + "documentAdditionOrUpdate": 0, + "documentDeletion": 0, + "documentEdition": 0, + "dumpCreation": 0, + "indexCreation": 3, + "indexDeletion": 0, + "indexSwap": 0, + "indexUpdate": 0, + "settingsUpdate": 0, + "snapshotCreation": 0, + "taskCancelation": 0, + "taskDeletion": 0, + "upgradeDatabase": 0 + } + } + "#); } #[test] @@ -899,7 +903,7 @@ fn create_and_list_index() { index_scheduler.index("kefir").unwrap(); let list = index_scheduler.get_paginated_indexes_stats(&AuthFilter::default(), 0, 20).unwrap(); - snapshot!(json_string!(list, { "[1][0][1].created_at" => "[date]", "[1][0][1].updated_at" => "[date]" }), @r#" + snapshot!(json_string!(list, { "[1][0][1].created_at" => "[date]", "[1][0][1].updated_at" => "[date]", "[1][0][1].used_database_size" => "[bytes]", "[1][0][1].database_size" => "[bytes]" }), @r#" [ 1, [ @@ -907,8 +911,8 @@ fn create_and_list_index() { "kefir", { "number_of_documents": 0, - "database_size": 24576, - "used_database_size": 8192, + "database_size": "[bytes]", + "used_database_size": "[bytes]", "primary_key": null, "field_distribution": {}, "created_at": "[date]", diff --git a/crates/index-scheduler/src/scheduler/test_failure.rs b/crates/index-scheduler/src/scheduler/test_failure.rs index cf835daa3..712fe01a5 100644 --- a/crates/index-scheduler/src/scheduler/test_failure.rs +++ b/crates/index-scheduler/src/scheduler/test_failure.rs @@ -6,6 +6,7 @@ use meili_snap::snapshot; use meilisearch_types::milli::obkv_to_json; use meilisearch_types::milli::update::IndexDocumentsMethod::*; use meilisearch_types::milli::update::Setting; +use meilisearch_types::tasks::Kind; use meilisearch_types::tasks::KindWithContent; use crate::insta_snapshot::snapshot_index_scheduler; @@ -249,3 +250,78 @@ fn panic_in_process_batch_for_index_creation() { // No matter what happens in process_batch, the index_scheduler should be internally consistent snapshot!(snapshot_index_scheduler(&index_scheduler), name: "index_creation_failed"); } + +#[test] +fn upgrade_failure() { + // By starting the index-scheduler at the v1.12.0 an upgrade task should be automatically enqueued + let (index_scheduler, mut handle) = + IndexScheduler::test_with_custom_config(vec![(1, FailureLocation::ProcessUpgrade)], |_| { + Some((1, 12, 0)) + }); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "register_automatic_upgrade_task"); + + let kind = index_creation_task("catto", "mouse"); + let _task = index_scheduler.register(kind, None, false).unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task_while_the_upgrade_task_is_enqueued"); + + handle.advance_one_failed_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_failed"); + + // We can still register tasks + let kind = index_creation_task("doggo", "bone"); + let _task = index_scheduler.register(kind, None, false).unwrap(); + + // But the scheduler is down and won't process anything ever again + handle.scheduler_is_down(); + + // =====> After a restart is it still working as expected? + let (index_scheduler, mut handle) = + handle.restart(index_scheduler, true, vec![(1, FailureLocation::ProcessUpgrade)], |_| { + Some((1, 12, 0)) // the upgrade task should be rerun automatically and nothing else should be enqueued + }); + + handle.advance_one_failed_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_failed_again"); + // We can still register tasks + let kind = index_creation_task("doggo", "bone"); + let _task = index_scheduler.register(kind, None, false).unwrap(); + // And the scheduler is still down and won't process anything ever again + handle.scheduler_is_down(); + + // =====> After a rerestart and without failure can we upgrade the indexes and process the tasks + let (index_scheduler, mut handle) = + handle.restart(index_scheduler, true, vec![], |_| Some((1, 12, 0))); + + handle.advance_one_successful_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_succeeded"); + // We can still register tasks + let kind = index_creation_task("girafo", "leaves"); + let _task = index_scheduler.register(kind, None, false).unwrap(); + // The scheduler is up and running + handle.advance_one_successful_batch(); + handle.advance_one_successful_batch(); + handle.advance_one_failed_batch(); // doggo already exists + handle.advance_one_successful_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_everything"); + + let (upgrade_tasks_ids, _) = index_scheduler + .get_task_ids_from_authorized_indexes( + &crate::Query { types: Some(vec![Kind::UpgradeDatabase]), ..Default::default() }, + &Default::default(), + ) + .unwrap(); + // When deleting the single upgrade task it should remove the associated batch + let _task = index_scheduler + .register( + KindWithContent::TaskDeletion { + query: String::from("types=upgradeDatabase"), + tasks: upgrade_tasks_ids, + }, + None, + false, + ) + .unwrap(); + + handle.advance_one_successful_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_upgrade_tasks"); +} diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index 4be944037..072220b6c 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -1,10 +1,18 @@ use std::io::{BufWriter, Write}; use std::sync::Arc; +use std::time::Duration; +use big_s::S; +use crossbeam_channel::RecvTimeoutError; use file_store::File; use meilisearch_types::document_formats::DocumentFormatError; use meilisearch_types::milli::update::IndexDocumentsMethod::ReplaceDocuments; +use meilisearch_types::milli::update::IndexerConfig; +use meilisearch_types::tasks::KindWithContent; +use meilisearch_types::{versioning, VERSION_FILE_NAME}; +use tempfile::{NamedTempFile, TempDir}; use uuid::Uuid; +use Breakpoint::*; use crate::insta_snapshot::snapshot_index_scheduler; use crate::{Error, IndexScheduler, IndexSchedulerOptions}; @@ -28,20 +36,13 @@ pub(crate) enum FailureLocation { InsideCreateBatch, InsideProcessBatch, PanicInsideProcessBatch, + ProcessUpgrade, AcquiringWtxn, UpdatingTaskAfterProcessBatchSuccess { task_uid: u32 }, UpdatingTaskAfterProcessBatchFailure, CommittingWtxn, } -use big_s::S; -use crossbeam_channel::RecvTimeoutError; -use meilisearch_types::milli::update::IndexerConfig; -use meilisearch_types::tasks::KindWithContent; -use meilisearch_types::VERSION_FILE_NAME; -use tempfile::{NamedTempFile, TempDir}; -use Breakpoint::*; - impl IndexScheduler { /// Blocks the thread until the test handle asks to progress to/through this breakpoint. /// @@ -55,7 +56,6 @@ impl IndexScheduler { /// As soon as we find it, the index scheduler is unblocked but then wait again on the call to /// `test_breakpoint_sdr.send(b, true)`. This message will only be able to send once the /// test asks to progress to the next `(b2, false)`. - #[cfg(test)] pub(crate) fn breakpoint(&self, b: Breakpoint) { // We send two messages. The first one will sync with the call // to `handle.wait_until(b)`. The second one will block until the @@ -75,12 +75,13 @@ impl IndexScheduler { ) -> (Self, IndexSchedulerHandle) { Self::test_with_custom_config(planned_failures, |config| { config.autobatching_enabled = autobatching_enabled; + None }) } pub(crate) fn test_with_custom_config( planned_failures: Vec<(usize, FailureLocation)>, - configuration: impl Fn(&mut IndexSchedulerOptions), + configuration: impl Fn(&mut IndexSchedulerOptions) -> Option<(u32, u32, u32)>, ) -> (Self, IndexSchedulerHandle) { let tempdir = TempDir::new().unwrap(); let (sender, receiver) = crossbeam_channel::bounded(0); @@ -109,10 +110,17 @@ impl IndexScheduler { max_number_of_batched_tasks: usize::MAX, batched_tasks_size_limit: u64::MAX, instance_features: Default::default(), + auto_upgrade: true, // Don't cost much and will ensure the happy path works }; - configuration(&mut options); + let version = configuration(&mut options).unwrap_or_else(|| { + ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ) + }); - let index_scheduler = Self::new(options, sender, planned_failures).unwrap(); + let index_scheduler = Self::new(options, version, sender, planned_failures).unwrap(); // To be 100% consistent between all test we're going to start the scheduler right now // and ensure it's in the expected starting state. @@ -224,6 +232,55 @@ pub struct IndexSchedulerHandle { } impl IndexSchedulerHandle { + /// Restarts the index-scheduler on the same database. + /// To use this function you must give back the index-scheduler that was given to you when + /// creating the handle the first time. + /// If the index-scheduler has been cloned in the meantime you must drop all copy otherwise + /// the function will panic. + pub(crate) fn restart( + self, + index_scheduler: IndexScheduler, + autobatching_enabled: bool, + planned_failures: Vec<(usize, FailureLocation)>, + configuration: impl Fn(&mut IndexSchedulerOptions) -> Option<(u32, u32, u32)>, + ) -> (IndexScheduler, Self) { + drop(index_scheduler); + let Self { _tempdir: tempdir, index_scheduler, test_breakpoint_rcv, last_breakpoint: _ } = + self; + let env = index_scheduler.env.clone(); + drop(index_scheduler); + + // We must ensure that the `run` function has stopped running before restarting the index scheduler + loop { + match test_breakpoint_rcv.recv_timeout(Duration::from_secs(5)) { + Ok((_, true)) => continue, + Ok((b, false)) => { + panic!("Scheduler is not stopped and passed {b:?}") + } + Err(RecvTimeoutError::Timeout) => panic!("The indexing loop is stuck somewhere"), + Err(RecvTimeoutError::Disconnected) => break, + } + } + let closed = env.prepare_for_closing().wait_timeout(Duration::from_secs(5)); + assert!(closed, "The index scheduler couldn't close itself, it seems like someone else is holding the env somewhere"); + + let (scheduler, mut handle) = + IndexScheduler::test_with_custom_config(planned_failures, |config| { + let version = configuration(config); + config.autobatching_enabled = autobatching_enabled; + config.version_file_path = tempdir.path().join(VERSION_FILE_NAME); + config.auth_path = tempdir.path().join("auth"); + config.tasks_path = tempdir.path().join("db_path"); + config.update_file_path = tempdir.path().join("file_store"); + config.indexes_path = tempdir.path().join("indexes"); + config.snapshots_path = tempdir.path().join("snapshots"); + config.dumps_path = tempdir.path().join("dumps"); + version + }); + handle._tempdir = tempdir; + (scheduler, handle) + } + /// Advance the scheduler to the next tick. /// Panic /// * If the scheduler is waiting for a task to be registered. @@ -349,4 +406,18 @@ impl IndexSchedulerHandle { } self.advance_till([AfterProcessing]); } + + // Wait for one failed batch. + #[track_caller] + pub(crate) fn scheduler_is_down(&mut self) { + loop { + match self + .test_breakpoint_rcv + .recv_timeout(std::time::Duration::from_secs(1)) { + Ok((_, true)) => continue, + Ok((b, false)) => panic!("The scheduler was supposed to be down but successfully moved to the next breakpoint: {b:?}"), + Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => break, + } + } + } } diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs new file mode 100644 index 000000000..4e850aa32 --- /dev/null +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -0,0 +1,108 @@ +use anyhow::bail; +use meilisearch_types::heed::{Env, RwTxn}; +use meilisearch_types::tasks::{Details, KindWithContent, Status, Task}; +use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; +use time::OffsetDateTime; +use tracing::info; + +use crate::queue::TaskQueue; +use crate::versioning::Versioning; + +trait UpgradeIndexScheduler { + fn upgrade(&self, env: &Env, wtxn: &mut RwTxn, original: (u32, u32, u32)) + -> anyhow::Result<()>; + fn target_version(&self) -> (u32, u32, u32); +} + +pub fn upgrade_index_scheduler( + env: &Env, + versioning: &Versioning, + from: (u32, u32, u32), + to: (u32, u32, u32), +) -> anyhow::Result<()> { + let current_major = to.0; + let current_minor = to.1; + let current_patch = to.2; + + let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[&V1_12_ToCurrent {}]; + + let start = match from { + (1, 12, _) => 0, + (major, minor, patch) => { + if major > current_major + || (major == current_major && minor > current_minor) + || (major == current_major && minor == current_minor && patch > current_patch) + { + bail!( + "Database version {major}.{minor}.{patch} is higher than the Meilisearch version {current_major}.{current_minor}.{current_patch}. Downgrade is not supported", + ); + } else if major < 1 || (major == current_major && minor < 12) { + bail!( + "Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{current_major}.{current_minor}.{current_patch}", + ); + } else { + bail!("Unknown database version: v{major}.{minor}.{patch}"); + } + } + }; + + let mut current_version = from; + + info!("Upgrading the task queue"); + for upgrade in upgrade_functions[start..].iter() { + let target = upgrade.target_version(); + info!( + "Upgrading from v{}.{}.{} to v{}.{}.{}", + from.0, from.1, from.2, current_version.0, current_version.1, current_version.2 + ); + let mut wtxn = env.write_txn()?; + upgrade.upgrade(env, &mut wtxn, from)?; + versioning.set_version(&mut wtxn, target)?; + wtxn.commit()?; + current_version = target; + } + + let mut wtxn = env.write_txn()?; + let queue = TaskQueue::new(env, &mut wtxn)?; + let uid = queue.next_task_id(&wtxn)?; + queue.register( + &mut wtxn, + &Task { + uid, + batch_uid: None, + enqueued_at: OffsetDateTime::now_utc(), + started_at: None, + finished_at: None, + error: None, + canceled_by: None, + details: Some(Details::UpgradeDatabase { from, to }), + status: Status::Enqueued, + kind: KindWithContent::UpgradeDatabase { from }, + }, + )?; + wtxn.commit()?; + + Ok(()) +} + +#[allow(non_camel_case_types)] +struct V1_12_ToCurrent {} + +impl UpgradeIndexScheduler for V1_12_ToCurrent { + fn upgrade( + &self, + _env: &Env, + _wtxn: &mut RwTxn, + _original: (u32, u32, u32), + ) -> anyhow::Result<()> { + Ok(()) + } + + fn target_version(&self) -> (u32, u32, u32) { + ( + VERSION_MAJOR.parse().unwrap(), + VERSION_MINOR.parse().unwrap(), + VERSION_PATCH.parse().unwrap(), + ) + } +} diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 1f861776f..80a0bb5ff 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -234,6 +234,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { K::TaskCancelation { .. } | K::TaskDeletion { .. } | K::DumpCreation { .. } + | K::UpgradeDatabase { .. } | K::SnapshotCreation => (), }; if let Some(Details::IndexSwap { swaps }) = &mut task.details { @@ -547,6 +548,9 @@ impl crate::IndexScheduler { Details::Dump { dump_uid: _ } => { assert_eq!(kind.as_kind(), Kind::DumpCreation); } + Details::UpgradeDatabase { from: _, to: _ } => { + assert_eq!(kind.as_kind(), Kind::UpgradeDatabase); + } } } diff --git a/crates/index-scheduler/src/versioning.rs b/crates/index-scheduler/src/versioning.rs new file mode 100644 index 000000000..f4c502b6f --- /dev/null +++ b/crates/index-scheduler/src/versioning.rs @@ -0,0 +1,75 @@ +use crate::{upgrade::upgrade_index_scheduler, Result}; +use meilisearch_types::{ + heed::{types::Str, Database, Env, RoTxn, RwTxn}, + milli::heed_codec::version::VersionCodec, + versioning, +}; + +/// The number of database used by queue itself +const NUMBER_OF_DATABASES: u32 = 1; +/// Database const names for the `IndexScheduler`. +mod db_name { + pub const VERSION: &str = "version"; +} +mod entry_name { + pub const MAIN: &str = "main"; +} + +#[derive(Clone)] +pub struct Versioning { + pub version: Database, +} + +impl Versioning { + pub(crate) const fn nb_db() -> u32 { + NUMBER_OF_DATABASES + } + + pub fn get_version(&self, rtxn: &RoTxn) -> Result> { + Ok(self.version.get(rtxn, entry_name::MAIN)?) + } + + pub fn set_version(&self, wtxn: &mut RwTxn, version: (u32, u32, u32)) -> Result<()> { + Ok(self.version.put(wtxn, entry_name::MAIN, &version)?) + } + + pub fn set_current_version(&self, wtxn: &mut RwTxn) -> Result<()> { + let major = versioning::VERSION_MAJOR.parse().unwrap(); + let minor = versioning::VERSION_MINOR.parse().unwrap(); + let patch = versioning::VERSION_PATCH.parse().unwrap(); + self.set_version(wtxn, (major, minor, patch)) + } + + /// Create an index scheduler and start its run loop. + pub(crate) fn new(env: &Env, db_version: (u32, u32, u32)) -> Result { + let mut wtxn = env.write_txn()?; + let version = env.create_database(&mut wtxn, Some(db_name::VERSION))?; + let this = Self { version }; + let from = match this.get_version(&wtxn)? { + Some(version) => version, + // fresh DB: use the db version + None => { + this.set_version(&mut wtxn, db_version)?; + db_version + } + }; + wtxn.commit()?; + + let bin_major: u32 = versioning::VERSION_MAJOR.parse().unwrap(); + let bin_minor: u32 = versioning::VERSION_MINOR.parse().unwrap(); + let bin_patch: u32 = versioning::VERSION_PATCH.parse().unwrap(); + let to = (bin_major, bin_minor, bin_patch); + + if from != to { + upgrade_index_scheduler(env, &this, from, to)?; + } + + // Once we reach this point it means the upgrade process, if there was one is entirely finished + // we can safely say we reached the latest version of the index scheduler + let mut wtxn = env.write_txn()?; + this.set_current_version(&mut wtxn)?; + wtxn.commit()?; + + Ok(this) + } +} diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 8caeb70c2..fa1d4a7d3 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -371,7 +371,8 @@ VectorEmbeddingError , InvalidRequest , BAD_REQUEST ; NotFoundSimilarId , InvalidRequest , BAD_REQUEST ; InvalidDocumentEditionContext , InvalidRequest , BAD_REQUEST ; InvalidDocumentEditionFunctionFilter , InvalidRequest , BAD_REQUEST ; -EditDocumentsByFunctionError , InvalidRequest , BAD_REQUEST +EditDocumentsByFunctionError , InvalidRequest , BAD_REQUEST ; +CouldNotUpgrade , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 6032843aa..7a6faee39 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -114,6 +114,10 @@ pub struct DetailsView { pub settings: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub swaps: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub upgrade_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub upgrade_to: Option, } impl DetailsView { @@ -234,6 +238,18 @@ impl DetailsView { Some(left) } }, + // We want the earliest version + upgrade_from: match (self.upgrade_from.clone(), other.upgrade_from.clone()) { + (None, None) => None, + (None, Some(from)) | (Some(from), None) => Some(from), + (Some(from), Some(_)) => Some(from), + }, + // And the latest + upgrade_to: match (self.upgrade_to.clone(), other.upgrade_to.clone()) { + (None, None) => None, + (None, Some(to)) | (Some(to), None) => Some(to), + (Some(_), Some(to)) => Some(to), + }, } } } @@ -311,6 +327,11 @@ impl From
for DetailsView { Details::IndexSwap { swaps } => { DetailsView { swaps: Some(swaps), ..Default::default() } } + Details::UpgradeDatabase { from, to } => DetailsView { + upgrade_from: Some(format!("v{}.{}.{}", from.0, from.1, from.2)), + upgrade_to: Some(format!("v{}.{}.{}", to.0, to.1, to.2)), + ..Default::default() + }, } } } diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 167cfcd80..6b237ee1f 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -16,7 +16,7 @@ use crate::batches::BatchId; use crate::error::ResponseError; use crate::keys::Key; use crate::settings::{Settings, Unchecked}; -use crate::InstanceUid; +use crate::{versioning, InstanceUid}; pub type TaskId = u32; @@ -50,6 +50,7 @@ impl Task { | SnapshotCreation | TaskCancelation { .. } | TaskDeletion { .. } + | UpgradeDatabase { .. } | IndexSwap { .. } => None, DocumentAdditionOrUpdate { index_uid, .. } | DocumentEdition { index_uid, .. } @@ -84,7 +85,8 @@ impl Task { | KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } | KindWithContent::DumpCreation { .. } - | KindWithContent::SnapshotCreation => None, + | KindWithContent::SnapshotCreation + | KindWithContent::UpgradeDatabase { .. } => None, } } } @@ -150,6 +152,9 @@ pub enum KindWithContent { instance_uid: Option, }, SnapshotCreation, + UpgradeDatabase { + from: (u32, u32, u32), + }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] @@ -175,6 +180,7 @@ impl KindWithContent { KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, KindWithContent::DumpCreation { .. } => Kind::DumpCreation, KindWithContent::SnapshotCreation => Kind::SnapshotCreation, + KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase, } } @@ -185,7 +191,8 @@ impl KindWithContent { DumpCreation { .. } | SnapshotCreation | TaskCancelation { .. } - | TaskDeletion { .. } => vec![], + | TaskDeletion { .. } + | UpgradeDatabase { .. } => vec![], DocumentAdditionOrUpdate { index_uid, .. } | DocumentEdition { index_uid, .. } | DocumentDeletion { index_uid, .. } @@ -262,6 +269,14 @@ impl KindWithContent { }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { + from: (from.0, from.1, from.2), + to: ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ), + }), } } @@ -320,6 +335,14 @@ impl KindWithContent { }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { + from: *from, + to: ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ), + }), } } } @@ -360,6 +383,14 @@ impl From<&KindWithContent> for Option
{ }), KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }), KindWithContent::SnapshotCreation => None, + KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { + from: *from, + to: ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ), + }), } } } @@ -468,6 +499,7 @@ pub enum Kind { TaskDeletion, DumpCreation, SnapshotCreation, + UpgradeDatabase, } impl Kind { @@ -484,6 +516,7 @@ impl Kind { | Kind::TaskCancelation | Kind::TaskDeletion | Kind::DumpCreation + | Kind::UpgradeDatabase | Kind::SnapshotCreation => false, } } @@ -503,6 +536,7 @@ impl Display for Kind { Kind::TaskDeletion => write!(f, "taskDeletion"), Kind::DumpCreation => write!(f, "dumpCreation"), Kind::SnapshotCreation => write!(f, "snapshotCreation"), + Kind::UpgradeDatabase => write!(f, "upgradeDatabase"), } } } @@ -534,6 +568,8 @@ impl FromStr for Kind { Ok(Kind::DumpCreation) } else if kind.eq_ignore_ascii_case("snapshotCreation") { Ok(Kind::SnapshotCreation) + } else if kind.eq_ignore_ascii_case("upgradeDatabase") { + Ok(Kind::UpgradeDatabase) } else { Err(ParseTaskKindError(kind.to_owned())) } @@ -607,6 +643,10 @@ pub enum Details { IndexSwap { swaps: Vec, }, + UpgradeDatabase { + from: (u32, u32, u32), + to: (u32, u32, u32), + }, } impl Details { @@ -627,6 +667,7 @@ impl Details { Self::SettingsUpdate { .. } | Self::IndexInfo { .. } | Self::Dump { .. } + | Self::UpgradeDatabase { .. } | Self::IndexSwap { .. } => (), } @@ -687,7 +728,9 @@ pub fn serialize_duration( #[cfg(test)] mod tests { - use super::Details; + use std::str::FromStr; + + use super::{Details, Kind}; use crate::heed::types::SerdeJson; use crate::heed::{BytesDecode, BytesEncode}; @@ -703,4 +746,13 @@ mod tests { meili_snap::snapshot!(format!("{:?}", details), @r###"TaskDeletion { matched_tasks: 1, deleted_tasks: None, original_filter: "hello" }"###); meili_snap::snapshot!(format!("{:?}", deserialised), @r###"TaskDeletion { matched_tasks: 1, deleted_tasks: None, original_filter: "hello" }"###); } + + #[test] + fn all_kind_can_be_from_str() { + for kind in enum_iterator::all::() { + let s = kind.to_string(); + let k = Kind::from_str(&s).map_err(|e| format!("Could not from_str {s}: {e}")).unwrap(); + assert_eq!(kind, k, "{kind}.to_string() returned {s} which was parsed as {k}"); + } + } } diff --git a/crates/meilisearch-types/src/versioning.rs b/crates/meilisearch-types/src/versioning.rs index 2ec9d9b0c..f009002d1 100644 --- a/crates/meilisearch-types/src/versioning.rs +++ b/crates/meilisearch-types/src/versioning.rs @@ -5,9 +5,39 @@ use std::path::Path; /// The name of the file that contains the version of the database. pub const VERSION_FILE_NAME: &str = "VERSION"; -static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); -static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); -static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); +pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); +pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); +pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); + +/// Persists the version of the current Meilisearch binary to a VERSION file +pub fn update_version_file_for_dumpless_upgrade( + db_path: &Path, + from: (u32, u32, u32), + to: (u32, u32, u32), +) -> Result<(), VersionFileError> { + let (from_major, from_minor, from_patch) = from; + let (to_major, to_minor, to_patch) = to; + + if from_major > to_major + || (from_major == to_major && from_minor > to_minor) + || (from_major == to_major && from_minor == to_minor && from_patch > to_patch) + { + Err(VersionFileError::DowngradeNotSupported { + major: from_major, + minor: from_minor, + patch: from_patch, + }) + } else if from_major < 1 || (from_major == to_major && from_minor < 12) { + Err(VersionFileError::TooOldForAutomaticUpgrade { + major: from_major, + minor: from_minor, + patch: from_patch, + }) + } else { + create_current_version_file(db_path)?; + Ok(()) + } +} /// Persists the version of the current Meilisearch binary to a VERSION file pub fn create_current_version_file(db_path: &Path) -> io::Result<()> { @@ -24,18 +54,7 @@ pub fn create_version_file( fs::write(version_path, format!("{}.{}.{}", major, minor, patch)) } -/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch. -pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> { - let (major, minor, patch) = get_version(db_path)?; - - if major != VERSION_MAJOR || minor != VERSION_MINOR { - return Err(VersionFileError::VersionMismatch { major, minor, patch }.into()); - } - - Ok(()) -} - -pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> { +pub fn get_version(db_path: &Path) -> Result<(u32, u32, u32), VersionFileError> { let version_path = db_path.join(VERSION_FILE_NAME); match fs::read_to_string(version_path) { @@ -47,11 +66,28 @@ pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFi } } -pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> { - let version_components = version.split('.').collect::>(); +pub fn parse_version(version: &str) -> Result<(u32, u32, u32), VersionFileError> { + let version_components = version.trim().split('.').collect::>(); let (major, minor, patch) = match &version_components[..] { - [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()), - _ => return Err(VersionFileError::MalformedVersionFile), + [major, minor, patch] => ( + major.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the major: {e}"), + })?, + minor.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the minor: {e}"), + })?, + patch.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the patch: {e}"), + })?, + ), + _ => { + return Err(VersionFileError::MalformedVersionFile { + context: format!( + "The version contains {} parts instead of 3 (major, minor and patch)", + version_components.len() + ), + }) + } }; Ok((major, minor, patch)) } @@ -64,14 +100,18 @@ pub enum VersionFileError { env!("CARGO_PKG_VERSION").to_string() )] MissingVersionFile, - #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")] - MalformedVersionFile, + #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database. {context}")] + MalformedVersionFile { context: String }, #[error( "Your database version ({major}.{minor}.{patch}) is incompatible with your current engine version ({}).\n\ To migrate data between Meilisearch versions, please follow our guide on https://www.meilisearch.com/docs/learn/update_and_migration/updating.", env!("CARGO_PKG_VERSION").to_string() )] - VersionMismatch { major: String, minor: String, patch: String }, + VersionMismatch { major: u32, minor: u32, patch: u32 }, + #[error("Database version {major}.{minor}.{patch} is higher than the Meilisearch version {VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}. Downgrade is not supported")] + DowngradeNotSupported { major: u32, minor: u32, patch: u32 }, + #[error("Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}")] + TooOldForAutomaticUpgrade { major: u32, minor: u32, patch: u32 }, #[error(transparent)] IoError(#[from] std::io::Error), diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index a97813089..9fc212cc4 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -189,6 +189,7 @@ struct Infos { experimental_drop_search_after: usize, experimental_nb_searches_per_core: usize, experimental_logs_mode: LogMode, + experimental_dumpless_upgrade: bool, experimental_replication_parameters: bool, experimental_enable_logs_route: bool, experimental_reduce_indexing_memory_usage: bool, @@ -235,6 +236,7 @@ impl Infos { experimental_drop_search_after, experimental_nb_searches_per_core, experimental_logs_mode, + experimental_dumpless_upgrade, experimental_replication_parameters, experimental_enable_logs_route, experimental_reduce_indexing_memory_usage, @@ -296,6 +298,7 @@ impl Infos { experimental_drop_search_after: experimental_drop_search_after.into(), experimental_nb_searches_per_core: experimental_nb_searches_per_core.into(), experimental_logs_mode, + experimental_dumpless_upgrade, experimental_replication_parameters, experimental_enable_logs_route: experimental_enable_logs_route | logs_route, experimental_reduce_indexing_memory_usage, diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index a8b8b8eba..4d41c63ea 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -34,11 +34,15 @@ use error::PayloadError; use extractors::payload::PayloadConfig; use index_scheduler::{IndexScheduler, IndexSchedulerOptions}; use meilisearch_auth::AuthController; +use meilisearch_types::milli::constants::VERSION_MAJOR; use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod}; use meilisearch_types::settings::apply_settings_to_builder; use meilisearch_types::tasks::KindWithContent; -use meilisearch_types::versioning::{check_version_file, create_current_version_file}; +use meilisearch_types::versioning::{ + create_current_version_file, get_version, update_version_file_for_dumpless_upgrade, + VersionFileError, VERSION_MINOR, VERSION_PATCH, +}; use meilisearch_types::{compression, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; @@ -206,13 +210,47 @@ enum OnFailure { } pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc)> { + let index_scheduler_opt = IndexSchedulerOptions { + version_file_path: opt.db_path.join(VERSION_FILE_NAME), + auth_path: opt.db_path.join("auth"), + tasks_path: opt.db_path.join("tasks"), + update_file_path: opt.db_path.join("update_files"), + indexes_path: opt.db_path.join("indexes"), + snapshots_path: opt.snapshot_dir.clone(), + dumps_path: opt.dump_dir.clone(), + webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), + webhook_authorization_header: opt.task_webhook_authorization_header.clone(), + task_db_size: opt.max_task_db_size.as_u64() as usize, + index_base_map_size: opt.max_index_size.as_u64() as usize, + enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, + indexer_config: Arc::new((&opt.indexer_options).try_into()?), + autobatching_enabled: true, + cleanup_enabled: !opt.experimental_replication_parameters, + max_number_of_tasks: 1_000_000, + max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks, + batched_tasks_size_limit: opt.experimental_limit_batched_tasks_total_size, + index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize, + index_count: DEFAULT_INDEX_COUNT, + instance_features: opt.to_instance_features(), + auto_upgrade: opt.experimental_dumpless_upgrade, + }; + let bin_major: u32 = VERSION_MAJOR.parse().unwrap(); + let bin_minor: u32 = VERSION_MINOR.parse().unwrap(); + let bin_patch: u32 = VERSION_PATCH.parse().unwrap(); + let binary_version = (bin_major, bin_minor, bin_patch); + let empty_db = is_empty_db(&opt.db_path); let (index_scheduler, auth_controller) = if let Some(ref snapshot_path) = opt.import_snapshot { let snapshot_path_exists = snapshot_path.exists(); // the db is empty and the snapshot exists, import it if empty_db && snapshot_path_exists { match compression::from_tar_gz(snapshot_path, &opt.db_path) { - Ok(()) => open_or_create_database_unchecked(opt, OnFailure::RemoveDb)?, + Ok(()) => open_or_create_database_unchecked( + opt, + index_scheduler_opt, + OnFailure::RemoveDb, + binary_version, // the db is empty + )?, Err(e) => { std::fs::remove_dir_all(&opt.db_path)?; return Err(e); @@ -229,14 +267,18 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< bail!("snapshot doesn't exist at {}", snapshot_path.display()) // the snapshot and the db exist, and we can ignore the snapshot because of the ignore_snapshot_if_db_exists flag } else { - open_or_create_database(opt, empty_db)? + open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)? } } else if let Some(ref path) = opt.import_dump { let src_path_exists = path.exists(); // the db is empty and the dump exists, import it if empty_db && src_path_exists { - let (mut index_scheduler, mut auth_controller) = - open_or_create_database_unchecked(opt, OnFailure::RemoveDb)?; + let (mut index_scheduler, mut auth_controller) = open_or_create_database_unchecked( + opt, + index_scheduler_opt, + OnFailure::RemoveDb, + binary_version, // the db is empty + )?; match import_dump(&opt.db_path, path, &mut index_scheduler, &mut auth_controller) { Ok(()) => (index_scheduler, auth_controller), Err(e) => { @@ -256,10 +298,10 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< // the dump and the db exist and we can ignore the dump because of the ignore_dump_if_db_exists flag // or, the dump is missing but we can ignore that because of the ignore_missing_dump flag } else { - open_or_create_database(opt, empty_db)? + open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)? } } else { - open_or_create_database(opt, empty_db)? + open_or_create_database(opt, index_scheduler_opt, empty_db, binary_version)? }; // We create a loop in a thread that registers snapshotCreation tasks @@ -287,37 +329,15 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< /// Try to start the IndexScheduler and AuthController without checking the VERSION file or anything. fn open_or_create_database_unchecked( opt: &Opt, + index_scheduler_opt: IndexSchedulerOptions, on_failure: OnFailure, + version: (u32, u32, u32), ) -> anyhow::Result<(IndexScheduler, AuthController)> { // we don't want to create anything in the data.ms yet, thus we // wrap our two builders in a closure that'll be executed later. let auth_controller = AuthController::new(&opt.db_path, &opt.master_key); - let instance_features = opt.to_instance_features(); - let index_scheduler_builder = || -> anyhow::Result<_> { - Ok(IndexScheduler::new(IndexSchedulerOptions { - version_file_path: opt.db_path.join(VERSION_FILE_NAME), - auth_path: opt.db_path.join("auth"), - tasks_path: opt.db_path.join("tasks"), - update_file_path: opt.db_path.join("update_files"), - indexes_path: opt.db_path.join("indexes"), - snapshots_path: opt.snapshot_dir.clone(), - dumps_path: opt.dump_dir.clone(), - webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), - webhook_authorization_header: opt.task_webhook_authorization_header.clone(), - task_db_size: opt.max_task_db_size.as_u64() as usize, - index_base_map_size: opt.max_index_size.as_u64() as usize, - enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, - indexer_config: Arc::new((&opt.indexer_options).try_into()?), - autobatching_enabled: true, - cleanup_enabled: !opt.experimental_replication_parameters, - max_number_of_tasks: 1_000_000, - max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks, - batched_tasks_size_limit: opt.experimental_limit_batched_tasks_total_size, - index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize, - index_count: DEFAULT_INDEX_COUNT, - instance_features, - })?) - }; + let index_scheduler_builder = + || -> anyhow::Result<_> { Ok(IndexScheduler::new(index_scheduler_opt, version)?) }; match ( index_scheduler_builder(), @@ -334,16 +354,42 @@ fn open_or_create_database_unchecked( } } +/// Ensures Meilisearch version is compatible with the database, returns an error in case of version mismatch. +/// Returns the version that was contained in the version file +fn check_version(opt: &Opt, binary_version: (u32, u32, u32)) -> anyhow::Result<(u32, u32, u32)> { + let (bin_major, bin_minor, bin_patch) = binary_version; + let (db_major, db_minor, db_patch) = get_version(&opt.db_path)?; + + if db_major != bin_major || db_minor != bin_minor || db_patch > bin_patch { + if opt.experimental_dumpless_upgrade { + update_version_file_for_dumpless_upgrade( + &opt.db_path, + (db_major, db_minor, db_patch), + (bin_major, bin_minor, bin_patch), + )?; + } else { + return Err(VersionFileError::VersionMismatch { + major: db_major, + minor: db_minor, + patch: db_patch, + } + .into()); + } + } + + Ok((db_major, db_minor, db_patch)) +} + /// Ensure you're in a valid state and open the IndexScheduler + AuthController for you. fn open_or_create_database( opt: &Opt, + index_scheduler_opt: IndexSchedulerOptions, empty_db: bool, + binary_version: (u32, u32, u32), ) -> anyhow::Result<(IndexScheduler, AuthController)> { - if !empty_db { - check_version_file(&opt.db_path)?; - } + let version = if !empty_db { check_version(opt, binary_version)? } else { binary_version }; - open_or_create_database_unchecked(opt, OnFailure::KeepDb) + open_or_create_database_unchecked(opt, index_scheduler_opt, OnFailure::KeepDb, version) } fn import_dump( diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index b5aa6b9e7..acf4393d3 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -49,6 +49,7 @@ const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; const MEILI_EXPERIMENTAL_LOGS_MODE: &str = "MEILI_EXPERIMENTAL_LOGS_MODE"; +const MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE: &str = "MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE"; const MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS: &str = "MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS"; const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE"; const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER"; @@ -400,6 +401,13 @@ pub struct Opt { #[serde(default)] pub experimental_logs_mode: LogMode, + /// Experimental dumpless upgrade. For more information, see: + /// + /// When set, Meilisearch will auto-update its database without using a dump. + #[clap(long, env = MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE, default_value_t)] + #[serde(default)] + pub experimental_dumpless_upgrade: bool, + /// Experimental logs route feature. For more information, /// see: /// @@ -535,6 +543,7 @@ impl Opt { experimental_drop_search_after, experimental_nb_searches_per_core, experimental_logs_mode, + experimental_dumpless_upgrade, experimental_enable_logs_route, experimental_replication_parameters, experimental_reduce_indexing_memory_usage, @@ -608,6 +617,10 @@ impl Opt { MEILI_EXPERIMENTAL_LOGS_MODE, experimental_logs_mode.to_string(), ); + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE, + experimental_dumpless_upgrade.to_string(), + ); export_to_env_if_not_present( MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS, experimental_replication_parameters.to_string(), diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index fce2bc8bf..90fdc9c16 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -912,14 +912,14 @@ mod tests { { let params = "types=createIndex"; let err = deserr_query_params::(params).unwrap_err(); - snapshot!(meili_snap::json_string!(err), @r###" + snapshot!(meili_snap::json_string!(err), @r#" { - "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.", + "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.", "code": "invalid_task_types", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_task_types" } - "###); + "#); } } #[test] diff --git a/crates/meilisearch/src/upgrade/mod.rs b/crates/meilisearch/src/upgrade/mod.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/crates/meilisearch/src/upgrade/mod.rs @@ -0,0 +1 @@ + diff --git a/crates/meilisearch/tests/batches/errors.rs b/crates/meilisearch/tests/batches/errors.rs index 2c3484bc1..7f5fedb6a 100644 --- a/crates/meilisearch/tests/batches/errors.rs +++ b/crates/meilisearch/tests/batches/errors.rs @@ -42,7 +42,7 @@ async fn batch_bad_types() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.", + "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.", "code": "invalid_task_types", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_task_types" diff --git a/crates/meilisearch/tests/integration.rs b/crates/meilisearch/tests/integration.rs index 85deb9cdf..7c3b8affe 100644 --- a/crates/meilisearch/tests/integration.rs +++ b/crates/meilisearch/tests/integration.rs @@ -14,6 +14,7 @@ mod snapshot; mod stats; mod swap_indexes; mod tasks; +mod upgrade; mod vector; // Tests are isolated by features in different modules to allow better readability, test diff --git a/crates/meilisearch/tests/tasks/errors.rs b/crates/meilisearch/tests/tasks/errors.rs index 932dd19d4..759531d42 100644 --- a/crates/meilisearch/tests/tasks/errors.rs +++ b/crates/meilisearch/tests/tasks/errors.rs @@ -95,36 +95,36 @@ async fn task_bad_types() { let (response, code) = server.tasks_filter("types=doggo").await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.", + "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.", "code": "invalid_task_types", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_task_types" } - "###); + "#); let (response, code) = server.cancel_tasks("types=doggo").await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.", + "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.", "code": "invalid_task_types", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_task_types" } - "###); + "#); let (response, code) = server.delete_tasks("types=doggo").await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.", + "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.", "code": "invalid_task_types", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_task_types" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs new file mode 100644 index 000000000..f26d99402 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -0,0 +1,87 @@ +mod v1_12; + +use std::path::Path; +use std::{fs, io}; + +use meili_snap::snapshot; +use meilisearch::Opt; + +use crate::common::{default_settings, Server}; + +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { + fs::create_dir_all(&dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} + +#[actix_rt::test] +async fn malformed_version_file() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("VERSION"), "kefir").unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Version file is corrupted and thus Meilisearch is unable to determine the version of the database. The version contains 1 parts instead of 3 (major, minor and patch)"); +} + +#[actix_rt::test] +async fn version_too_old() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.13.0"); +} + +#[actix_rt::test] +async fn version_requires_downgrade() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + let major = meilisearch_types::versioning::VERSION_MAJOR; + let minor = meilisearch_types::versioning::VERSION_MINOR; + let patch = meilisearch_types::versioning::VERSION_PATCH.parse::().unwrap() + 1; + std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Database version 1.13.1 is higher than the Meilisearch version 1.13.0. Downgrade is not supported"); +} + +#[actix_rt::test] +async fn upgrade_to_the_current_version() { + let temp = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(default_settings(temp.path())).await.unwrap(); + drop(server); + + let server = Server::new_with_options(Opt { + experimental_dumpless_upgrade: true, + ..default_settings(temp.path()) + }) + .await + .unwrap(); + // The upgrade tasks should NOT be spawned => task queue is empty + let (tasks, _status) = server.tasks().await; + snapshot!(tasks, @r#" + { + "results": [], + "total": 0, + "limit": 20, + "from": null, + "next": null + } + "#); +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/mod.rs b/crates/meilisearch/tests/upgrade/v1_12/mod.rs new file mode 100644 index 000000000..e84a0aa43 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/mod.rs @@ -0,0 +1 @@ +mod v1_12_0; diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap new file mode 100644 index 000000000..e836fa4b3 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap @@ -0,0 +1,78 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [ + "age", + "surname" + ], + "sortableAttributes": [ + "age" + ], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": [ + "le", + "un" + ], + "nonSeparatorTokens": [], + "separatorTokens": [], + "dictionary": [], + "synonyms": { + "boubou": [ + "kefir" + ] + }, + "distinctAttribute": null, + "proximityPrecision": "byWord", + "typoTolerance": { + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 4, + "twoTypos": 9 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + }, + "faceting": { + "maxValuesPerFacet": 99, + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + }, + "pagination": { + "maxTotalHits": 15 + }, + "embedders": {}, + "searchCutoffMs": 8000, + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fra" + ] + } + ], + "facetSearch": true, + "prefixSearch": "indexingTime" +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_sort_and_filter.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_sort_and_filter.snap new file mode 100644 index 000000000..11ebae310 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_sort_and_filter.snap @@ -0,0 +1,25 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "hits": [ + { + "id": 1, + "name": "kefir", + "surname": [ + "kef", + "kefkef", + "kefirounet", + "boubou" + ], + "age": 1.4, + "description": "kefir est un petit chien blanc très mignon" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 1 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batch_by_batchUids_after_deletion.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batch_by_batchUids_after_deletion.snap new file mode 100644 index 000000000..3fbfb7c60 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batch_by_batchUids_after_deletion.snap @@ -0,0 +1,11 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [], + "total": 0, + "limit": 20, + "from": null, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..6fe049b02 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,505 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 24, + "progress": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "upgradeDatabase": 1 + }, + "indexUids": {} + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 23, + "progress": null, + "details": { + "deletedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.004146631S", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 22, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.102738497S", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 21, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.005108474S", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 20, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.027954894S", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 19, + "progress": null, + "details": { + "deletedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.006903297S", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.000481257S", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000407005S", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000403716S", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000417016S", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT12.086284842S", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "progress": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.011506614S", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "progress": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007640163S", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "progress": null, + "details": { + "searchCutoffMs": 8000 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007307840S", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007391353S", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007445825S", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.012020083S", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007440092S", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "progress": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007565161S", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + }, + { + "uid": 5, + "progress": null, + "details": { + "stopWords": [ + "le", + "un" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.016307263S", + "startedAt": "2025-01-16T16:53:19.913351957Z", + "finishedAt": "2025-01-16T16:53:19.92965922Z" + } + ], + "total": 23, + "limit": 20, + "from": 24, + "next": 4 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..6fe049b02 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,505 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 24, + "progress": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "upgradeDatabase": 1 + }, + "indexUids": {} + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 23, + "progress": null, + "details": { + "deletedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.004146631S", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 22, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.102738497S", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 21, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.005108474S", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 20, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.027954894S", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 19, + "progress": null, + "details": { + "deletedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.006903297S", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.000481257S", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000407005S", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000403716S", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000417016S", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT12.086284842S", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "progress": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.011506614S", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "progress": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007640163S", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "progress": null, + "details": { + "searchCutoffMs": 8000 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007307840S", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007391353S", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007445825S", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.012020083S", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007440092S", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "progress": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007565161S", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + }, + { + "uid": 5, + "progress": null, + "details": { + "stopWords": [ + "le", + "un" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.016307263S", + "startedAt": "2025-01-16T16:53:19.913351957Z", + "finishedAt": "2025-01-16T16:53:19.92965922Z" + } + ], + "total": 23, + "limit": 20, + "from": 24, + "next": 4 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..6fe049b02 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,505 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 24, + "progress": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "upgradeDatabase": 1 + }, + "indexUids": {} + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 23, + "progress": null, + "details": { + "deletedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.004146631S", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 22, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.102738497S", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 21, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.005108474S", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 20, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.027954894S", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 19, + "progress": null, + "details": { + "deletedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.006903297S", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.000481257S", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000407005S", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000403716S", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000417016S", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT12.086284842S", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "progress": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.011506614S", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "progress": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007640163S", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "progress": null, + "details": { + "searchCutoffMs": 8000 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007307840S", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007391353S", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007445825S", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.012020083S", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007440092S", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "progress": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007565161S", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + }, + { + "uid": 5, + "progress": null, + "details": { + "stopWords": [ + "le", + "un" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.016307263S", + "startedAt": "2025-01-16T16:53:19.913351957Z", + "finishedAt": "2025-01-16T16:53:19.92965922Z" + } + ], + "total": 23, + "limit": 20, + "from": 24, + "next": 4 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap new file mode 100644 index 000000000..737b3ed06 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap @@ -0,0 +1,39 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 10, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..b6634ceaf --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,57 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.111055654S", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..b6634ceaf --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,57 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.111055654S", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..b6634ceaf --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,57 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.111055654S", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap new file mode 100644 index 000000000..5fbeb8a59 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap @@ -0,0 +1,40 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 18, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap new file mode 100644 index 000000000..5fbeb8a59 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap @@ -0,0 +1,40 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 18, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap new file mode 100644 index 000000000..737b3ed06 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap @@ -0,0 +1,39 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 10, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/task_by_batchUids_after_deletion.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/task_by_batchUids_after_deletion.snap new file mode 100644 index 000000000..3fbfb7c60 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/task_by_batchUids_after_deletion.snap @@ -0,0 +1,11 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [], + "total": 0, + "limit": 20, + "from": null, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..102e21b73 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,397 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 25, + "batchUid": 24, + "indexUid": null, + "status": "succeeded", + "type": "upgradeDatabase", + "canceledBy": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 24, + "batchUid": 23, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 0 + }, + "error": null, + "duration": "PT0.004146631S", + "enqueuedAt": "2025-01-23T11:38:57.000009177Z", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 23, + "batchUid": 22, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.102738497S", + "enqueuedAt": "2025-01-23T11:36:22.53917994Z", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 22, + "batchUid": 21, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Document doesn't have a `id` attribute: `{\"age\":1.4}`.", + "code": "missing_document_id", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#missing_document_id" + }, + "duration": "PT0.005108474S", + "enqueuedAt": "2025-01-23T11:36:04.115475071Z", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 21, + "batchUid": 20, + "indexUid": "mieli", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "The primary key inference failed as the engine did not find any field ending with `id` in its name. Please specify the primary key manually using the `primaryKey` query parameter.", + "code": "index_primary_key_no_candidate_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_primary_key_no_candidate_found" + }, + "duration": "PT0.027954894S", + "enqueuedAt": "2025-01-23T11:35:53.625718309Z", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 20, + "batchUid": 19, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 19546 + }, + "error": null, + "duration": "PT0.006903297S", + "enqueuedAt": "2025-01-20T11:50:52.862223877Z", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 19, + "batchUid": 18, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.618121963Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.596815611Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "batchUid": 17, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000407005S", + "enqueuedAt": "2025-01-20T11:47:53.498618093Z", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "batchUid": 16, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000403716S", + "enqueuedAt": "2025-01-20T11:47:48.426597451Z", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "batchUid": 15, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "error": null, + "duration": "PT0.000417016S", + "enqueuedAt": "2025-01-20T11:47:42.414346511Z", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "batchUid": 14, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "error": null, + "duration": "PT12.086284842S", + "enqueuedAt": "2025-01-20T11:47:03.079292487Z", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "batchUid": 13, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "error": null, + "duration": "PT0.011506614S", + "enqueuedAt": "2025-01-16T17:18:43.280901282Z", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "batchUid": 12, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "error": null, + "duration": "PT0.007640163S", + "enqueuedAt": "2025-01-16T17:02:52.527382964Z", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "batchUid": 11, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "searchCutoffMs": 8000 + }, + "error": null, + "duration": "PT0.007307840S", + "enqueuedAt": "2025-01-16T17:01:14.100316617Z", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "PT0.007391353S", + "enqueuedAt": "2025-01-16T17:00:29.188815062Z", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "batchUid": 9, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "error": null, + "duration": "PT0.007445825S", + "enqueuedAt": "2025-01-16T17:00:15.759501709Z", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "batchUid": 8, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "error": null, + "duration": "PT0.012020083S", + "enqueuedAt": "2025-01-16T16:59:42.727292501Z", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "batchUid": 7, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "error": null, + "duration": "PT0.007440092S", + "enqueuedAt": "2025-01-16T16:58:41.203145044Z", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "batchUid": 6, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "error": null, + "duration": "PT0.007565161S", + "enqueuedAt": "2025-01-16T16:54:51.927866243Z", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + } + ], + "total": 24, + "limit": 20, + "from": 25, + "next": 5 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..102e21b73 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,397 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 25, + "batchUid": 24, + "indexUid": null, + "status": "succeeded", + "type": "upgradeDatabase", + "canceledBy": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 24, + "batchUid": 23, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 0 + }, + "error": null, + "duration": "PT0.004146631S", + "enqueuedAt": "2025-01-23T11:38:57.000009177Z", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 23, + "batchUid": 22, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.102738497S", + "enqueuedAt": "2025-01-23T11:36:22.53917994Z", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 22, + "batchUid": 21, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Document doesn't have a `id` attribute: `{\"age\":1.4}`.", + "code": "missing_document_id", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#missing_document_id" + }, + "duration": "PT0.005108474S", + "enqueuedAt": "2025-01-23T11:36:04.115475071Z", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 21, + "batchUid": 20, + "indexUid": "mieli", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "The primary key inference failed as the engine did not find any field ending with `id` in its name. Please specify the primary key manually using the `primaryKey` query parameter.", + "code": "index_primary_key_no_candidate_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_primary_key_no_candidate_found" + }, + "duration": "PT0.027954894S", + "enqueuedAt": "2025-01-23T11:35:53.625718309Z", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 20, + "batchUid": 19, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 19546 + }, + "error": null, + "duration": "PT0.006903297S", + "enqueuedAt": "2025-01-20T11:50:52.862223877Z", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 19, + "batchUid": 18, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.618121963Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.596815611Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "batchUid": 17, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000407005S", + "enqueuedAt": "2025-01-20T11:47:53.498618093Z", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "batchUid": 16, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000403716S", + "enqueuedAt": "2025-01-20T11:47:48.426597451Z", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "batchUid": 15, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "error": null, + "duration": "PT0.000417016S", + "enqueuedAt": "2025-01-20T11:47:42.414346511Z", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "batchUid": 14, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "error": null, + "duration": "PT12.086284842S", + "enqueuedAt": "2025-01-20T11:47:03.079292487Z", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "batchUid": 13, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "error": null, + "duration": "PT0.011506614S", + "enqueuedAt": "2025-01-16T17:18:43.280901282Z", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "batchUid": 12, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "error": null, + "duration": "PT0.007640163S", + "enqueuedAt": "2025-01-16T17:02:52.527382964Z", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "batchUid": 11, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "searchCutoffMs": 8000 + }, + "error": null, + "duration": "PT0.007307840S", + "enqueuedAt": "2025-01-16T17:01:14.100316617Z", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "PT0.007391353S", + "enqueuedAt": "2025-01-16T17:00:29.188815062Z", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "batchUid": 9, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "error": null, + "duration": "PT0.007445825S", + "enqueuedAt": "2025-01-16T17:00:15.759501709Z", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "batchUid": 8, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "error": null, + "duration": "PT0.012020083S", + "enqueuedAt": "2025-01-16T16:59:42.727292501Z", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "batchUid": 7, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "error": null, + "duration": "PT0.007440092S", + "enqueuedAt": "2025-01-16T16:58:41.203145044Z", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "batchUid": 6, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "error": null, + "duration": "PT0.007565161S", + "enqueuedAt": "2025-01-16T16:54:51.927866243Z", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + } + ], + "total": 24, + "limit": 20, + "from": 25, + "next": 5 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..102e21b73 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,397 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 25, + "batchUid": 24, + "indexUid": null, + "status": "succeeded", + "type": "upgradeDatabase", + "canceledBy": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 24, + "batchUid": 23, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 0 + }, + "error": null, + "duration": "PT0.004146631S", + "enqueuedAt": "2025-01-23T11:38:57.000009177Z", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 23, + "batchUid": 22, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.102738497S", + "enqueuedAt": "2025-01-23T11:36:22.53917994Z", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 22, + "batchUid": 21, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Document doesn't have a `id` attribute: `{\"age\":1.4}`.", + "code": "missing_document_id", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#missing_document_id" + }, + "duration": "PT0.005108474S", + "enqueuedAt": "2025-01-23T11:36:04.115475071Z", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 21, + "batchUid": 20, + "indexUid": "mieli", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "The primary key inference failed as the engine did not find any field ending with `id` in its name. Please specify the primary key manually using the `primaryKey` query parameter.", + "code": "index_primary_key_no_candidate_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_primary_key_no_candidate_found" + }, + "duration": "PT0.027954894S", + "enqueuedAt": "2025-01-23T11:35:53.625718309Z", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 20, + "batchUid": 19, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 19546 + }, + "error": null, + "duration": "PT0.006903297S", + "enqueuedAt": "2025-01-20T11:50:52.862223877Z", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 19, + "batchUid": 18, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.618121963Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.596815611Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "batchUid": 17, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000407005S", + "enqueuedAt": "2025-01-20T11:47:53.498618093Z", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "batchUid": 16, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000403716S", + "enqueuedAt": "2025-01-20T11:47:48.426597451Z", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "batchUid": 15, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "error": null, + "duration": "PT0.000417016S", + "enqueuedAt": "2025-01-20T11:47:42.414346511Z", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "batchUid": 14, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "error": null, + "duration": "PT12.086284842S", + "enqueuedAt": "2025-01-20T11:47:03.079292487Z", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "batchUid": 13, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "error": null, + "duration": "PT0.011506614S", + "enqueuedAt": "2025-01-16T17:18:43.280901282Z", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "batchUid": 12, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "error": null, + "duration": "PT0.007640163S", + "enqueuedAt": "2025-01-16T17:02:52.527382964Z", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "batchUid": 11, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "searchCutoffMs": 8000 + }, + "error": null, + "duration": "PT0.007307840S", + "enqueuedAt": "2025-01-16T17:01:14.100316617Z", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "PT0.007391353S", + "enqueuedAt": "2025-01-16T17:00:29.188815062Z", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "batchUid": 9, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "error": null, + "duration": "PT0.007445825S", + "enqueuedAt": "2025-01-16T17:00:15.759501709Z", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "batchUid": 8, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "error": null, + "duration": "PT0.012020083S", + "enqueuedAt": "2025-01-16T16:59:42.727292501Z", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "batchUid": 7, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "error": null, + "duration": "PT0.007440092S", + "enqueuedAt": "2025-01-16T16:58:41.203145044Z", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "batchUid": 6, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "error": null, + "duration": "PT0.007565161S", + "enqueuedAt": "2025-01-16T16:54:51.927866243Z", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + } + ], + "total": 24, + "limit": 20, + "from": 25, + "next": 5 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_batchUids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_batchUids_equal_10.snap new file mode 100644 index 000000000..9ca68d4f4 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_batchUids_equal_10.snap @@ -0,0 +1,33 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 10, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..d644e59f3 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,45 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.111055654S", + "enqueuedAt": "2025-01-16T16:45:16.003570092Z", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..d644e59f3 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,45 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.111055654S", + "enqueuedAt": "2025-01-16T16:45:16.003570092Z", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap new file mode 100644 index 000000000..d644e59f3 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap @@ -0,0 +1,45 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +{ + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.111055654S", + "enqueuedAt": "2025-01-16T16:45:16.003570092Z", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 2, + "limit": 20, + "from": 1, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_canceledBy_equal_19.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_canceledBy_equal_19.snap new file mode 100644 index 000000000..8e0249837 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_canceledBy_equal_19.snap @@ -0,0 +1,29 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 18, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_statuses_equal_canceled.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_statuses_equal_canceled.snap new file mode 100644 index 000000000..8e0249837 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_statuses_equal_canceled.snap @@ -0,0 +1,29 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 18, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_uids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_uids_equal_10.snap new file mode 100644 index 000000000..9ca68d4f4 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_uids_equal_10.snap @@ -0,0 +1,33 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 1, + "limit": 20, + "from": 10, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap new file mode 100644 index 000000000..63308dc64 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -0,0 +1,624 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 24, + "progress": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "upgradeDatabase": 1 + }, + "indexUids": {} + }, + "duration": "[duration]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 23, + "progress": null, + "details": { + "deletedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.004146631S", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 22, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.102738497S", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 21, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.005108474S", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 20, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.027954894S", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 19, + "progress": null, + "details": { + "deletedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.006903297S", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 18, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0, + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 2, + "status": { + "succeeded": 1, + "canceled": 1 + }, + "types": { + "documentAdditionOrUpdate": 1, + "taskCancelation": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.000481257S", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000407005S", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000403716S", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "progress": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "taskCancelation": 1 + }, + "indexUids": {} + }, + "duration": "PT0.000417016S", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "progress": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT12.086284842S", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "progress": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.011506614S", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "progress": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007640163S", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "progress": null, + "details": { + "searchCutoffMs": 8000 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007307840S", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007391353S", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "progress": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007445825S", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.012020083S", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "progress": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007440092S", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "progress": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007565161S", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + }, + { + "uid": 5, + "progress": null, + "details": { + "stopWords": [ + "le", + "un" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.016307263S", + "startedAt": "2025-01-16T16:53:19.913351957Z", + "finishedAt": "2025-01-16T16:53:19.92965922Z" + }, + { + "uid": 4, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.087655941S", + "startedAt": "2025-01-16T16:52:32.631145531Z", + "finishedAt": "2025-01-16T16:52:32.718801472Z" + }, + { + "uid": 3, + "progress": null, + "details": { + "sortableAttributes": [ + "age" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.007593573S", + "startedAt": "2025-01-16T16:47:53.677901409Z", + "finishedAt": "2025-01-16T16:47:53.685494982Z" + }, + { + "uid": 2, + "progress": null, + "details": { + "filterableAttributes": [ + "age", + "surname" + ] + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.017769760S", + "startedAt": "2025-01-16T16:47:41.211587682Z", + "finishedAt": "2025-01-16T16:47:41.229357442Z" + }, + { + "uid": 1, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.066095506S", + "startedAt": "2025-01-16T16:47:10.217299609Z", + "finishedAt": "2025-01-16T16:47:10.283395115Z" + }, + { + "uid": 0, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.111055654S", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 25, + "limit": 1000, + "from": 24, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap new file mode 100644 index 000000000..d965b9b68 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -0,0 +1,505 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "uid": 25, + "batchUid": 24, + "indexUid": null, + "status": "succeeded", + "type": "upgradeDatabase", + "canceledBy": null, + "details": { + "upgradeFrom": "v1.12.0", + "upgradeTo": "v1.13.0" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 24, + "batchUid": 23, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 0 + }, + "error": null, + "duration": "PT0.004146631S", + "enqueuedAt": "2025-01-23T11:38:57.000009177Z", + "startedAt": "2025-01-23T11:38:57.012591321Z", + "finishedAt": "2025-01-23T11:38:57.016737952Z" + }, + { + "uid": 23, + "batchUid": 22, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.102738497S", + "enqueuedAt": "2025-01-23T11:36:22.53917994Z", + "startedAt": "2025-01-23T11:36:22.551906856Z", + "finishedAt": "2025-01-23T11:36:22.654645353Z" + }, + { + "uid": 22, + "batchUid": 21, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Document doesn't have a `id` attribute: `{\"age\":1.4}`.", + "code": "missing_document_id", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#missing_document_id" + }, + "duration": "PT0.005108474S", + "enqueuedAt": "2025-01-23T11:36:04.115475071Z", + "startedAt": "2025-01-23T11:36:04.132670526Z", + "finishedAt": "2025-01-23T11:36:04.137779Z" + }, + { + "uid": 21, + "batchUid": 20, + "indexUid": "mieli", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "The primary key inference failed as the engine did not find any field ending with `id` in its name. Please specify the primary key manually using the `primaryKey` query parameter.", + "code": "index_primary_key_no_candidate_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_primary_key_no_candidate_found" + }, + "duration": "PT0.027954894S", + "enqueuedAt": "2025-01-23T11:35:53.625718309Z", + "startedAt": "2025-01-23T11:35:53.631082795Z", + "finishedAt": "2025-01-23T11:35:53.659037689Z" + }, + { + "uid": 20, + "batchUid": 19, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 19546 + }, + "error": null, + "duration": "PT0.006903297S", + "enqueuedAt": "2025-01-20T11:50:52.862223877Z", + "startedAt": "2025-01-20T11:50:52.874106134Z", + "finishedAt": "2025-01-20T11:50:52.881009431Z" + }, + { + "uid": 19, + "batchUid": 18, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 1, + "canceledTasks": 1, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.618121963Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 18, + "batchUid": 18, + "indexUid": "mieli", + "status": "canceled", + "type": "documentAdditionOrUpdate", + "canceledBy": 19, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 0 + }, + "error": null, + "duration": "PT0.000481257S", + "enqueuedAt": "2025-01-20T11:48:04.596815611Z", + "startedAt": "2025-01-20T11:48:04.92820416Z", + "finishedAt": "2025-01-20T11:48:04.928685417Z" + }, + { + "uid": 17, + "batchUid": 17, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000407005S", + "enqueuedAt": "2025-01-20T11:47:53.498618093Z", + "startedAt": "2025-01-20T11:47:53.509403957Z", + "finishedAt": "2025-01-20T11:47:53.509810962Z" + }, + { + "uid": 16, + "batchUid": 16, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing%2Cenqueued" + }, + "error": null, + "duration": "PT0.000403716S", + "enqueuedAt": "2025-01-20T11:47:48.426597451Z", + "startedAt": "2025-01-20T11:47:48.430653005Z", + "finishedAt": "2025-01-20T11:47:48.431056721Z" + }, + { + "uid": 15, + "batchUid": 15, + "indexUid": null, + "status": "succeeded", + "type": "taskCancelation", + "canceledBy": null, + "details": { + "matchedTasks": 0, + "canceledTasks": 0, + "originalFilter": "?statuses=processing" + }, + "error": null, + "duration": "PT0.000417016S", + "enqueuedAt": "2025-01-20T11:47:42.414346511Z", + "startedAt": "2025-01-20T11:47:42.429678617Z", + "finishedAt": "2025-01-20T11:47:42.430095633Z" + }, + { + "uid": 14, + "batchUid": 14, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 19547, + "indexedDocuments": 19546 + }, + "error": null, + "duration": "PT12.086284842S", + "enqueuedAt": "2025-01-20T11:47:03.079292487Z", + "startedAt": "2025-01-20T11:47:03.092181576Z", + "finishedAt": "2025-01-20T11:47:15.178466418Z" + }, + { + "uid": 13, + "batchUid": 13, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "localizedAttributes": [ + { + "attributePatterns": [ + "description" + ], + "locales": [ + "fr" + ] + } + ] + }, + "error": null, + "duration": "PT0.011506614S", + "enqueuedAt": "2025-01-16T17:18:43.280901282Z", + "startedAt": "2025-01-16T17:18:43.29334923Z", + "finishedAt": "2025-01-16T17:18:43.304855844Z" + }, + { + "uid": 12, + "batchUid": 12, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "sortFacetValuesBy": { + "*": "alpha", + "age": "count" + } + } + }, + "error": null, + "duration": "PT0.007640163S", + "enqueuedAt": "2025-01-16T17:02:52.527382964Z", + "startedAt": "2025-01-16T17:02:52.539749853Z", + "finishedAt": "2025-01-16T17:02:52.547390016Z" + }, + { + "uid": 11, + "batchUid": 11, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "searchCutoffMs": 8000 + }, + "error": null, + "duration": "PT0.007307840S", + "enqueuedAt": "2025-01-16T17:01:14.100316617Z", + "startedAt": "2025-01-16T17:01:14.112756687Z", + "finishedAt": "2025-01-16T17:01:14.120064527Z" + }, + { + "uid": 10, + "batchUid": 10, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 99 + }, + "pagination": { + "maxTotalHits": 15 + } + }, + "error": null, + "duration": "PT0.007391353S", + "enqueuedAt": "2025-01-16T17:00:29.188815062Z", + "startedAt": "2025-01-16T17:00:29.201180268Z", + "finishedAt": "2025-01-16T17:00:29.208571621Z" + }, + { + "uid": 9, + "batchUid": 9, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "faceting": { + "maxValuesPerFacet": 100 + }, + "pagination": { + "maxTotalHits": 1000 + } + }, + "error": null, + "duration": "PT0.007445825S", + "enqueuedAt": "2025-01-16T17:00:15.759501709Z", + "startedAt": "2025-01-16T17:00:15.77629445Z", + "finishedAt": "2025-01-16T17:00:15.783740275Z" + }, + { + "uid": 8, + "batchUid": 8, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + }, + "disableOnWords": [ + "kefir" + ], + "disableOnAttributes": [ + "surname" + ] + } + }, + "error": null, + "duration": "PT0.012020083S", + "enqueuedAt": "2025-01-16T16:59:42.727292501Z", + "startedAt": "2025-01-16T16:59:42.744086671Z", + "finishedAt": "2025-01-16T16:59:42.756106754Z" + }, + { + "uid": 7, + "batchUid": 7, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "typoTolerance": { + "minWordSizeForTypos": { + "oneTypo": 4 + } + } + }, + "error": null, + "duration": "PT0.007440092S", + "enqueuedAt": "2025-01-16T16:58:41.203145044Z", + "startedAt": "2025-01-16T16:58:41.2155771Z", + "finishedAt": "2025-01-16T16:58:41.223017192Z" + }, + { + "uid": 6, + "batchUid": 6, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "synonyms": { + "boubou": [ + "kefir" + ] + } + }, + "error": null, + "duration": "PT0.007565161S", + "enqueuedAt": "2025-01-16T16:54:51.927866243Z", + "startedAt": "2025-01-16T16:54:51.940332781Z", + "finishedAt": "2025-01-16T16:54:51.947897942Z" + }, + { + "uid": 5, + "batchUid": 5, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "stopWords": [ + "le", + "un" + ] + }, + "error": null, + "duration": "PT0.016307263S", + "enqueuedAt": "2025-01-16T16:53:19.900781991Z", + "startedAt": "2025-01-16T16:53:19.913351957Z", + "finishedAt": "2025-01-16T16:53:19.92965922Z" + }, + { + "uid": 4, + "batchUid": 4, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.087655941S", + "enqueuedAt": "2025-01-16T16:52:32.618659861Z", + "startedAt": "2025-01-16T16:52:32.631145531Z", + "finishedAt": "2025-01-16T16:52:32.718801472Z" + }, + { + "uid": 3, + "batchUid": 3, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "sortableAttributes": [ + "age" + ] + }, + "error": null, + "duration": "PT0.007593573S", + "enqueuedAt": "2025-01-16T16:47:53.665616298Z", + "startedAt": "2025-01-16T16:47:53.677901409Z", + "finishedAt": "2025-01-16T16:47:53.685494982Z" + }, + { + "uid": 2, + "batchUid": 2, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "filterableAttributes": [ + "age", + "surname" + ] + }, + "error": null, + "duration": "PT0.017769760S", + "enqueuedAt": "2025-01-16T16:47:41.194872913Z", + "startedAt": "2025-01-16T16:47:41.211587682Z", + "finishedAt": "2025-01-16T16:47:41.229357442Z" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.066095506S", + "enqueuedAt": "2025-01-16T16:47:10.200537203Z", + "startedAt": "2025-01-16T16:47:10.217299609Z", + "finishedAt": "2025-01-16T16:47:10.283395115Z" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.111055654S", + "enqueuedAt": "2025-01-16T16:45:16.003570092Z", + "startedAt": "2025-01-16T16:45:16.020248085Z", + "finishedAt": "2025-01-16T16:45:16.131303739Z" + } + ], + "total": 26, + "limit": 1000, + "from": 25, + "next": null +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys.snap new file mode 100644 index 000000000..de7e5ffed --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys.snap @@ -0,0 +1,42 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "name": "Kefir", + "description": "My little kefirino key", + "key": "760c6345918b5ab1d251c1a3e8f9666547628a710d91f6b1d558ba944ef15746", + "uid": "9a77a636-e4e2-4f1a-93ac-978c368fd596", + "actions": [ + "stats.get", + "documents.*" + ], + "indexes": [ + "kefir" + ], + "expiresAt": null, + "createdAt": "2025-01-16T14:43:20.863318893Z", + "updatedAt": "[date]" + }, + { + "name": "Default Search API Key", + "description": "Use it to search from the frontend", + "key": "4d9376547ed779a05dde416148e7e98bd47530e28c500be674c9e60b2accb814", + "uid": "dc699ff0-a053-4956-a46a-912e51b3316b", + "actions": [ + "search" + ], + "indexes": [ + "*" + ], + "expiresAt": null, + "createdAt": "2025-01-16T14:24:46.264041777Z", + "updatedAt": "[date]" + } + ], + "offset": 0, + "limit": 20, + "total": 2 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys_after_removing_kefir.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys_after_removing_kefir.snap new file mode 100644 index 000000000..bdc0d9b0a --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_keys/list_all_keys_after_removing_kefir.snap @@ -0,0 +1,26 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +snapshot_kind: text +--- +{ + "results": [ + { + "name": "kefir", + "description": "the patou", + "key": "4d9376547ed779a05dde416148e7e98bd47530e28c500be674c9e60b2accb814", + "uid": "dc699ff0-a053-4956-a46a-912e51b3316b", + "actions": [ + "search" + ], + "indexes": [ + "*" + ], + "expiresAt": null, + "createdAt": "2025-01-16T14:24:46.264041777Z", + "updatedAt": "[date]" + } + ], + "offset": 0, + "limit": 20, + "total": 1 +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/VERSION b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/VERSION new file mode 100644 index 000000000..32bd932f3 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/VERSION @@ -0,0 +1 @@ +1.12.0 \ No newline at end of file diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/data.mdb new file mode 100644 index 000000000..421b40a8c Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/data.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb new file mode 100644 index 000000000..4c80ffe2c Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb new file mode 100644 index 000000000..c31db3415 Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb new file mode 100644 index 000000000..c99608b77 Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/instance-uid b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/instance-uid new file mode 100644 index 000000000..1584db263 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/instance-uid @@ -0,0 +1 @@ +34ebe3d9-306d-4ead-885d-65d33b4bd60a \ No newline at end of file diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb new file mode 100644 index 000000000..226be2332 Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb new file mode 100644 index 000000000..6d38eab08 Binary files /dev/null and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs new file mode 100644 index 000000000..da809de7f --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -0,0 +1,268 @@ +// This test is the first test of the dumpless upgrade. +// It must test pretty much all the features of meilisearch because the other tests will only tests +// the new features they introduced. + +use manifest_dir_macros::exist_relative_path; +use meili_snap::{json_string, snapshot}; +use meilisearch::Opt; + +use crate::common::{default_settings, Server, Value}; +use crate::json; +use crate::upgrade::copy_dir_all; + +#[actix_rt::test] +async fn import_v1_12_0() { + let temp = tempfile::tempdir().unwrap(); + let original_db_path = exist_relative_path!("tests/upgrade/v1_12/v1_12_0.ms"); + let options = Opt { + experimental_dumpless_upgrade: true, + master_key: Some("kefir".to_string()), + ..default_settings(temp.path()) + }; + copy_dir_all(original_db_path, &options.db_path).unwrap(); + let mut server = Server::new_with_options(options).await.unwrap(); + server.use_api_key("kefir"); + + check_the_keys(&server).await; + check_the_index_scheduler(&server).await; + check_the_index_features(&server).await; +} + +/// We must ensure that the keys database is still working: +/// 1. Check its content +/// 2. Ensure we can still query the keys +/// 3. Ensure we can still update the keys +async fn check_the_keys(server: &Server) { + // All the api keys are still present + let (keys, _) = server.list_api_keys("").await; + snapshot!(json_string!(keys, { ".results[].updatedAt" => "[date]" }), name: "list_all_keys"); + + // We can still query the keys + let (by_uid, _) = server.get_api_key("9a77a636-e4e2-4f1a-93ac-978c368fd596").await; + let (by_key, _) = server + .get_api_key("760c6345918b5ab1d251c1a3e8f9666547628a710d91f6b1d558ba944ef15746") + .await; + + assert_eq!(by_uid, by_key); + snapshot!(json_string!(by_uid, { ".updatedAt" => "[date]" }), @r#" + { + "name": "Kefir", + "description": "My little kefirino key", + "key": "760c6345918b5ab1d251c1a3e8f9666547628a710d91f6b1d558ba944ef15746", + "uid": "9a77a636-e4e2-4f1a-93ac-978c368fd596", + "actions": [ + "stats.get", + "documents.*" + ], + "indexes": [ + "kefir" + ], + "expiresAt": null, + "createdAt": "2025-01-16T14:43:20.863318893Z", + "updatedAt": "[date]" + } + "#); + + // Remove a key + let (_value, status) = server.delete_api_key("9a77a636-e4e2-4f1a-93ac-978c368fd596").await; + snapshot!(status, @"204 No Content"); + + // Update a key + let (value, _) = server + .patch_api_key( + "dc699ff0-a053-4956-a46a-912e51b3316b", + json!({ "name": "kefir", "description": "the patou" }), + ) + .await; + snapshot!(json_string!(value, { ".updatedAt" => "[date]" }), @r#" + { + "name": "kefir", + "description": "the patou", + "key": "4d9376547ed779a05dde416148e7e98bd47530e28c500be674c9e60b2accb814", + "uid": "dc699ff0-a053-4956-a46a-912e51b3316b", + "actions": [ + "search" + ], + "indexes": [ + "*" + ], + "expiresAt": null, + "createdAt": "2025-01-16T14:24:46.264041777Z", + "updatedAt": "[date]" + } + "#); + + // Everything worked + let (keys, _) = server.list_api_keys("").await; + snapshot!(json_string!(keys, { ".results[].updatedAt" => "[date]" }), name: "list_all_keys_after_removing_kefir"); +} + +/// We must ensure the index-scheduler database is still working: +/// 1. We can query the indexes and their metadata +/// 2. The upgrade task has been spawned and has been processed (wait for it to finish or it'll be flaky) +/// 3. Snapshot the whole queue, the tasks and batches should always be the same after update +/// 4. Query the batches and tasks on all filters => the databases should still works +/// 5. Ensure we can still update the queue +/// 5.1. Delete tasks until a batch is removed +/// 5.2. Enqueue a new task +/// 5.3. Create an index +async fn check_the_index_scheduler(server: &Server) { + // All the indexes are still present + let (indexes, _) = server.list_indexes(None, None).await; + snapshot!(indexes, @r#" + { + "results": [ + { + "uid": "kefir", + "createdAt": "2025-01-16T16:45:16.020663157Z", + "updatedAt": "2025-01-23T11:36:22.634859166Z", + "primaryKey": "id" + } + ], + "offset": 0, + "limit": 20, + "total": 1 + } + "#); + // And their metadata are still right + let (stats, _) = server.stats().await; + snapshot!(stats, @r#" + { + "databaseSize": 438272, + "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "indexes": { + "kefir": { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "age": 1, + "description": 1, + "id": 1, + "name": 1, + "surname": 1 + } + } + } + } + "#); + + // Wait until the upgrade has been applied to all indexes to avoid flakyness + let (tasks, _) = server.tasks_filter("types=upgradeDatabase&limit=1").await; + server.wait_task(Value(tasks["results"][0].clone()).uid()).await.succeeded(); + + // Tasks and batches should still work + // We rewrite the first task for all calls because it may be the upgrade database with unknown dates and duration. + // The other tasks should NOT change + let (tasks, _) = server.tasks_filter("limit=1000").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed"); + let (batches, _) = server.batches_filter("limit=1000").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_batch_queue_once_everything_has_been_processed"); + + // Tests all the tasks query parameters + let (tasks, _) = server.tasks_filter("uids=10").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_uids_equal_10"); + let (tasks, _) = server.tasks_filter("batchUids=10").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_batchUids_equal_10"); + let (tasks, _) = server.tasks_filter("statuses=canceled").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_statuses_equal_canceled"); + // types has already been tested above to retrieve the upgrade database + let (tasks, _) = server.tasks_filter("canceledBy=19").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_canceledBy_equal_19"); + let (tasks, _) = server.tasks_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); + let (tasks, _) = server.tasks_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); + let (tasks, _) = server.tasks_filter("beforeStartedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); + let (tasks, _) = server.tasks_filter("afterStartedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41"); + let (tasks, _) = server.tasks_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); + let (tasks, _) = server.tasks_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); + + // Tests all the batches query parameters + let (batches, _) = server.batches_filter("uids=10").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_uids_equal_10"); + let (batches, _) = server.batches_filter("batchUids=10").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_batchUids_equal_10"); + let (batches, _) = server.batches_filter("statuses=canceled").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_statuses_equal_canceled"); + // types has already been tested above to retrieve the upgrade database + let (batches, _) = server.batches_filter("canceledBy=19").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_canceledBy_equal_19"); + let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); + let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); + let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); + let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41"); + let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); + let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; + snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); + + let (stats, _) = server.stats().await; + snapshot!(stats, @r#" + { + "databaseSize": 438272, + "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "indexes": { + "kefir": { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "age": 1, + "description": 1, + "id": 1, + "name": 1, + "surname": 1 + } + } + } + } + "#); + let index = server.index("kefir"); + let (stats, _) = index.stats().await; + snapshot!(stats, @r#" + { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "age": 1, + "description": 1, + "id": 1, + "name": 1, + "surname": 1 + } + } + "#); + + // Delete all the tasks of a specific batch + let (task, _) = server.delete_tasks("batchUids=10").await; + server.wait_task(task.uid()).await.succeeded(); + + let (tasks, _) = server.tasks_filter("batchUids=10").await; + snapshot!(tasks, name: "task_by_batchUids_after_deletion"); + let (tasks, _) = server.batches_filter("batchUids=10").await; + snapshot!(tasks, name: "batch_by_batchUids_after_deletion"); + + let index = server.index("kefirausaurus"); + let (task, _) = index.create(Some("kefid")).await; + server.wait_task(task.uid()).await.succeeded(); +} + +/// Ensuring the index roughly works with filter and sort. +/// More specific test will be made for the next versions everytime they updates a feature +async fn check_the_index_features(server: &Server) { + let kefir = server.index("kefir"); + + let (settings, _) = kefir.settings().await; + snapshot!(settings, name: "kefir_settings"); + + let (results, _status) = + kefir.search_post(json!({ "sort": ["age:asc"], "filter": "surname = kefirounet" })).await; + snapshot!(results, name: "search_with_sort_and_filter"); +} diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 599bc3274..c2444fab6 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -187,7 +187,7 @@ fn export_a_dump( db_path: PathBuf, dump_dir: PathBuf, skip_enqueued_tasks: bool, - detected_version: (String, String, String), + detected_version: (u32, u32, u32), ) -> Result<(), anyhow::Error> { let started_at = OffsetDateTime::now_utc(); @@ -253,12 +253,7 @@ fn export_a_dump( if status == Status::Enqueued { let content_file = file_store.get_update(content_file_uuid)?; - if ( - detected_version.0.as_str(), - detected_version.1.as_str(), - detected_version.2.as_str(), - ) < ("1", "12", "0") - { + if (detected_version.0, detected_version.1, detected_version.2) < (1, 12, 0) { eprintln!("Dumping the enqueued tasks reading them in obkv format..."); let reader = DocumentsBatchReader::from_reader(content_file).with_context(|| { @@ -303,7 +298,7 @@ fn export_a_dump( for result in index_mapping.iter(&rtxn)? { let (uid, uuid) = result?; let index_path = db_path.join("indexes").join(uuid.to_string()); - let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| { + let index = Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| { format!("While trying to open the index at path {:?}", index_path.display()) })?; diff --git a/crates/meilitool/src/upgrade/mod.rs b/crates/meilitool/src/upgrade/mod.rs index 2d5230341..bfaa6683d 100644 --- a/crates/meilitool/src/upgrade/mod.rs +++ b/crates/meilitool/src/upgrade/mod.rs @@ -14,8 +14,8 @@ use crate::upgrade::v1_11::v1_10_to_v1_11; pub struct OfflineUpgrade { pub db_path: PathBuf, - pub current_version: (String, String, String), - pub target_version: (String, String, String), + pub current_version: (u32, u32, u32), + pub target_version: (u32, u32, u32), } impl OfflineUpgrade { @@ -50,7 +50,7 @@ impl OfflineUpgrade { let upgrade_list = [ ( - v1_9_to_v1_10 as fn(&Path, &str, &str, &str) -> Result<(), anyhow::Error>, + v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, "1", "10", "0", @@ -62,18 +62,14 @@ impl OfflineUpgrade { let no_upgrade: usize = upgrade_list.len(); - let (current_major, current_minor, current_patch) = &self.current_version; + let (current_major, current_minor, current_patch) = self.current_version; - let start_at = match ( - current_major.as_str(), - current_minor.as_str(), - current_patch.as_str(), - ) { - ("1", "9", _) => 0, - ("1", "10", _) => 1, - ("1", "11", _) => 2, - ("1", "12", "0" | "1" | "2") => 3, - ("1", "12", "3" | "4" | "5") => no_upgrade, + let start_at = match (current_major, current_minor, current_patch) { + (1, 9, _) => 0, + (1, 10, _) => 1, + (1, 11, _) => 2, + (1, 12, 0..=2) => 3, + (1, 12, 3..=5) => no_upgrade, _ => { bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from versions in range [{}-{}]", FIRST_SUPPORTED_UPGRADE_FROM_VERSION, @@ -81,16 +77,13 @@ impl OfflineUpgrade { } }; - let (target_major, target_minor, target_patch) = &self.target_version; + let (target_major, target_minor, target_patch) = self.target_version; - let ends_at = match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) { - ("1", "10", _) => 0, - ("1", "11", _) => 1, - ("1", "12", "0" | "1" | "2") => 2, - ("1", "12", "3" | "4" | "5") => 3, - (major, _, _) if major.starts_with('v') => { - bail!("Target version must not starts with a `v`. Instead of writing `v1.9.0` write `1.9.0` for example.") - } + let ends_at = match (target_major, target_minor, target_patch) { + (1, 10, _) => 0, + (1, 11, _) => 1, + (1, 12, x) if x == 0 || x == 1 || x == 2 => 2, + (1, 12, 3..=5) => 3, _ => { bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to versions in range [{}-{}]", FIRST_SUPPORTED_UPGRADE_TO_VERSION, @@ -102,8 +95,13 @@ impl OfflineUpgrade { if start_at == no_upgrade { println!("No upgrade operation to perform, writing VERSION file"); - create_version_file(&self.db_path, target_major, target_minor, target_patch) - .context("while writing VERSION file after the upgrade")?; + create_version_file( + &self.db_path, + &target_major.to_string(), + &target_minor.to_string(), + &target_patch.to_string(), + ) + .context("while writing VERSION file after the upgrade")?; println!("Success"); return Ok(()); } diff --git a/crates/meilitool/src/upgrade/v1_10.rs b/crates/meilitool/src/upgrade/v1_10.rs index a35fd4184..043520e82 100644 --- a/crates/meilitool/src/upgrade/v1_10.rs +++ b/crates/meilitool/src/upgrade/v1_10.rs @@ -153,9 +153,9 @@ fn date_round_trip( pub fn v1_9_to_v1_10( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.9.0 to v1.10.0"); // 2 changes here diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index e24a35e8b..44aeb125f 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -16,9 +16,9 @@ use crate::{try_opening_database, try_opening_poly_database}; pub fn v1_10_to_v1_11( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.10.0 to v1.11.0"); diff --git a/crates/meilitool/src/upgrade/v1_12.rs b/crates/meilitool/src/upgrade/v1_12.rs index 593fb833c..3ad171c31 100644 --- a/crates/meilitool/src/upgrade/v1_12.rs +++ b/crates/meilitool/src/upgrade/v1_12.rs @@ -25,9 +25,9 @@ use crate::uuid_codec::UuidCodec; pub fn v1_11_to_v1_12( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.11.0 to v1.12.0"); @@ -38,13 +38,13 @@ pub fn v1_11_to_v1_12( pub fn v1_12_to_v1_12_3( db_path: &Path, - origin_major: &str, - origin_minor: &str, - origin_patch: &str, + origin_major: u32, + origin_minor: u32, + origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.12.{{0, 1, 2}} to v1.12.3"); - if origin_minor == "12" { + if origin_minor == 12 { rebuild_field_distribution(db_path)?; } else { println!("Not rebuilding field distribution as it wasn't corrupted coming from v{origin_major}.{origin_minor}.{origin_patch}"); @@ -173,10 +173,11 @@ fn rebuild_field_distribution(db_path: &Path) -> anyhow::Result<()> { println!("\t- Rebuilding field distribution"); - let index = meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path) - .with_context(|| { - format!("while opening index {uid} at '{}'", index_path.display()) - })?; + let index = + meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path, false) + .with_context(|| { + format!("while opening index {uid} at '{}'", index_path.display()) + })?; let mut index_txn = index.write_txn()?; diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index d22829045..5eb89ea53 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -20,6 +20,7 @@ bytemuck = { version = "1.21.0", features = ["extern_crate_alloc"] } byteorder = "1.5.0" charabia = { version = "0.9.2", default-features = false } concat-arrays = "0.1.2" +convert_case = "0.6.0" crossbeam-channel = "0.5.14" deserr = "0.6.3" either = { version = "1.13.0", features = ["serde"] } diff --git a/crates/milli/src/constants.rs b/crates/milli/src/constants.rs index 3dd787f1c..39b449661 100644 --- a/crates/milli/src/constants.rs +++ b/crates/milli/src/constants.rs @@ -1,2 +1,6 @@ +pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); +pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); +pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); + pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors"; pub const RESERVED_GEO_FIELD_NAME: &str = "_geo"; diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 79e7770f0..c8ed1912f 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -74,6 +74,8 @@ pub enum InternalError { AbortedIndexation, #[error("The matching words list contains at least one invalid member")] InvalidMatchingWords, + #[error("Cannot upgrade to the following version: v{0}.{1}.{2}.")] + CannotUpgradeToVersion(u32, u32, u32), #[error(transparent)] ArroyError(#[from] arroy::Error), #[error(transparent)] diff --git a/crates/milli/src/heed_codec/mod.rs b/crates/milli/src/heed_codec/mod.rs index 575b886bd..45f7f7075 100644 --- a/crates/milli/src/heed_codec/mod.rs +++ b/crates/milli/src/heed_codec/mod.rs @@ -10,6 +10,7 @@ mod roaring_bitmap_length; mod str_beu32_codec; mod str_ref; mod str_str_u8_codec; +pub mod version; pub use byte_slice_ref::BytesRefCodec; use heed::BoxedError; diff --git a/crates/milli/src/heed_codec/version.rs b/crates/milli/src/heed_codec/version.rs new file mode 100644 index 000000000..c73b581e3 --- /dev/null +++ b/crates/milli/src/heed_codec/version.rs @@ -0,0 +1,43 @@ +use std::borrow::Cow; +use std::mem::{size_of, size_of_val}; + +use byteorder::{BigEndian, ByteOrder}; +use heed::{BoxedError, BytesDecode, BytesEncode}; + +const VERSION_SIZE: usize = std::mem::size_of::() * 3; + +#[derive(thiserror::Error, Debug)] +#[error( + "Could not decode the version: Expected {VERSION_SIZE} bytes but instead received {0} bytes" +)] +pub struct DecodeVersionError(usize); + +pub struct VersionCodec; +impl<'a> BytesEncode<'a> for VersionCodec { + type EItem = (u32, u32, u32); + + fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { + let mut ret = Vec::with_capacity(size_of::() * 3); + ret.extend(&item.0.to_be_bytes()); + ret.extend(&item.1.to_be_bytes()); + ret.extend(&item.2.to_be_bytes()); + Ok(Cow::Owned(ret)) + } +} +impl<'a> BytesDecode<'a> for VersionCodec { + type DItem = (u32, u32, u32); + + fn bytes_decode(bytes: &'a [u8]) -> Result { + if bytes.len() != VERSION_SIZE { + Err(Box::new(DecodeVersionError(bytes.len()))) + } else { + let major = BigEndian::read_u32(bytes); + let bytes = &bytes[size_of_val(&major)..]; + let minor = BigEndian::read_u32(bytes); + let bytes = &bytes[size_of_val(&major)..]; + let patch = BigEndian::read_u32(bytes); + + Ok((major, minor, patch)) + } + } +} diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 9829df2ee..944fb6cd4 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -10,7 +10,7 @@ use roaring::RoaringBitmap; use rstar::RTree; use serde::{Deserialize, Serialize}; -use crate::constants::RESERVED_VECTORS_FIELD_NAME; +use crate::constants::{self, RESERVED_VECTORS_FIELD_NAME}; use crate::documents::PrimaryKey; use crate::error::{InternalError, UserError}; use crate::fields_ids_map::FieldsIdsMap; @@ -18,6 +18,7 @@ use crate::heed_codec::facet::{ FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec, FieldIdCodec, OrderedF64Codec, }; +use crate::heed_codec::version::VersionCodec; use crate::heed_codec::{BEU16StrCodec, FstSetCodec, StrBEU16Codec, StrRefCodec}; use crate::order_by_map::OrderByMap; use crate::proximity::ProximityPrecision; @@ -33,6 +34,7 @@ pub const DEFAULT_MIN_WORD_LEN_ONE_TYPO: u8 = 5; pub const DEFAULT_MIN_WORD_LEN_TWO_TYPOS: u8 = 9; pub mod main_key { + pub const VERSION_KEY: &str = "version"; pub const CRITERIA_KEY: &str = "criteria"; pub const DISPLAYED_FIELDS_KEY: &str = "displayed-fields"; pub const DISTINCT_FIELD_KEY: &str = "distinct-field-key"; @@ -176,6 +178,7 @@ impl Index { path: P, created_at: time::OffsetDateTime, updated_at: time::OffsetDateTime, + creation: bool, ) -> Result { use db_name::*; @@ -223,12 +226,9 @@ impl Index { let vector_arroy = env.create_database(&mut wtxn, Some(VECTOR_ARROY))?; let documents = env.create_database(&mut wtxn, Some(DOCUMENTS))?; - wtxn.commit()?; - Index::set_creation_dates(&env, main, created_at, updated_at)?; - - Ok(Index { - env, + let this = Index { + env: env.clone(), main, external_documents_ids, word_docids, @@ -253,12 +253,31 @@ impl Index { vector_arroy, embedder_category_id, documents, - }) + }; + if this.get_version(&wtxn)?.is_none() && creation { + this.put_version( + &mut wtxn, + ( + constants::VERSION_MAJOR.parse().unwrap(), + constants::VERSION_MINOR.parse().unwrap(), + constants::VERSION_PATCH.parse().unwrap(), + ), + )?; + } + wtxn.commit()?; + + Index::set_creation_dates(&this.env, this.main, created_at, updated_at)?; + + Ok(this) } - pub fn new>(options: heed::EnvOpenOptions, path: P) -> Result { + pub fn new>( + options: heed::EnvOpenOptions, + path: P, + creation: bool, + ) -> Result { let now = time::OffsetDateTime::now_utc(); - Self::new_with_creation_dates(options, path, now, now) + Self::new_with_creation_dates(options, path, now, now, creation) } fn set_creation_dates( @@ -331,6 +350,26 @@ impl Index { self.env.prepare_for_closing() } + /* version */ + + /// Writes the version of the database. + pub(crate) fn put_version( + &self, + wtxn: &mut RwTxn<'_>, + (major, minor, patch): (u32, u32, u32), + ) -> heed::Result<()> { + self.main.remap_types::().put( + wtxn, + main_key::VERSION_KEY, + &(major, minor, patch), + ) + } + + /// Get the version of the database. `None` if it was never set. + pub(crate) fn get_version(&self, rtxn: &RoTxn<'_>) -> heed::Result> { + self.main.remap_types::().get(rtxn, main_key::VERSION_KEY) + } + /* documents ids */ /// Writes the documents ids that corresponds to the user-ids-documents-ids FST. @@ -1768,7 +1807,7 @@ pub(crate) mod tests { let mut options = EnvOpenOptions::new(); options.map_size(size); let _tempdir = TempDir::new_in(".").unwrap(); - let inner = Index::new(options, _tempdir.path()).unwrap(); + let inner = Index::new(options, _tempdir.path(), true).unwrap(); let indexer_config = IndexerConfig::default(); let index_documents_config = IndexDocumentsConfig::default(); Self { inner, indexer_config, index_documents_config, _tempdir } diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index 622ec9842..3837e173a 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -1,5 +1,6 @@ use std::any::TypeId; use std::borrow::Cow; +use std::marker::PhantomData; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; @@ -88,19 +89,24 @@ impl Step for AtomicSubStep { } } +#[doc(hidden)] +pub use convert_case as _private_convert_case; +#[doc(hidden)] +pub use enum_iterator as _private_enum_iterator; + #[macro_export] macro_rules! make_enum_progress { ($visibility:vis enum $name:ident { $($variant:ident,)+ }) => { #[repr(u8)] - #[derive(Debug, Clone, Copy, PartialEq, Eq, Sequence)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, $crate::progress::_private_enum_iterator::Sequence)] #[allow(clippy::enum_variant_names)] $visibility enum $name { $($variant),+ } - impl Step for $name { - fn name(&self) -> Cow<'static, str> { - use convert_case::Casing; + impl $crate::progress::Step for $name { + fn name(&self) -> std::borrow::Cow<'static, str> { + use $crate::progress::_private_convert_case::Casing; match self { $( @@ -114,6 +120,7 @@ macro_rules! make_enum_progress { } fn total(&self) -> u32 { + use $crate::progress::_private_enum_iterator::Sequence; Self::CARDINALITY as u32 } } @@ -153,3 +160,41 @@ pub struct ProgressStepView { pub finished: u32, pub total: u32, } + +/// Used when the name can change but it's still the same step. +/// To avoid conflicts on the `TypeId`, create a unique type every time you use this step: +/// ```text +/// enum UpgradeVersion {} +/// +/// progress.update_progress(VariableNameStep::::new( +/// "v1 to v2", +/// 0, +/// 10, +/// )); +/// ``` +pub struct VariableNameStep { + name: String, + current: u32, + total: u32, + phantom: PhantomData, +} + +impl VariableNameStep { + pub fn new(name: impl Into, current: u32, total: u32) -> Self { + Self { name: name.into(), current, total, phantom: PhantomData } + } +} + +impl Step for VariableNameStep { + fn name(&self) -> Cow<'static, str> { + self.name.clone().into() + } + + fn current(&self) -> u32 { + self.current + } + + fn total(&self) -> u32 { + self.total + } +} diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index fc15b5f12..99d5dc033 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -17,7 +17,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/src/update/mod.rs b/crates/milli/src/update/mod.rs index 5888a20db..68268db35 100644 --- a/crates/milli/src/update/mod.rs +++ b/crates/milli/src/update/mod.rs @@ -21,6 +21,7 @@ mod indexer_config; pub mod new; pub(crate) mod settings; mod update_step; +pub mod upgrade; mod word_prefix_docids; mod words_prefix_integer_docids; mod words_prefixes_fst; diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs new file mode 100644 index 000000000..5b7fda303 --- /dev/null +++ b/crates/milli/src/update/upgrade/mod.rs @@ -0,0 +1,68 @@ +mod v1_12; + +use heed::RwTxn; +use v1_12::{V1_12_3_To_Current, V1_12_To_V1_12_3}; + +use crate::progress::{Progress, VariableNameStep}; +use crate::{Index, InternalError, Result}; + +trait UpgradeIndex { + /// Returns true if the index scheduler must regenerate its cached stats + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + original: (u32, u32, u32), + progress: Progress, + ) -> Result; + fn target_version(&self) -> (u32, u32, u32); +} + +/// Return true if the cached stats of the index must be regenerated +pub fn upgrade( + wtxn: &mut RwTxn, + index: &Index, + db_version: (u32, u32, u32), + progress: Progress, +) -> Result { + let from = index.get_version(wtxn)?.unwrap_or(db_version); + let upgrade_functions: &[&dyn UpgradeIndex] = &[&V1_12_To_V1_12_3 {}, &V1_12_3_To_Current()]; + + let start = match from { + (1, 12, 0..=2) => 0, + (1, 12, 3..) => 1, + // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. + (1, 13, _) => return Ok(false), + (major, minor, patch) => { + return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) + } + }; + + enum UpgradeVersion {} + let upgrade_path = &upgrade_functions[start..]; + + let mut current_version = from; + + let mut regenerate_stats = false; + for (i, upgrade) in upgrade_path.iter().enumerate() { + let target = upgrade.target_version(); + progress.update_progress(VariableNameStep::::new( + format!( + "Upgrading from v{}.{}.{} to v{}.{}.{}", + current_version.0, + current_version.1, + current_version.2, + target.0, + target.1, + target.2 + ), + i as u32, + upgrade_path.len() as u32, + )); + regenerate_stats |= upgrade.upgrade(wtxn, index, from, progress.clone())?; + index.put_version(wtxn, target)?; + current_version = target; + } + + Ok(regenerate_stats) +} diff --git a/crates/milli/src/update/upgrade/v1_12.rs b/crates/milli/src/update/upgrade/v1_12.rs new file mode 100644 index 000000000..e48ecfe36 --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_12.rs @@ -0,0 +1,56 @@ +use heed::RwTxn; + +use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; +use crate::progress::Progress; +use crate::{make_enum_progress, Index, Result}; + +use super::UpgradeIndex; + +#[allow(non_camel_case_types)] +pub(super) struct V1_12_To_V1_12_3 {} + +impl UpgradeIndex for V1_12_To_V1_12_3 { + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + _original: (u32, u32, u32), + progress: Progress, + ) -> Result { + make_enum_progress! { + enum FieldDistribution { + RebuildingFieldDistribution, + } + }; + progress.update_progress(FieldDistribution::RebuildingFieldDistribution); + crate::update::new::reindex::field_distribution(index, wtxn, &progress)?; + Ok(true) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 12, 3) + } +} + +#[allow(non_camel_case_types)] +pub(super) struct V1_12_3_To_Current(); + +impl UpgradeIndex for V1_12_3_To_Current { + fn upgrade( + &self, + _wtxn: &mut RwTxn, + _index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + ( + VERSION_MAJOR.parse().unwrap(), + VERSION_MINOR.parse().unwrap(), + VERSION_PATCH.parse().unwrap(), + ) + } +} diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index ced81409d..db9f86357 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -15,7 +15,7 @@ fn test_facet_distribution_with_no_facet_values() { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index 30690969b..662715638 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -34,7 +34,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 304059915..d47c9539d 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -264,7 +264,7 @@ fn criteria_ascdesc() { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(12 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index d33d79e54..b640fa910 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -110,7 +110,7 @@ fn test_typo_disabled_on_word() { let tmp = tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let index = Index::new(options, tmp.path()).unwrap(); + let index = Index::new(options, tmp.path(), true).unwrap(); let doc1: Object = from_value(json!({ "id": 1usize, "data": "zealand" })).unwrap(); let doc2: Object = from_value(json!({ "id": 2usize, "data": "zearand" })).unwrap();