Add the new tasks with most of the job done

crates/meilisearch/src/analytics/segment_analytics.rs

@@ -189,6 +189,7 @@ struct Infos {
     experimental_drop_search_after: usize,
     experimental_nb_searches_per_core: usize,
     experimental_logs_mode: LogMode,
+    experimental_dumpless_upgrade: bool,
     experimental_replication_parameters: bool,
     experimental_enable_logs_route: bool,
     experimental_reduce_indexing_memory_usage: bool,
@@ -235,6 +236,7 @@ impl Infos {
             experimental_drop_search_after,
             experimental_nb_searches_per_core,
             experimental_logs_mode,
+            experimental_dumpless_upgrade,
             experimental_replication_parameters,
             experimental_enable_logs_route,
             experimental_reduce_indexing_memory_usage,
@@ -296,6 +298,7 @@ impl Infos {
             experimental_drop_search_after: experimental_drop_search_after.into(),
             experimental_nb_searches_per_core: experimental_nb_searches_per_core.into(),
             experimental_logs_mode,
+            experimental_dumpless_upgrade,
             experimental_replication_parameters,
             experimental_enable_logs_route: experimental_enable_logs_route | logs_route,
             experimental_reduce_indexing_memory_usage,

crates/meilisearch/src/lib.rs

@@ -32,13 +32,16 @@ use analytics::Analytics;
 use anyhow::bail;
 use error::PayloadError;
 use extractors::payload::PayloadConfig;
+use index_scheduler::upgrade::upgrade_task_queue;
 use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
 use meilisearch_auth::AuthController;
 use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
 use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::KindWithContent;
-use meilisearch_types::versioning::{check_version_file, create_current_version_file};
+use meilisearch_types::versioning::{
+    create_current_version_file, get_version, VersionFileError, VERSION_MAJOR, VERSION_MINOR,
+};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
@@ -316,6 +319,7 @@ fn open_or_create_database_unchecked(
             index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize,
             index_count: DEFAULT_INDEX_COUNT,
             instance_features,
+            auto_upgrade: opt.experimental_dumpless_upgrade,
         })?)
     };
 
@@ -334,13 +338,36 @@ fn open_or_create_database_unchecked(
     }
 }
 
+/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
+fn check_version_and_update_task_queue(
+    db_path: &Path,
+    experimental_dumpless_upgrade: bool,
+) -> anyhow::Result<()> {
+    let (major, minor, patch) = get_version(db_path)?;
+
+    if major != VERSION_MAJOR || minor != VERSION_MINOR {
+        if experimental_dumpless_upgrade {
+            let version = (
+                major.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
+                minor.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
+                patch.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
+            );
+            return upgrade_task_queue(&db_path.join("tasks"), version);
+        } else {
+            return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
+        }
+    }
+
+    Ok(())
+}
+
 /// Ensure you're in a valid state and open the IndexScheduler + AuthController for you.
 fn open_or_create_database(
     opt: &Opt,
     empty_db: bool,
 ) -> anyhow::Result<(IndexScheduler, AuthController)> {
     if !empty_db {
-        check_version_file(&opt.db_path)?;
+        check_version_and_update_task_queue(&opt.db_path, opt.experimental_dumpless_upgrade)?;
     }
 
     open_or_create_database_unchecked(opt, OnFailure::KeepDb)

crates/meilisearch/src/option.rs

@@ -49,6 +49,7 @@ const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
 const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR";
 const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
 const MEILI_EXPERIMENTAL_LOGS_MODE: &str = "MEILI_EXPERIMENTAL_LOGS_MODE";
+const MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE: &str = "MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE";
 const MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS: &str = "MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS";
 const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE";
 const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER";
@@ -400,6 +401,13 @@ pub struct Opt {
     #[serde(default)]
     pub experimental_logs_mode: LogMode,
 
+    /// Experimental dumpless upgrade. For more information, see: <https://github.com/orgs/meilisearch/discussions/723>
+    ///
+    /// When set, Meilisearch will auto-update its database without using a dump.
+    #[clap(long, env = MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE, default_value_t)]
+    #[serde(default)]
+    pub experimental_dumpless_upgrade: bool,
+
     /// Experimental logs route feature. For more information,
     /// see: <https://github.com/orgs/meilisearch/discussions/721>
     ///
@@ -535,6 +543,7 @@ impl Opt {
             experimental_drop_search_after,
             experimental_nb_searches_per_core,
             experimental_logs_mode,
+            experimental_dumpless_upgrade,
             experimental_enable_logs_route,
             experimental_replication_parameters,
             experimental_reduce_indexing_memory_usage,
@@ -608,6 +617,10 @@ impl Opt {
             MEILI_EXPERIMENTAL_LOGS_MODE,
             experimental_logs_mode.to_string(),
         );
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE,
+            experimental_dumpless_upgrade.to_string(),
+        );
         export_to_env_if_not_present(
             MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS,
             experimental_replication_parameters.to_string(),

crates/meilisearch/src/routes/tasks.rs

@@ -912,14 +912,14 @@ mod tests {
         {
             let params = "types=createIndex";
             let err = deserr_query_params::<TaskDeletionOrCancelationQuery>(params).unwrap_err();
-            snapshot!(meili_snap::json_string!(err), @r###"
+            snapshot!(meili_snap::json_string!(err), @r#"
             {
-              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
               "code": "invalid_task_types",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_task_types"
             }
-            "###);
+            "#);
         }
     }
     #[test]

crates/meilisearch/src/upgrade/mod.rs

@@ -0,0 +1 @@
+

crates/meilisearch/tests/batches/errors.rs

@@ -42,7 +42,7 @@ async fn batch_bad_types() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r#"
     {
-      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
       "code": "invalid_task_types",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_task_types"

crates/meilisearch/tests/tasks/errors.rs

@@ -95,36 +95,36 @@ async fn task_bad_types() {
 
     let (response, code) = server.tasks_filter("types=doggo").await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(json_string!(response), @r#"
     {
-      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
       "code": "invalid_task_types",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_task_types"
     }
-    "###);
+    "#);
 
     let (response, code) = server.cancel_tasks("types=doggo").await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(json_string!(response), @r#"
     {
-      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
       "code": "invalid_task_types",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_task_types"
     }
-    "###);
+    "#);
 
     let (response, code) = server.delete_tasks("types=doggo").await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(json_string!(response), @r#"
     {
-      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+      "message": "Invalid value in parameter `types`: `doggo` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
       "code": "invalid_task_types",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_task_types"
     }
-    "###);
+    "#);
 }
 
 #[actix_rt::test]