Merge branch 'meilisearch:main' into fix-3037

config.toml

@@ -56,7 +56,7 @@ disable_auto_batching = false
 ### DUMPS ###
 #############
 
-dumps_dir = "dumps/"
+dump_dir = "dumps/"
 # Sets the directory where Meilisearch will create dump files.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#dumps-destination
 

dump/src/reader/compat/v1_to_v2.rs

@@ -1,8 +1,8 @@
-use std::{collections::BTreeSet, str::FromStr};
+use std::collections::BTreeSet;
-
+use std::str::FromStr;
-use crate::reader::{v1, v2, Document};
 
 use super::v2_to_v3::CompatV2ToV3;
+use crate::reader::{v1, v2, Document};
 use crate::Result;
 
 pub struct CompatV1ToV2 {
@@ -367,12 +367,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
         }
         "###);
 
@@ -382,12 +382,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
         }
         "###);
 
@@ -397,12 +397,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
         }
         "###);
 

dump/src/reader/mod.rs

@@ -299,12 +299,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:39.360187055Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:40.603035979Z"
         }
         "###);
 
@@ -314,12 +314,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:38.710611568Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:49.785862546Z"
         }
         "###);
 
@@ -329,12 +329,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:40.831649057Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:41.116036186Z"
         }
         "###);
 
@@ -562,12 +562,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
         }
         "###);
 
@@ -577,12 +577,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
         }
         "###);
 
@@ -592,12 +592,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
         }
         "###);
 

dump/src/reader/v1/mod.rs

@@ -1,15 +1,14 @@
-use std::{
+use std::fs::{self, File};
-    fs::{self, File},
+use std::io::{BufRead, BufReader};
-    io::{BufRead, BufReader},
+use std::path::{Path, PathBuf};
-    path::{Path, PathBuf},
-};
 
+use serde::Deserialize;
 use tempfile::TempDir;
 use time::OffsetDateTime;
 
-use super::{compat::v1_to_v2::CompatV1ToV2, Document};
+use super::compat::v1_to_v2::CompatV1ToV2;
+use super::Document;
 use crate::{IndexMetadata, Result, Version};
-use serde::Deserialize;
 
 pub mod settings;
 pub mod update;
@@ -204,12 +203,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
         }
         "###);
 
@@ -223,12 +222,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"91de507f206ad21964584021932ba7a7");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
         }
         "###);
 
@@ -242,12 +241,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"55eef4de2bef7e84c5ce0bee47488f56");
 
         // spells
-        insta::assert_json_snapshot!(dnd_spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(dnd_spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
         }
         "###);
 

dump/src/reader/v4/mod.rs

@@ -13,7 +13,7 @@ pub mod meta;
 pub mod settings;
 pub mod tasks;
 
-use self::meta::{DumpMeta, IndexUuid};
+use self::meta::{DumpMeta, IndexMeta, IndexUuid};
 use super::compat::v4_to_v5::CompatV4ToV5;
 use crate::{Error, IndexMetadata, Result, Version};
 
@@ -100,6 +100,10 @@ impl V4Reader {
             V4IndexReader::new(
                 index.uid.clone(),
                 &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()),
+                &index.index_meta,
+                BufReader::new(
+                    File::open(&self.dump.path().join("updates").join("data.jsonl")).unwrap(),
+                ),
             )
         }))
     }
@@ -147,16 +151,44 @@ pub struct V4IndexReader {
 }
 
 impl V4IndexReader {
-    pub fn new(name: String, path: &Path) -> Result<Self> {
+    pub fn new(
+        name: String,
+        path: &Path,
+        index_metadata: &IndexMeta,
+        tasks: BufReader<File>,
+    ) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
+        let mut created_at = None;
+        let mut updated_at = None;
+
+        for line in tasks.lines() {
+            let task: Task = serde_json::from_str(&line?)?;
+
+            if task.index_uid.to_string() == name {
+                // The first task to match our index_uid that succeeded (ie. processed_at returns Some)
+                // is our `last_updated_at`.
+                if updated_at.is_none() {
+                    updated_at = task.processed_at()
+                }
+
+                // Once we reach the `creation_task_id` we can stop iterating on the task queue and
+                // this task represents our `created_at`.
+                if task.id as usize == index_metadata.creation_task_id {
+                    created_at = task.created_at();
+                    break;
+                }
+            }
+        }
+
+        let current_time = OffsetDateTime::now_utc();
+
         let metadata = IndexMetadata {
             uid: name,
             primary_key: meta.primary_key,
-            // FIXME: Iterate over the whole task queue to find the creation and last update date.
+            created_at: created_at.unwrap_or(current_time),
-            created_at: OffsetDateTime::now_utc(),
+            updated_at: updated_at.unwrap_or(current_time),
-            updated_at: OffsetDateTime::now_utc(),
         };
 
         let ret = V4IndexReader {
@@ -259,12 +291,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:39.360187055Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:40.603035979Z"
         }
         "###);
 
@@ -274,12 +306,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:38.710611568Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:49.785862546Z"
         }
         "###);
 
@@ -289,12 +321,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
+          "createdAt": "2022-10-06T12:53:40.831649057Z",
-          "updatedAt": "[now]"
+          "updatedAt": "2022-10-06T12:53:41.116036186Z"
         }
         "###);
 

dump/src/reader/v4/tasks.rs

@@ -104,6 +104,41 @@ impl Task {
         })
     }
 
+    pub fn processed_at(&self) -> Option<OffsetDateTime> {
+        match self.events.last() {
+            Some(TaskEvent::Succeded { result: _, timestamp }) => Some(*timestamp),
+            _ => None,
+        }
+    }
+
+    pub fn created_at(&self) -> Option<OffsetDateTime> {
+        match &self.content {
+            TaskContent::IndexCreation { primary_key: _ } => match self.events.first() {
+                Some(TaskEvent::Created(ts)) => Some(*ts),
+                _ => None,
+            },
+            TaskContent::DocumentAddition {
+                content_uuid: _,
+                merge_strategy: _,
+                primary_key: _,
+                documents_count: _,
+                allow_index_creation: _,
+            } => match self.events.first() {
+                Some(TaskEvent::Created(ts)) => Some(*ts),
+                _ => None,
+            },
+            TaskContent::SettingsUpdate {
+                settings: _,
+                is_deletion: _,
+                allow_index_creation: _,
+            } => match self.events.first() {
+                Some(TaskEvent::Created(ts)) => Some(*ts),
+                _ => None,
+            },
+            _ => None,
+        }
+    }
+
     /// Return the content_uuid of the `Task` if there is one.
     pub fn get_content_uuid(&self) -> Option<Uuid> {
         match self {

meilisearch-http/Cargo.toml

@@ -119,5 +119,5 @@ japanese = ["meilisearch-types/japanese"]
 thai = ["meilisearch-types/thai"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.3/build.zip"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.4/build.zip"
-sha1 = "fb893012023cc33090c549e0eaf10adff335cf6f"
+sha1 = "b53c2edb51d4ce1984d5586333b91c4ad3a1b4e4"

meilisearch-http/src/analytics/segment_analytics.rs

@@ -216,7 +216,7 @@ struct Infos {
     env: String,
     db_path: bool,
     import_dump: bool,
-    dumps_dir: bool,
+    dump_dir: bool,
     ignore_missing_dump: bool,
     ignore_dump_if_db_exists: bool,
     import_snapshot: bool,
@@ -272,7 +272,7 @@ impl From<Opt> for Infos {
             import_dump,
             ignore_missing_dump,
             ignore_dump_if_db_exists,
-            dumps_dir,
+            dump_dir,
             log_level,
             indexer_options,
             scheduler_options,
@@ -295,7 +295,7 @@ impl From<Opt> for Infos {
             env,
             db_path: db_path != PathBuf::from("./data.ms"),
             import_dump: import_dump.is_some(),
-            dumps_dir: dumps_dir != PathBuf::from("dumps/"),
+            dump_dir: dump_dir != PathBuf::from("dumps/"),
             ignore_missing_dump,
             ignore_dump_if_db_exists,
             import_snapshot: import_snapshot.is_some(),

meilisearch-http/src/lib.rs

@@ -121,7 +121,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Auth
             update_file_path: opt.db_path.join("update_files"),
             indexes_path: opt.db_path.join("indexes"),
             snapshots_path: opt.snapshot_dir.clone(),
-            dumps_path: opt.dumps_dir.clone(),
+            dumps_path: opt.dump_dir.clone(),
             task_db_size: opt.max_task_db_size.get_bytes() as usize,
             index_size: opt.max_index_size.get_bytes() as usize,
             indexer_config: (&opt.indexer_options).try_into()?,

meilisearch-http/src/option.rs

@@ -47,7 +47,7 @@ const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC";
 const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP";
 const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
 const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
-const MEILI_DUMPS_DIR: &str = "MEILI_DUMPS_DIR";
+const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR";
 const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
 #[cfg(feature = "metrics")]
 const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE";
@@ -61,7 +61,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB";
 const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
 const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
 const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
-const DEFAULT_DUMPS_DIR: &str = "dumps/";
+const DEFAULT_DUMP_DIR: &str = "dumps/";
 const DEFAULT_LOG_LEVEL: &str = "INFO";
 
 const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
@@ -219,9 +219,9 @@ pub struct Opt {
     pub ignore_dump_if_db_exists: bool,
 
     /// Sets the directory where Meilisearch will create dump files.
-    #[clap(long, env = MEILI_DUMPS_DIR, default_value_os_t = default_dumps_dir())]
+    #[clap(long, env = MEILI_DUMP_DIR, default_value_os_t = default_dump_dir())]
-    #[serde(default = "default_dumps_dir")]
+    #[serde(default = "default_dump_dir")]
-    pub dumps_dir: PathBuf,
+    pub dump_dir: PathBuf,
 
     /// Defines how much detail should be present in Meilisearch's logs.
     ///
@@ -320,7 +320,7 @@ impl Opt {
             snapshot_dir,
             schedule_snapshot,
             snapshot_interval_sec,
-            dumps_dir,
+            dump_dir,
             log_level,
             indexer_options,
             scheduler_options,
@@ -373,7 +373,7 @@ impl Opt {
             MEILI_SNAPSHOT_INTERVAL_SEC,
             snapshot_interval_sec.to_string(),
         );
-        export_to_env_if_not_present(MEILI_DUMPS_DIR, dumps_dir);
+        export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
         export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level);
         #[cfg(feature = "metrics")]
         {
@@ -708,8 +708,8 @@ fn default_snapshot_interval_sec() -> u64 {
     DEFAULT_SNAPSHOT_INTERVAL_SEC
 }
 
-fn default_dumps_dir() -> PathBuf {
+fn default_dump_dir() -> PathBuf {
-    PathBuf::from(DEFAULT_DUMPS_DIR)
+    PathBuf::from(DEFAULT_DUMP_DIR)
 }
 
 fn default_log_level() -> String {

meilisearch-http/tests/common/server.rs

@@ -184,7 +184,7 @@ impl Server {
 pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
     Opt {
         db_path: dir.as_ref().join("db"),
-        dumps_dir: dir.as_ref().join("dump"),
+        dump_dir: dir.as_ref().join("dumps"),
         env: "development".to_owned(),
         #[cfg(all(not(debug_assertions), feature = "analytics"))]
         no_analytics: true,