From ab3056cc662668ab8639aa3ae560781fdde926d5 Mon Sep 17 00:00:00 2001 From: funilrys Date: Mon, 31 Oct 2022 18:57:40 +0100 Subject: [PATCH 01/22] Extract the dates out of the dumpv4. This patch possibly fixes #2987. This patch introduces a way to fill the IndexMetadata.created_at and IndexMetadata.updated_at keys from the tasks events. This is done by reading the creation date of the first event (created_at) and the creation date of the last event (updated_at). --- dump/src/reader/v4/mod.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 3aad71ddb..c335d4289 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -23,6 +23,7 @@ pub type Checked = settings::Checked; pub type Unchecked = settings::Unchecked; pub type Task = tasks::Task; +pub type TaskEvent = tasks::TaskEvent; pub type Key = keys::Key; // everything related to the settings @@ -100,6 +101,7 @@ impl V4Reader { V4IndexReader::new( index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), + BufReader::new(self.tasks.get_ref().try_clone().unwrap()), ) })) } @@ -147,16 +149,31 @@ pub struct V4IndexReader { } impl V4IndexReader { - pub fn new(name: String, path: &Path) -> Result { + pub fn new(name: String, path: &Path, tasks: BufReader) -> Result { let meta = File::open(path.join("meta.json"))?; let meta: DumpMeta = serde_json::from_reader(meta)?; + let mut index_tasks: Vec = vec![]; + + for line in tasks.lines() { + let task: Task = serde_json::from_str(&line?)?; + + if task.index_uid.to_string() == name { + index_tasks.push(task) + } + } + let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - // FIXME: Iterate over the whole task queue to find the creation and last update date. - created_at: OffsetDateTime::now_utc(), - updated_at: OffsetDateTime::now_utc(), + created_at: match index_tasks.first().unwrap().events.first() { + Some(TaskEvent::Created(ts)) => *ts, + _ => OffsetDateTime::now_utc(), + }, + updated_at: match index_tasks.last().unwrap().events.last() { + Some(TaskEvent::Created(ts)) => *ts, + _ => OffsetDateTime::now_utc(), + }, }; let ret = V4IndexReader { From 953b2ec4380736136c015ccbf4d508acdf7fc21a Mon Sep 17 00:00:00 2001 From: funilrys Date: Wed, 2 Nov 2022 17:49:37 +0100 Subject: [PATCH 02/22] fixup! Extract the dates out of the dumpv4. --- dump/src/reader/v4/mod.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index c335d4289..5cf7b112d 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -24,6 +24,7 @@ pub type Unchecked = settings::Unchecked; pub type Task = tasks::Task; pub type TaskEvent = tasks::TaskEvent; +pub type TaskContent = tasks::TaskContent; pub type Key = keys::Key; // everything related to the settings @@ -153,27 +154,41 @@ impl V4IndexReader { let meta = File::open(path.join("meta.json"))?; let meta: DumpMeta = serde_json::from_reader(meta)?; - let mut index_tasks: Vec = vec![]; + let mut created_at = None; + let mut updated_at = None; for line in tasks.lines() { let task: Task = serde_json::from_str(&line?)?; if task.index_uid.to_string() == name { - index_tasks.push(task) + if updated_at.is_none() { + updated_at = match task.events.last() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }; + } + + if created_at.is_none() { + created_at = match task.content { + TaskContent::IndexCreation { primary_key } => match task.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + _ => None, + }; + } + + if created_at.is_some() { + break; + } } } let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - created_at: match index_tasks.first().unwrap().events.first() { - Some(TaskEvent::Created(ts)) => *ts, - _ => OffsetDateTime::now_utc(), - }, - updated_at: match index_tasks.last().unwrap().events.last() { - Some(TaskEvent::Created(ts)) => *ts, - _ => OffsetDateTime::now_utc(), - }, + created_at: created_at.unwrap_or(OffsetDateTime::now_utc()), + updated_at: updated_at.unwrap_or(OffsetDateTime::now_utc()), }; let ret = V4IndexReader { From 06e7db7a1ff08dc25593d58086993001a4f4e2e3 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sat, 12 Nov 2022 18:28:23 +0100 Subject: [PATCH 03/22] fixup! Extract the dates out of the dumpv4. --- dump/src/reader/v4/mod.rs | 27 +++++++++++---------------- dump/src/reader/v4/tasks.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 5cf7b112d..140890d78 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -13,7 +13,7 @@ pub mod meta; pub mod settings; pub mod tasks; -use self::meta::{DumpMeta, IndexUuid}; +use self::meta::{DumpMeta, IndexMeta, IndexUuid}; use super::compat::v4_to_v5::CompatV4ToV5; use crate::{Error, IndexMetadata, Result, Version}; @@ -23,8 +23,6 @@ pub type Checked = settings::Checked; pub type Unchecked = settings::Unchecked; pub type Task = tasks::Task; -pub type TaskEvent = tasks::TaskEvent; -pub type TaskContent = tasks::TaskContent; pub type Key = keys::Key; // everything related to the settings @@ -102,6 +100,7 @@ impl V4Reader { V4IndexReader::new( index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), + &index.index_meta, BufReader::new(self.tasks.get_ref().try_clone().unwrap()), ) })) @@ -150,7 +149,12 @@ pub struct V4IndexReader { } impl V4IndexReader { - pub fn new(name: String, path: &Path, tasks: BufReader) -> Result { + pub fn new( + name: String, + path: &Path, + index_metadata: &IndexMeta, + tasks: BufReader, + ) -> Result { let meta = File::open(path.join("meta.json"))?; let meta: DumpMeta = serde_json::from_reader(meta)?; @@ -162,23 +166,14 @@ impl V4IndexReader { if task.index_uid.to_string() == name { if updated_at.is_none() { - updated_at = match task.events.last() { - Some(TaskEvent::Created(ts)) => Some(*ts), - _ => None, - }; + updated_at = task.updated_at() } if created_at.is_none() { - created_at = match task.content { - TaskContent::IndexCreation { primary_key } => match task.events.first() { - Some(TaskEvent::Created(ts)) => Some(*ts), - _ => None, - }, - _ => None, - }; + created_at = task.created_at() } - if created_at.is_some() { + if task.id as usize == index_metadata.creation_task_id { break; } } diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index e1bdde0c7..c075fecc7 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -104,6 +104,33 @@ impl Task { }) } + pub fn updated_at(&self) -> Option { + match self.events.last() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + } + } + + pub fn created_at(&self) -> Option { + match &self.content { + TaskContent::IndexCreation { primary_key: _ } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + TaskContent::DocumentAddition { + content_uuid: _, + merge_strategy: _, + primary_key: _, + documents_count: _, + allow_index_creation: _, + } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + _ => None, + } + } + /// Return the content_uuid of the `Task` if there is one. pub fn get_content_uuid(&self) -> Option { match self { From 079357ee1f2a90e5b46490bf1db2d960ea3223b5 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sat, 12 Nov 2022 20:57:27 +0100 Subject: [PATCH 04/22] Fix linting issues. --- dump/src/reader/v4/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 140890d78..34df609b1 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -182,8 +182,8 @@ impl V4IndexReader { let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - created_at: created_at.unwrap_or(OffsetDateTime::now_utc()), - updated_at: updated_at.unwrap_or(OffsetDateTime::now_utc()), + created_at: created_at.unwrap_or_else(OffsetDateTime::now_utc), + updated_at: updated_at.unwrap_or_else(OffsetDateTime::now_utc), }; let ret = V4IndexReader { From 8a14f6f5455c6c0bfed015a67944b5e26002ffc1 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sun, 13 Nov 2022 10:12:51 +0100 Subject: [PATCH 05/22] Add Task.processed_at. --- dump/src/reader/v4/mod.rs | 2 ++ dump/src/reader/v4/tasks.rs | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 34df609b1..885f59d97 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -174,6 +174,8 @@ impl V4IndexReader { } if task.id as usize == index_metadata.creation_task_id { + created_at = task.processed_at(); + break; } } diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index c075fecc7..be98f9ee2 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -104,6 +104,13 @@ impl Task { }) } + pub fn processed_at(&self) -> Option { + match self.events.last() { + Some(TaskEvent::Succeded { result: _, timestamp }) => Some(timestamp.clone()), + _ => None, + } + } + pub fn updated_at(&self) -> Option { match self.events.last() { Some(TaskEvent::Created(ts)) => Some(*ts), From 0a102d601c774805f9d68733b233647a09c56fa8 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sun, 13 Nov 2022 10:14:20 +0100 Subject: [PATCH 06/22] Update Task.created_at Indeed, before this patch we weren't considering the TaskContent::SetingsUpdate while trying to find the creation date. --- dump/src/reader/v4/tasks.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index be98f9ee2..e563274d9 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -134,6 +134,14 @@ impl Task { Some(TaskEvent::Created(ts)) => Some(*ts), _ => None, }, + TaskContent::SettingsUpdate { + settings: _, + is_deletion: _, + allow_index_creation: _, + } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, _ => None, } } From e81b349658031439be0a479b70af1e3b5a170d61 Mon Sep 17 00:00:00 2001 From: funilrys Date: Mon, 14 Nov 2022 18:51:34 +0100 Subject: [PATCH 07/22] Fix linting issue. --- dump/src/reader/v4/tasks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index e563274d9..c9920ffd6 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -106,7 +106,7 @@ impl Task { pub fn processed_at(&self) -> Option { match self.events.last() { - Some(TaskEvent::Succeded { result: _, timestamp }) => Some(timestamp.clone()), + Some(TaskEvent::Succeded { result: _, timestamp }) => Some(*timestamp), _ => None, } } From e35db5e59bd2a3ef0524d0e6e9dc7dde09043e4c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 15:39:36 +0100 Subject: [PATCH 08/22] Rename `dumps-dir` to `dump-dir` in CLI Also rename the associated environment variable --- meilisearch-http/src/lib.rs | 2 +- meilisearch-http/src/option.rs | 18 +++++++++--------- meilisearch-http/tests/common/server.rs | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 6fa6b77d8..b11f063d2 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -121,7 +121,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Auth update_file_path: opt.db_path.join("update_files"), indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), - dumps_path: opt.dumps_dir.clone(), + dumps_path: opt.dump_dir.clone(), task_db_size: opt.max_task_db_size.get_bytes() as usize, index_size: opt.max_index_size.get_bytes() as usize, indexer_config: (&opt.indexer_options).try_into()?, diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 82d67d5a0..0d4ce133a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -47,7 +47,7 @@ const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC"; const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP"; const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; -const MEILI_DUMPS_DIR: &str = "MEILI_DUMPS_DIR"; +const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; #[cfg(feature = "metrics")] const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; @@ -61,7 +61,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB"; const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; -const DEFAULT_DUMPS_DIR: &str = "dumps/"; +const DEFAULT_DUMP_DIR: &str = "dumps/"; const DEFAULT_LOG_LEVEL: &str = "INFO"; const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; @@ -219,9 +219,9 @@ pub struct Opt { pub ignore_dump_if_db_exists: bool, /// Sets the directory where Meilisearch will create dump files. - #[clap(long, env = MEILI_DUMPS_DIR, default_value_os_t = default_dumps_dir())] - #[serde(default = "default_dumps_dir")] - pub dumps_dir: PathBuf, + #[clap(long, env = MEILI_DUMP_DIR, default_value_os_t = default_dump_dir())] + #[serde(default = "default_dump_dir")] + pub dump_dir: PathBuf, /// Defines how much detail should be present in Meilisearch's logs. /// @@ -320,7 +320,7 @@ impl Opt { snapshot_dir, schedule_snapshot, snapshot_interval_sec, - dumps_dir, + dump_dir, log_level, indexer_options, scheduler_options, @@ -373,7 +373,7 @@ impl Opt { MEILI_SNAPSHOT_INTERVAL_SEC, snapshot_interval_sec.to_string(), ); - export_to_env_if_not_present(MEILI_DUMPS_DIR, dumps_dir); + export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level); #[cfg(feature = "metrics")] { @@ -708,8 +708,8 @@ fn default_snapshot_interval_sec() -> u64 { DEFAULT_SNAPSHOT_INTERVAL_SEC } -fn default_dumps_dir() -> PathBuf { - PathBuf::from(DEFAULT_DUMPS_DIR) +fn default_dump_dir() -> PathBuf { + PathBuf::from(DEFAULT_DUMP_DIR) } fn default_log_level() -> String { diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 3f72248c5..e6f5e95ab 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -184,7 +184,7 @@ impl Server { pub fn default_settings(dir: impl AsRef) -> Opt { Opt { db_path: dir.as_ref().join("db"), - dumps_dir: dir.as_ref().join("dump"), + dump_dir: dir.as_ref().join("dump"), env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, From daa0f2e9aa0c574eff3f061110fcfddcee43152d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 15:40:44 +0100 Subject: [PATCH 09/22] Change dumps_dir to dump_dir in config.toml --- config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.toml b/config.toml index 8933dd22f..6ef9b77f1 100644 --- a/config.toml +++ b/config.toml @@ -56,7 +56,7 @@ disable_auto_batching = false ### DUMPS ### ############# -dumps_dir = "dumps/" +dump_dir = "dumps/" # Sets the directory where Meilisearch will create dump files. # https://docs.meilisearch.com/learn/configuration/instance_options.html#dumps-destination From a92d67b79ffde317766b822a902e2e0bd2b6cb3d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 16:24:02 +0100 Subject: [PATCH 10/22] Also update analytics --- meilisearch-http/src/analytics/segment_analytics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index afec4c5cb..a8fe9fead 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -216,7 +216,7 @@ struct Infos { env: String, db_path: bool, import_dump: bool, - dumps_dir: bool, + dump_dir: bool, ignore_missing_dump: bool, ignore_dump_if_db_exists: bool, import_snapshot: bool, @@ -272,7 +272,7 @@ impl From for Infos { import_dump, ignore_missing_dump, ignore_dump_if_db_exists, - dumps_dir, + dump_dir, log_level, indexer_options, scheduler_options, @@ -295,7 +295,7 @@ impl From for Infos { env, db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), - dumps_dir: dumps_dir != PathBuf::from("dumps/"), + dump_dir: dump_dir != PathBuf::from("dumps/"), ignore_missing_dump, ignore_dump_if_db_exists, import_snapshot: import_snapshot.is_some(), From f056fc118fcdba2f7270f5b8517f50664bc601ca Mon Sep 17 00:00:00 2001 From: funilrys Date: Sat, 3 Dec 2022 17:29:41 +0100 Subject: [PATCH 11/22] Re-open tasks queue. Indeed, before this patch, I was (probably) breaking every usage of the tasks BufReader. This patch solves the issue by reopening the the tasks file every time its needed. --- dump/src/reader/v4/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 885f59d97..db2cec4e5 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -101,7 +101,7 @@ impl V4Reader { index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), &index.index_meta, - BufReader::new(self.tasks.get_ref().try_clone().unwrap()), + BufReader::new(File::open(dump.path().join("updates").join("data.jsonl")).unwrap()), ) })) } From e510ace1794495a724626a382832908bd08c0964 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sat, 3 Dec 2022 17:41:33 +0100 Subject: [PATCH 12/22] fixup! Re-open tasks queue. --- dump/src/reader/v4/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index db2cec4e5..04ad346b7 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -101,7 +101,7 @@ impl V4Reader { index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), &index.index_meta, - BufReader::new(File::open(dump.path().join("updates").join("data.jsonl")).unwrap()), + BufReader::new(File::open(&self.dump.path().join("updates").join("data.jsonl")).unwrap()), ) })) } From 8b6eba4f0b678679a8ea3901790c7246de9ca844 Mon Sep 17 00:00:00 2001 From: funilrys Date: Sat, 3 Dec 2022 17:47:02 +0100 Subject: [PATCH 13/22] Apply fmt. --- dump/src/reader/v4/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 04ad346b7..2cef6c86d 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -101,7 +101,9 @@ impl V4Reader { index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), &index.index_meta, - BufReader::new(File::open(&self.dump.path().join("updates").join("data.jsonl")).unwrap()), + BufReader::new( + File::open(&self.dump.path().join("updates").join("data.jsonl")).unwrap(), + ), ) })) } From 293efb74854f44fc4cd60b9bf41b626705e86e01 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Mon, 5 Dec 2022 14:54:01 +0100 Subject: [PATCH 14/22] Update Cargo.toml --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6a4cc0336..1d451f9d0 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -119,5 +119,5 @@ japanese = ["meilisearch-types/japanese"] thai = ["meilisearch-types/thai"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.3/build.zip" -sha1 = "fb893012023cc33090c549e0eaf10adff335cf6f" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.4/build.zip" +sha1 = "b53c2edb51d4ce1984d5586333b91c4ad3a1b4e4" From 492fd2829a9127c020b2e983a79bc7a18127f118 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 5 Dec 2022 16:56:28 +0100 Subject: [PATCH 15/22] use a consistent dump directory name in tests changed from 'dump' to 'dumps' to be consistent with the default settings Co-authored-by: Tamo --- meilisearch-http/tests/common/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index e6f5e95ab..f96c659a4 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -184,7 +184,7 @@ impl Server { pub fn default_settings(dir: impl AsRef) -> Opt { Opt { db_path: dir.as_ref().join("db"), - dump_dir: dir.as_ref().join("dump"), + dump_dir: dir.as_ref().join("dumps"), env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, From 688911ed3464708617f4b36d5fc2b0002fce7635 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Dec 2022 17:05:37 +0100 Subject: [PATCH 16/22] re-enable the dump test on the dates --- dump/src/reader/compat/v1_to_v2.rs | 24 +++++++++++------------ dump/src/reader/mod.rs | 18 ++++++++--------- dump/src/reader/v1/mod.rs | 31 +++++++++++++++--------------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/dump/src/reader/compat/v1_to_v2.rs b/dump/src/reader/compat/v1_to_v2.rs index 23e4529dc..741d18fa8 100644 --- a/dump/src/reader/compat/v1_to_v2.rs +++ b/dump/src/reader/compat/v1_to_v2.rs @@ -1,8 +1,8 @@ -use std::{collections::BTreeSet, str::FromStr}; - -use crate::reader::{v1, v2, Document}; +use std::collections::BTreeSet; +use std::str::FromStr; use super::v2_to_v3::CompatV2ToV3; +use crate::reader::{v1, v2, Document}; use crate::Result; pub struct CompatV1ToV2 { @@ -367,12 +367,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -382,12 +382,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -397,12 +397,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index efbca06d0..5e493d77e 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -562,12 +562,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -577,12 +577,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -592,12 +592,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); diff --git a/dump/src/reader/v1/mod.rs b/dump/src/reader/v1/mod.rs index 1932b602a..ac7324d9a 100644 --- a/dump/src/reader/v1/mod.rs +++ b/dump/src/reader/v1/mod.rs @@ -1,15 +1,14 @@ -use std::{ - fs::{self, File}, - io::{BufRead, BufReader}, - path::{Path, PathBuf}, -}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use serde::Deserialize; use tempfile::TempDir; use time::OffsetDateTime; -use super::{compat::v1_to_v2::CompatV1ToV2, Document}; +use super::compat::v1_to_v2::CompatV1ToV2; +use super::Document; use crate::{IndexMetadata, Result, Version}; -use serde::Deserialize; pub mod settings; pub mod update; @@ -204,12 +203,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -223,12 +222,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"91de507f206ad21964584021932ba7a7"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -242,12 +241,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"55eef4de2bef7e84c5ce0bee47488f56"); // spells - insta::assert_json_snapshot!(dnd_spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(dnd_spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); From afe520a67eb326f87568e4562439d186979f22f0 Mon Sep 17 00:00:00 2001 From: Ivan Ionut Date: Mon, 5 Dec 2022 17:49:15 +0100 Subject: [PATCH 17/22] Upgrade alpine 3.16 to 3.17 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e54fcdae..aaf64cd85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Compile -FROM rust:alpine3.16 AS compiler +FROM rust:alpine3.17 AS compiler RUN apk add -q --update-cache --no-cache build-base openssl-dev @@ -19,7 +19,7 @@ RUN set -eux; \ cargo build --release # Run -FROM alpine:3.16 +FROM alpine:3.17 ENV MEILI_HTTP_ADDR 0.0.0.0:7700 ENV MEILI_SERVER_PROVIDER docker From b0cf431614a0d98254a1c0adce6218b6f6d58ad1 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Dec 2022 18:08:35 +0100 Subject: [PATCH 18/22] Fix the dump date-import of the dumpv4 --- dump/src/reader/v4/mod.rs | 31 +++++++++++++++---------------- dump/src/reader/v4/tasks.rs | 7 ------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 38a81fd05..104896353 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -167,17 +167,16 @@ impl V4IndexReader { let task: Task = serde_json::from_str(&line?)?; if task.index_uid.to_string() == name { + // The first task to match our index_uid that succeeded (ie. processed_at returns Some) + // is our `last_updated_at`. if updated_at.is_none() { - updated_at = task.updated_at() - } - - if created_at.is_none() { - created_at = task.created_at() + updated_at = task.processed_at() } + // Once we reach the `creation_task_id` we can stop iterating on the task queue and + // this task represent our `created_at`. if task.id as usize == index_metadata.creation_task_id { - created_at = task.processed_at(); - + created_at = task.created_at(); break; } } @@ -290,12 +289,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -305,12 +304,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -320,12 +319,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index c9920ffd6..a701d837d 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -111,13 +111,6 @@ impl Task { } } - pub fn updated_at(&self) -> Option { - match self.events.last() { - Some(TaskEvent::Created(ts)) => Some(*ts), - _ => None, - } - } - pub fn created_at(&self) -> Option { match &self.content { TaskContent::IndexCreation { primary_key: _ } => match self.events.first() { From 9c89e3dadccf366c3c72e0fce7c16edc19f4a768 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Dec 2022 18:15:29 +0100 Subject: [PATCH 19/22] uncomment more test for the dump v4 --- dump/src/reader/mod.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index efbca06d0..2b8d77991 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -299,12 +299,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -314,12 +314,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -329,12 +329,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); From 180511795b3603f6a169bfa36c7530c069393503 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 6 Dec 2022 10:53:43 +0100 Subject: [PATCH 20/22] Update dump/src/reader/v4/mod.rs fix typo Co-authored-by: Louis Dureuil --- dump/src/reader/v4/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 104896353..6f10477ad 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -174,7 +174,7 @@ impl V4IndexReader { } // Once we reach the `creation_task_id` we can stop iterating on the task queue and - // this task represent our `created_at`. + // this task represents our `created_at`. if task.id as usize == index_metadata.creation_task_id { created_at = task.created_at(); break; From bef81065f9c2fbd542ccf6579b4a7a830c1adc5a Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 6 Dec 2022 11:03:23 +0100 Subject: [PATCH 21/22] return the same time in case we didn't found a created or updated at --- dump/src/reader/v4/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 6f10477ad..4dfbf3502 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -182,11 +182,13 @@ impl V4IndexReader { } } + let current_time = OffsetDateTime::now_utc(); + let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - created_at: created_at.unwrap_or_else(OffsetDateTime::now_utc), - updated_at: updated_at.unwrap_or_else(OffsetDateTime::now_utc), + created_at: created_at.unwrap_or(current_time), + updated_at: updated_at.unwrap_or(current_time), }; let ret = V4IndexReader { From cbb8d0f97b9e440b12587b78a14224a20010f8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar=20-=20curqui?= Date: Tue, 6 Dec 2022 11:09:57 +0100 Subject: [PATCH 22/22] Revert "Upgrade alpine 3.16 to 3.17" --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index aaf64cd85..0e54fcdae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Compile -FROM rust:alpine3.17 AS compiler +FROM rust:alpine3.16 AS compiler RUN apk add -q --update-cache --no-cache build-base openssl-dev @@ -19,7 +19,7 @@ RUN set -eux; \ cargo build --release # Run -FROM alpine:3.17 +FROM alpine:3.16 ENV MEILI_HTTP_ADDR 0.0.0.0:7700 ENV MEILI_SERVER_PROVIDER docker