From fed59cc1d55ff9932b7b9e0e3898999871453317 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Wed, 18 Oct 2023 13:30:24 +0530 Subject: [PATCH 1/3] extract created_at and updated_at dates from v2 dumps --- dump/src/reader/v2/mod.rs | 41 ++++++++++++++++++++++++++++++----- dump/src/reader/v2/updates.rs | 10 +++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/dump/src/reader/v2/mod.rs b/dump/src/reader/v2/mod.rs index 4016e6341..6f10f77b8 100644 --- a/dump/src/reader/v2/mod.rs +++ b/dump/src/reader/v2/mod.rs @@ -46,6 +46,7 @@ pub type Checked = settings::Checked; pub type Unchecked = settings::Unchecked; pub type Task = updates::UpdateEntry; +pub type Kind = updates::UpdateMeta; // everything related to the errors pub type ResponseError = errors::ResponseError; @@ -107,8 +108,11 @@ impl V2Reader { pub fn indexes(&self) -> Result> + '_> { Ok(self.index_uuid.iter().map(|index| -> Result<_> { V2IndexReader::new( - index.uid.clone(), &self.dump.path().join("indexes").join(format!("index-{}", index.uuid)), + index, + BufReader::new( + File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(), + ), ) })) } @@ -143,16 +147,41 @@ pub struct V2IndexReader { } impl V2IndexReader { - pub fn new(name: String, path: &Path) -> Result { + pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader) -> Result { let meta = File::open(path.join("meta.json"))?; let meta: DumpMeta = serde_json::from_reader(meta)?; + let mut created_at = None; + let mut updated_at = None; + + for line in tasks.lines() { + let task: Task = serde_json::from_str(&line?)?; + if !(task.uuid == index_uuid.uuid && task.is_finished()) { + continue; + } + + let new_created_at = match task.update.meta() { + Kind::DocumentsAddition { .. } | Kind::Settings(_) => task.update.finished_at(), + _ => None, + }; + let new_updated_at = task.update.finished_at(); + + if created_at.is_none() || created_at > new_created_at { + created_at = new_created_at; + } + + if updated_at.is_none() || updated_at < new_updated_at { + updated_at = new_updated_at; + } + } + + let current_time = OffsetDateTime::now_utc(); + let metadata = IndexMetadata { - uid: name, + uid: index_uuid.uid.clone(), primary_key: meta.primary_key, - // FIXME: Iterate over the whole task queue to find the creation and last update date. - created_at: OffsetDateTime::now_utc(), - updated_at: OffsetDateTime::now_utc(), + created_at: created_at.unwrap_or(current_time), + updated_at: updated_at.unwrap_or(current_time), }; let ret = V2IndexReader { diff --git a/dump/src/reader/v2/updates.rs b/dump/src/reader/v2/updates.rs index 33d88d46f..bf1227c7a 100644 --- a/dump/src/reader/v2/updates.rs +++ b/dump/src/reader/v2/updates.rs @@ -227,4 +227,14 @@ impl UpdateStatus { _ => None, } } + + pub fn finished_at(&self) -> Option { + match self { + UpdateStatus::Processing(_) => None, + UpdateStatus::Enqueued(_) => None, + UpdateStatus::Processed(u) => Some(u.processed_at), + UpdateStatus::Aborted(_) => None, + UpdateStatus::Failed(u) => Some(u.failed_at), + } + } } From 62cc97ba70439b0f01eebb8af7c3a24e9cf6aa24 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Wed, 18 Oct 2023 13:31:39 +0530 Subject: [PATCH 2/3] update tests to include created_at and updated-at in v2 dumps --- dump/src/reader/v2/mod.rs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/dump/src/reader/v2/mod.rs b/dump/src/reader/v2/mod.rs index 6f10f77b8..a0ff13a3b 100644 --- a/dump/src/reader/v2/mod.rs +++ b/dump/src/reader/v2/mod.rs @@ -277,12 +277,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:22.688964637Z", + "updatedAt": "2022-10-09T20:27:23.951017769Z" } "###); @@ -292,12 +292,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:22.197788495Z", + "updatedAt": "2022-10-09T20:28:01.93111053Z" } "###); @@ -322,12 +322,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:24.242683494Z", + "updatedAt": "2022-10-09T20:27:24.312809641Z" } "###); @@ -369,12 +369,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:56.595257Z", + "updatedAt": "2023-01-30T16:25:58.70348Z" } "###); @@ -384,12 +384,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:56.192178Z", + "updatedAt": "2023-01-30T16:25:56.455714Z" } "###); @@ -399,12 +399,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:58.876405Z", + "updatedAt": "2023-01-30T16:25:59.079906Z" } "###); From f28f09ae2f36ea9d25103925bdfff4076200a763 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Tue, 24 Oct 2023 14:10:46 +0530 Subject: [PATCH 3/3] update tests for v2 dumps --- dump/src/reader/mod.rs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index af02888d2..603c557d6 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -526,12 +526,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:22.688964637Z", + "updatedAt": "2022-10-09T20:27:23.951017769Z" } "###); @@ -541,12 +541,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:22.197788495Z", + "updatedAt": "2022-10-09T20:28:01.93111053Z" } "###); @@ -571,12 +571,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-09T20:27:24.242683494Z", + "updatedAt": "2022-10-09T20:27:24.312809641Z" } "###); @@ -617,12 +617,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:56.595257Z", + "updatedAt": "2023-01-30T16:25:58.70348Z" } "###); @@ -632,12 +632,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:56.192178Z", + "updatedAt": "2023-01-30T16:25:56.455714Z" } "###); @@ -647,12 +647,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2023-01-30T16:25:58.876405Z", + "updatedAt": "2023-01-30T16:25:59.079906Z" } "###);