From 72a906ae75825492f2213f6944b546f697d0db5e Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 13 Oct 2022 16:03:23 +0200 Subject: [PATCH] fix the tests --- dump/src/lib.rs | 204 +++++++++++++++++++++++++---- dump/src/reader/compat/v5_to_v6.rs | 16 +-- dump/src/reader/mod.rs | 72 +++++----- dump/src/reader/v6/mod.rs | 4 +- dump/src/reader/v6/tasks.rs | 80 ----------- dump/src/writer.rs | 6 +- index-scheduler/src/batch.rs | 2 +- 7 files changed, 226 insertions(+), 158 deletions(-) diff --git a/dump/src/lib.rs b/dump/src/lib.rs index 8451f7495..fd628426e 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -1,3 +1,9 @@ +use meilisearch_types::{ + error::ResponseError, + milli::update::IndexDocumentsMethod, + settings::Unchecked, + tasks::{Details, KindWithContent, Status, Task, TaskId}, +}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -43,6 +49,140 @@ pub enum Version { V6, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TaskDump { + pub uid: TaskId, + #[serde(default)] + pub index_uid: Option, + pub status: Status, + // TODO use our own Kind for the user + #[serde(rename = "type")] + pub kind: KindDump, + + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option
, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + + #[serde(with = "time::serde::rfc3339")] + pub enqueued_at: OffsetDateTime, + #[serde( + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none", + default + )] + pub started_at: Option, + #[serde( + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none", + default + )] + pub finished_at: Option, +} + +// A `Kind` specific version made for the dump. If modified you may break the dump. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum KindDump { + DocumentImport { + primary_key: Option, + method: IndexDocumentsMethod, + documents_count: u64, + allow_index_creation: bool, + }, + DocumentDeletion { + documents_ids: Vec, + }, + DocumentClear, + Settings { + settings: meilisearch_types::settings::Settings, + is_deletion: bool, + allow_index_creation: bool, + }, + IndexDeletion, + IndexCreation { + primary_key: Option, + }, + IndexUpdate { + primary_key: Option, + }, + IndexSwap { + lhs: String, + rhs: String, + }, + CancelTask { + tasks: Vec, + }, + DeleteTasks { + query: String, + tasks: Vec, + }, + DumpExport, + Snapshot, +} + +impl From for TaskDump { + fn from(task: Task) -> Self { + TaskDump { + uid: task.uid, + index_uid: task.index_uid().map(|uid| uid.to_string()), + status: task.status, + kind: task.kind.into(), + details: task.details, + error: task.error, + enqueued_at: task.enqueued_at, + started_at: task.started_at, + finished_at: task.finished_at, + } + } +} + +impl From for KindDump { + fn from(kind: KindWithContent) -> Self { + match kind { + KindWithContent::DocumentImport { + primary_key, + method, + documents_count, + allow_index_creation, + .. + } => KindDump::DocumentImport { + primary_key, + method, + documents_count, + allow_index_creation, + }, + KindWithContent::DocumentDeletion { documents_ids, .. } => { + KindDump::DocumentDeletion { documents_ids } + } + KindWithContent::DocumentClear { .. } => KindDump::DocumentClear, + KindWithContent::Settings { + new_settings, + is_deletion, + allow_index_creation, + .. + } => KindDump::Settings { + settings: new_settings, + is_deletion, + allow_index_creation, + }, + KindWithContent::IndexDeletion { .. } => KindDump::IndexDeletion, + KindWithContent::IndexCreation { primary_key, .. } => { + KindDump::IndexCreation { primary_key } + } + KindWithContent::IndexUpdate { primary_key, .. } => { + KindDump::IndexUpdate { primary_key } + } + KindWithContent::IndexSwap { lhs, rhs } => KindDump::IndexSwap { lhs, rhs }, + KindWithContent::CancelTask { tasks } => KindDump::CancelTask { tasks }, + KindWithContent::DeleteTasks { query, tasks } => KindDump::DeleteTasks { query, tasks }, + KindWithContent::DumpExport { .. } => KindDump::DumpExport, + KindWithContent::Snapshot => KindDump::Snapshot, + } + } +} + #[cfg(test)] pub(crate) mod test { use std::{ @@ -53,18 +193,27 @@ pub(crate) mod test { use big_s::S; use maplit::btreeset; - use meilisearch_types::keys::{Action, Key}; - use meilisearch_types::milli::{self, update::Setting}; - use meilisearch_types::settings::{Checked, Settings}; use meilisearch_types::tasks::{Kind, Status}; use meilisearch_types::{index_uid::IndexUid, star_or::StarOr}; + use meilisearch_types::{ + keys::{Action, Key}, + tasks::Task, + }; + use meilisearch_types::{ + milli::{self, update::Setting}, + tasks::KindWithContent, + }; + use meilisearch_types::{ + settings::{Checked, Settings}, + tasks::Details, + }; use serde_json::{json, Map, Value}; use time::{macros::datetime, Duration}; use uuid::Uuid; use crate::{ reader::{self, Document}, - DumpWriter, IndexMetadata, Version, + DumpWriter, IndexMetadata, KindDump, TaskDump, Version, }; pub fn create_test_instance_uid() -> Uuid { @@ -115,26 +264,24 @@ pub(crate) mod test { settings.check() } - pub fn create_test_tasks() -> Vec<(Task, Option>)> { + pub fn create_test_tasks() -> Vec<(TaskDump, Option>)> { vec![ ( - TaskView { + TaskDump { uid: 0, - index_uid: Some(S("doggos")), + index_uid: Some(S("doggo")), status: Status::Succeeded, - kind: Kind::DocumentImport { + kind: KindDump::DocumentImport { method: milli::update::IndexDocumentsMethod::UpdateDocuments, allow_index_creation: true, + primary_key: Some(S("bone")), + documents_count: 12, }, - details: todo!(), - /* - Some(DetailsView::DocumentAddition { - received_documents: 10_000, - indexed_documents: 3, + details: Some(Details::DocumentAddition { + received_documents: 12, + indexed_documents: Some(10), }), - */ error: None, - duration: Some(Duration::DAY), enqueued_at: datetime!(2022-11-11 0:00 UTC), started_at: Some(datetime!(2022-11-20 0:00 UTC)), finished_at: Some(datetime!(2022-11-21 0:00 UTC)), @@ -142,20 +289,24 @@ pub(crate) mod test { None, ), ( - TaskView { + TaskDump { uid: 1, - index_uid: Some(S("doggos")), + index_uid: Some(S("doggo")), status: Status::Enqueued, - kind: Kind::DocumentImport { + kind: KindDump::DocumentImport { method: milli::update::IndexDocumentsMethod::UpdateDocuments, allow_index_creation: true, + primary_key: None, + documents_count: 2, }, - details: None, + details: Some(Details::DocumentAddition { + received_documents: 2, + indexed_documents: None, + }), error: None, - duration: Some(Duration::DAY), enqueued_at: datetime!(2022-11-11 0:00 UTC), - started_at: Some(datetime!(2022-11-20 0:00 UTC)), - finished_at: Some(datetime!(2022-11-21 0:00 UTC)), + started_at: None, + finished_at: None, }, Some(vec![ json!({ "id": 4, "race": "leonberg" }) @@ -169,14 +320,13 @@ pub(crate) mod test { ]), ), ( - TaskView { + TaskDump { uid: 5, - index_uid: Some(S("doggos")), + index_uid: Some(S("catto")), status: Status::Enqueued, - kind: Kind::IndexDeletion, + kind: KindDump::IndexDeletion, details: None, error: None, - duration: None, enqueued_at: datetime!(2022-11-15 0:00 UTC), started_at: None, finished_at: None, @@ -223,7 +373,7 @@ pub(crate) mod test { pub fn create_test_dump() -> File { let instance_uid = create_test_instance_uid(); - let dump = DumpWriter::new(instance_uid.clone()).unwrap(); + let dump = DumpWriter::new(Some(instance_uid.clone())).unwrap(); // ========== Adding an index let documents = create_test_documents(); diff --git a/dump/src/reader/compat/v5_to_v6.rs b/dump/src/reader/compat/v5_to_v6.rs index cfcf7b545..ced41fc65 100644 --- a/dump/src/reader/compat/v5_to_v6.rs +++ b/dump/src/reader/compat/v5_to_v6.rs @@ -412,7 +412,7 @@ pub(crate) mod test { // tasks let tasks = dump.tasks().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); - meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"0fff3c32487e3d3058d51ed951c1057f"); assert_eq!(update_files.len(), 22); assert!(update_files[0].is_none()); // the dump creation assert!(update_files[1].is_some()); // the enqueued document addition @@ -420,7 +420,7 @@ pub(crate) mod test { // keys let keys = dump.keys().collect::>>().unwrap(); - meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb"); // indexes let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); @@ -442,14 +442,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"8e5cadabf74aebe1160bf51c3d489efe"); let documents = products .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -461,14 +461,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4894ac1e74b9e1069ed5ee262b7a1aca"); let documents = movies .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 200); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a"); // spells insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -480,13 +480,13 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"054dbf08a79e08bb9becba6f5d090f13"); let documents = spells .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } } diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index 9f6eca4c5..6542786f2 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -72,7 +72,7 @@ pub(crate) mod test { // tasks let tasks = dump.tasks().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); - meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"0fff3c32487e3d3058d51ed951c1057f"); assert_eq!(update_files.len(), 22); assert!(update_files[0].is_none()); // the dump creation assert!(update_files[1].is_some()); // the enqueued document addition @@ -80,7 +80,7 @@ pub(crate) mod test { // keys let keys = dump.keys().collect::>>().unwrap(); - meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb"); // indexes let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); @@ -102,14 +102,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"8e5cadabf74aebe1160bf51c3d489efe"); let documents = products .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -121,14 +121,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4894ac1e74b9e1069ed5ee262b7a1aca"); let documents = movies .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 200); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a"); // spells insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -140,14 +140,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"054dbf08a79e08bb9becba6f5d090f13"); let documents = spells .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } #[test] @@ -162,14 +162,14 @@ pub(crate) mod test { // tasks let tasks = dump.tasks().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); - meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"0903b293c6ff8dc0819cbd3406848ef2"); assert_eq!(update_files.len(), 10); assert!(update_files[0].is_some()); // the enqueued document addition assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed // keys let keys = dump.keys().collect::>>().unwrap(); - meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"23afab5753c5a99d8c530075bf0ebd9c"); // indexes let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); @@ -191,14 +191,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"1f9da51a4518166fb440def5437eafdb"); let documents = products .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -210,14 +210,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"488816aba82c1bd65f1609630055c611"); let documents = movies .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 110); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -229,14 +229,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"7b4f66dad597dc651650f35fe34be27f"); let documents = spells .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } #[test] @@ -251,14 +251,14 @@ pub(crate) mod test { // tasks let tasks = dump.tasks().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); - meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"891538c6fe0ba5187853a4f04890f9b5"); assert_eq!(update_files.len(), 10); assert!(update_files[0].is_some()); // the enqueued document addition assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed // keys let keys = dump.keys().collect::>>().unwrap(); - meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce"); // indexes let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); @@ -281,14 +281,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"855f3165dec609b919171ff83f82b364"); let documents = products .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -300,14 +300,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"43e0bf1746c3ea1d64c1e10ea544c190"); let documents = movies .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 110); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022"); // movies2 insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -319,14 +319,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"5fd06a5038f49311600379d43412b655"); let documents = movies2 .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 0); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce"); // spells insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -338,14 +338,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"5fd06a5038f49311600379d43412b655"); let documents = spells .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } #[test] @@ -360,14 +360,14 @@ pub(crate) mod test { // tasks let tasks = dump.tasks().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); - meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"c52c07e1b356cce6982e2aeea7d0bf5e"); assert_eq!(update_files.len(), 9); assert!(update_files[0].is_some()); // the enqueued document addition assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed // keys let keys = dump.keys().collect::>>().unwrap(); - meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @""); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce"); // indexes let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); @@ -390,14 +390,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"b15b71f56dd082d8e8ec5182e688bf36"); let documents = products .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5"); // movies insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -409,14 +409,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"1e51f7fdc322176408f471a6d90d7698"); let documents = movies .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 110); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022"); // movies2 insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -428,14 +428,14 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"8aebab01301d266acf3e18dd449c008f"); let documents = movies2 .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 0); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce"); // spells insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" @@ -447,13 +447,13 @@ pub(crate) mod test { } "###); - meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @""); + meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"8aebab01301d266acf3e18dd449c008f"); let documents = spells .documents() .unwrap() .collect::>>() .unwrap(); assert_eq!(documents.len(), 10); - meili_snap::snapshot_hash!(format!("{:#?}", documents), @""); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } } diff --git a/dump/src/reader/v6/mod.rs b/dump/src/reader/v6/mod.rs index d69fb0542..9752187e3 100644 --- a/dump/src/reader/v6/mod.rs +++ b/dump/src/reader/v6/mod.rs @@ -23,13 +23,13 @@ pub type Settings = meilisearch_types::settings::Settings; pub type Checked = meilisearch_types::settings::Checked; pub type Unchecked = meilisearch_types::settings::Unchecked; -pub type Task = tasks::TaskDump; +pub type Task = crate::TaskDump; pub type Key = meilisearch_types::keys::Key; // ===== Other types to clarify the code of the compat module // everything related to the tasks pub type Status = meilisearch_types::tasks::Status; -pub type Kind = tasks::KindDump; +pub type Kind = crate::KindDump; pub type Details = meilisearch_types::tasks::Details; // everything related to the settings diff --git a/dump/src/reader/v6/tasks.rs b/dump/src/reader/v6/tasks.rs index 25693e779..8b1378917 100644 --- a/dump/src/reader/v6/tasks.rs +++ b/dump/src/reader/v6/tasks.rs @@ -1,81 +1 @@ -use meilisearch_types::{ - error::ResponseError, - milli::update::IndexDocumentsMethod, - settings::Unchecked, - tasks::{Details, Status, TaskId}, -}; -use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TaskDump { - pub uid: TaskId, - #[serde(default)] - pub index_uid: Option, - pub status: Status, - // TODO use our own Kind for the user - #[serde(rename = "type")] - pub kind: KindDump, - - #[serde(skip_serializing_if = "Option::is_none")] - pub details: Option
, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - - #[serde(with = "time::serde::rfc3339")] - pub enqueued_at: OffsetDateTime, - #[serde( - with = "time::serde::rfc3339::option", - skip_serializing_if = "Option::is_none", - default - )] - pub started_at: Option, - #[serde( - with = "time::serde::rfc3339::option", - skip_serializing_if = "Option::is_none", - default - )] - pub finished_at: Option, -} - -// A `Kind` specific version made for the dump. If modified you may break the dump. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum KindDump { - DocumentImport { - primary_key: Option, - method: IndexDocumentsMethod, - documents_count: u64, - allow_index_creation: bool, - }, - DocumentDeletion { - documents_ids: Vec, - }, - DocumentClear, - Settings { - settings: meilisearch_types::settings::Settings, - is_deletion: bool, - allow_index_creation: bool, - }, - IndexDeletion, - IndexCreation { - primary_key: Option, - }, - IndexUpdate { - primary_key: Option, - }, - IndexSwap { - lhs: String, - rhs: String, - }, - CancelTask { - tasks: Vec, - }, - DeleteTasks { - query: String, - tasks: Vec, - }, - DumpExport, - Snapshot, -} diff --git a/dump/src/writer.rs b/dump/src/writer.rs index c52399b71..92ba5fb7a 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -15,7 +15,7 @@ use tempfile::TempDir; use time::OffsetDateTime; use uuid::Uuid; -use crate::{reader::Document, IndexMetadata, Metadata, Result, CURRENT_DUMP_VERSION}; +use crate::{reader::Document, IndexMetadata, Metadata, Result, TaskDump, CURRENT_DUMP_VERSION}; pub struct DumpWriter { dir: TempDir, @@ -109,7 +109,7 @@ impl TaskWriter { /// Pushes tasks in the dump. /// If the tasks has an associated `update_file` it'll use the `task_id` as its name. - pub fn push_task(&mut self, task: &Task) -> Result { + pub fn push_task(&mut self, task: &TaskDump) -> Result { self.queue.write_all(&serde_json::to_vec(task)?)?; self.queue.write_all(b"\n")?; @@ -328,8 +328,6 @@ pub(crate) mod test { // ==== checking the task queue let tasks_queue = fs::read_to_string(dump_path.join("tasks/queue.jsonl")).unwrap(); for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) { - // TODO: This can be removed once `Duration` from the `TaskView` is implemented. - expected.0.duration = None; // TODO: uncomment this one once the we write the dump integration in the index-scheduler // assert_eq!(serde_json::from_str::(task).unwrap(), expected.0); diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index b383a1360..b52da0f55 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -503,7 +503,7 @@ impl IndexScheduler { let mut tasks = dump.create_tasks_queue()?; for ret in self.all_tasks.iter(&rtxn)? { let (_, task) = ret?; - let mut dump_content_file = tasks.push_task(&task)?; + let mut dump_content_file = tasks.push_task((&task).into())?; // 2.1. Dump the `content_file` associated with the task if there is one. if let Some(content_file) = task.content_uuid() {