From 6bd6321226c42b393c580db964bd6792c6c4c096 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 17 Oct 2022 17:38:31 +0200 Subject: [PATCH] dump the content of the dump tasks instead of recreating at import time with wrong API keys --- dump/src/lib.rs | 18 +++++++++++++--- dump/src/reader/compat/v5_to_v6.rs | 34 ++++++++++++++++++------------ dump/src/reader/mod.rs | 8 +++---- dump/src/reader/v5/mod.rs | 13 ++++++------ index-scheduler/src/lib.rs | 12 +++++------ meilisearch-http/src/lib.rs | 6 +++--- 6 files changed, 54 insertions(+), 37 deletions(-) diff --git a/dump/src/lib.rs b/dump/src/lib.rs index b0dfdce91..e526171bf 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -1,8 +1,10 @@ use meilisearch_types::{ error::ResponseError, + keys::Key, milli::update::IndexDocumentsMethod, settings::Unchecked, tasks::{Details, KindWithContent, Status, Task, TaskId}, + InstanceUid, }; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; @@ -120,6 +122,8 @@ pub enum KindDump { }, DumpExport { dump_uid: String, + keys: Vec, + instance_uid: Option, }, Snapshot, } @@ -181,7 +185,15 @@ impl From for KindDump { KindWithContent::TaskDeletion { query, tasks } => { KindDump::DeleteTasks { query, tasks } } - KindWithContent::DumpExport { dump_uid, .. } => KindDump::DumpExport { dump_uid }, + KindWithContent::DumpExport { + dump_uid, + keys, + instance_uid, + } => KindDump::DumpExport { + dump_uid, + keys, + instance_uid, + }, KindWithContent::Snapshot => KindDump::Snapshot, } } @@ -444,7 +456,7 @@ pub(crate) mod test { drop(indexes); // ==== checking the task queue - for (task, expected) in dump.tasks().zip(create_test_tasks()) { + for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) { let (task, content_file) = task.unwrap(); assert_eq!(task, expected.0); @@ -463,7 +475,7 @@ pub(crate) mod test { } // ==== checking the keys - for (key, expected) in dump.keys().zip(create_test_api_keys()) { + for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) { assert_eq!(key.unwrap(), expected); } } diff --git a/dump/src/reader/compat/v5_to_v6.rs b/dump/src/reader/compat/v5_to_v6.rs index 6b99be81c..aacdbd4e0 100644 --- a/dump/src/reader/compat/v5_to_v6.rs +++ b/dump/src/reader/compat/v5_to_v6.rs @@ -54,13 +54,16 @@ impl CompatV5ToV6 { pub fn tasks( &mut self, - ) -> Box>)>> + '_> { + ) -> Result>)>> + '_>> { + let instance_uid = self.instance_uid().ok().flatten().map(|uid| uid.clone()); + let keys = self.keys()?.collect::>>()?; + let tasks = match self { CompatV5ToV6::V5(v5) => v5.tasks(), CompatV5ToV6::Compat(compat) => compat.tasks(), }; - Box::new(tasks.map(|task| { - task.map(|(task, content_file)| { + Ok(Box::new(tasks.map(move |task| { + task.and_then(|(task, content_file)| { let task_view: v5::tasks::TaskView = task.clone().into(); let task = v6::Task { @@ -116,9 +119,11 @@ impl CompatV5ToV6 { allow_index_creation, settings: settings.into(), }, - v5::tasks::TaskContent::Dump { uid } => { - v6::Kind::DumpExport { dump_uid: uid } - } + v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpExport { + dump_uid: uid, + keys: keys.clone(), + instance_uid: instance_uid.clone(), + }, }, details: task_view.details.map(|details| match details { v5::Details::DocumentAddition { @@ -152,17 +157,18 @@ impl CompatV5ToV6 { finished_at: task_view.finished_at, }; - (task, content_file) + Ok((task, content_file)) }) - })) + }))) } - pub fn keys(&mut self) -> Box> + '_> { + pub fn keys(&mut self) -> Result> + '_>> { let keys = match self { - CompatV5ToV6::V5(v5) => v5.keys(), + CompatV5ToV6::V5(v5) => v5.keys()?, CompatV5ToV6::Compat(compat) => compat.keys(), }; - Box::new(keys.map(|key| { + + Ok(Box::new(keys.map(|key| { key.map(|key| v6::Key { description: key.description, name: key.name, @@ -186,7 +192,7 @@ impl CompatV5ToV6 { created_at: key.created_at, updated_at: key.updated_at, }) - })) + }))) } } @@ -412,7 +418,7 @@ pub(crate) mod test { insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d"); // tasks - let tasks = dump.tasks().collect::>>().unwrap(); + let tasks = dump.tasks().unwrap().collect::>>().unwrap(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac"); assert_eq!(update_files.len(), 22); @@ -421,7 +427,7 @@ pub(crate) mod test { assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed // keys - let keys = dump.keys().collect::>>().unwrap(); + let keys = dump.keys().unwrap().collect::>>().unwrap(); meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb"); // indexes diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index 67fadeac4..3c31bb2d7 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -98,16 +98,16 @@ impl DumpReader { pub fn tasks( &mut self, - ) -> Box>)>> + '_> { + ) -> Result>)>> + '_>> { match self { - DumpReader::Current(current) => current.tasks(), + DumpReader::Current(current) => Ok(current.tasks()), DumpReader::Compat(compat) => compat.tasks(), } } - pub fn keys(&mut self) -> Box> + '_> { + pub fn keys(&mut self) -> Result> + '_>> { match self { - DumpReader::Current(current) => current.keys(), + DumpReader::Current(current) => Ok(current.keys()), DumpReader::Compat(compat) => compat.keys(), } } diff --git a/dump/src/reader/v5/mod.rs b/dump/src/reader/v5/mod.rs index a6d97bc4a..9381f2ce4 100644 --- a/dump/src/reader/v5/mod.rs +++ b/dump/src/reader/v5/mod.rs @@ -34,7 +34,7 @@ use std::{ fs::{self, File}, - io::{BufRead, BufReader}, + io::{BufRead, BufReader, Seek, SeekFrom}, path::Path, }; @@ -176,12 +176,11 @@ impl V5Reader { })) } - pub fn keys(&mut self) -> Box> + '_> { - Box::new( - (&mut self.keys) - .lines() - .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }), - ) + pub fn keys(&mut self) -> Result> + '_>> { + self.keys.seek(SeekFrom::Start(0))?; + Ok(Box::new((&mut self.keys).lines().map( + |line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }, + ))) } } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 58e1f1a6d..6b50750b4 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -11,10 +11,8 @@ pub type TaskId = u32; use dump::{KindDump, TaskDump, UpdateFile}; pub use error::Error; -use meilisearch_types::keys::Key; use meilisearch_types::milli::documents::DocumentsBatchBuilder; use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; -use meilisearch_types::InstanceUid; use std::path::PathBuf; use std::sync::{Arc, RwLock}; @@ -400,8 +398,6 @@ impl IndexScheduler { &mut self, task: TaskDump, content_file: Option>, - keys: &[Key], - instance_uid: Option, ) -> Result { // Currently we don't need to access the tasks queue while loading a dump thus I can block everything. let mut wtxn = self.env.write_txn()?; @@ -479,9 +475,13 @@ impl IndexScheduler { KindDump::DeleteTasks { query, tasks } => { KindWithContent::TaskDeletion { query, tasks } } - KindDump::DumpExport { dump_uid } => KindWithContent::DumpExport { + KindDump::DumpExport { dump_uid, - keys: keys.to_vec(), + keys, + instance_uid, + } => KindWithContent::DumpExport { + dump_uid, + keys, instance_uid, }, KindDump::Snapshot => KindWithContent::Snapshot, diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 3c98bc517..5b2d9b89d 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -190,7 +190,7 @@ fn import_dump( // 2. Import the `Key`s. let mut keys = Vec::new(); auth.raw_delete_all_keys()?; - for key in dump_reader.keys() { + for key in dump_reader.keys()? { let key = key?; auth.raw_insert_key(key.clone())?; keys.push(key); @@ -259,9 +259,9 @@ fn import_dump( } // 4. Import the tasks. - for ret in dump_reader.tasks() { + for ret in dump_reader.tasks()? { let (task, file) = ret?; - index_scheduler.register_dumped_task(task, file, &keys, instance_uid)?; + index_scheduler.register_dumped_task(task, file)?; } Ok(()) }