dump the content of the dump tasks instead of recreating at import time with wrong API keys

dump/src/lib.rs

@@ -1,8 +1,10 @@
 use meilisearch_types::{
     error::ResponseError,
+    keys::Key,
     milli::update::IndexDocumentsMethod,
     settings::Unchecked,
     tasks::{Details, KindWithContent, Status, Task, TaskId},
+    InstanceUid,
 };
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
@@ -120,6 +122,8 @@ pub enum KindDump {
     },
     DumpExport {
         dump_uid: String,
+        keys: Vec<Key>,
+        instance_uid: Option<InstanceUid>,
     },
     Snapshot,
 }
@@ -181,7 +185,15 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::TaskDeletion { query, tasks } => {
                 KindDump::DeleteTasks { query, tasks }
             }
-            KindWithContent::DumpExport { dump_uid, .. } => KindDump::DumpExport { dump_uid },
+            KindWithContent::DumpExport {
+                dump_uid,
+                keys,
+                instance_uid,
+            } => KindDump::DumpExport {
+                dump_uid,
+                keys,
+                instance_uid,
+            },
             KindWithContent::Snapshot => KindDump::Snapshot,
         }
     }
@@ -444,7 +456,7 @@ pub(crate) mod test {
         drop(indexes);
 
         // ==== checking the task queue
-        for (task, expected) in dump.tasks().zip(create_test_tasks()) {
+        for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) {
             let (task, content_file) = task.unwrap();
             assert_eq!(task, expected.0);
 
@@ -463,7 +475,7 @@ pub(crate) mod test {
         }
 
         // ==== checking the keys
-        for (key, expected) in dump.keys().zip(create_test_api_keys()) {
+        for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
             assert_eq!(key.unwrap(), expected);
         }
     }

dump/src/reader/compat/v5_to_v6.rs

@@ -54,13 +54,16 @@ impl CompatV5ToV6 {
 
     pub fn tasks(
         &mut self,
-    ) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> {
+    ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
+        let instance_uid = self.instance_uid().ok().flatten().map(|uid| uid.clone());
+        let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
+
         let tasks = match self {
             CompatV5ToV6::V5(v5) => v5.tasks(),
             CompatV5ToV6::Compat(compat) => compat.tasks(),
         };
-        Box::new(tasks.map(|task| {
+        Ok(Box::new(tasks.map(move |task| {
-            task.map(|(task, content_file)| {
+            task.and_then(|(task, content_file)| {
                 let task_view: v5::tasks::TaskView = task.clone().into();
 
                 let task = v6::Task {
@@ -116,9 +119,11 @@ impl CompatV5ToV6 {
                             allow_index_creation,
                             settings: settings.into(),
                         },
-                        v5::tasks::TaskContent::Dump { uid } => {
+                        v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpExport {
-                            v6::Kind::DumpExport { dump_uid: uid }
+                            dump_uid: uid,
-                        }
+                            keys: keys.clone(),
+                            instance_uid: instance_uid.clone(),
+                        },
                     },
                     details: task_view.details.map(|details| match details {
                         v5::Details::DocumentAddition {
@@ -152,17 +157,18 @@ impl CompatV5ToV6 {
                     finished_at: task_view.finished_at,
                 };
 
-                (task, content_file)
+                Ok((task, content_file))
             })
-        }))
+        })))
     }
 
-    pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> {
+    pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
         let keys = match self {
-            CompatV5ToV6::V5(v5) => v5.keys(),
+            CompatV5ToV6::V5(v5) => v5.keys()?,
             CompatV5ToV6::Compat(compat) => compat.keys(),
         };
-        Box::new(keys.map(|key| {
+
+        Ok(Box::new(keys.map(|key| {
             key.map(|key| v6::Key {
                 description: key.description,
                 name: key.name,
@@ -186,7 +192,7 @@ impl CompatV5ToV6 {
                 created_at: key.created_at,
                 updated_at: key.updated_at,
             })
-        }))
+        })))
     }
 }
 
@@ -412,7 +418,7 @@ pub(crate) mod test {
         insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
-        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
+        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
         meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac");
         assert_eq!(update_files.len(), 22);
@@ -421,7 +427,7 @@ pub(crate) mod test {
         assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
 
         // keys
-        let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
+        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
         meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
 
         // indexes

dump/src/reader/mod.rs

@@ -98,16 +98,16 @@ impl DumpReader {
 
     pub fn tasks(
         &mut self,
-    ) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> {
+    ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
         match self {
-            DumpReader::Current(current) => current.tasks(),
+            DumpReader::Current(current) => Ok(current.tasks()),
             DumpReader::Compat(compat) => compat.tasks(),
         }
     }
 
-    pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> {
+    pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
         match self {
-            DumpReader::Current(current) => current.keys(),
+            DumpReader::Current(current) => Ok(current.keys()),
             DumpReader::Compat(compat) => compat.keys(),
         }
     }

dump/src/reader/v5/mod.rs

@@ -34,7 +34,7 @@
 
 use std::{
     fs::{self, File},
-    io::{BufRead, BufReader},
+    io::{BufRead, BufReader, Seek, SeekFrom},
     path::Path,
 };
 
@@ -176,12 +176,11 @@ impl V5Reader {
         }))
     }
 
-    pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
+    pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<Key>> + '_>> {
-        Box::new(
+        self.keys.seek(SeekFrom::Start(0))?;
-            (&mut self.keys)
+        Ok(Box::new((&mut self.keys).lines().map(
-                .lines()
+            |line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
-                .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
+        )))
-        )
     }
 }
 

index-scheduler/src/lib.rs

@@ -11,10 +11,8 @@ pub type TaskId = u32;
 
 use dump::{KindDump, TaskDump, UpdateFile};
 pub use error::Error;
-use meilisearch_types::keys::Key;
 use meilisearch_types::milli::documents::DocumentsBatchBuilder;
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
-use meilisearch_types::InstanceUid;
 
 use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
@@ -400,8 +398,6 @@ impl IndexScheduler {
         &mut self,
         task: TaskDump,
         content_file: Option<Box<UpdateFile>>,
-        keys: &[Key],
-        instance_uid: Option<InstanceUid>,
     ) -> Result<Task> {
         // Currently we don't need to access the tasks queue while loading a dump thus I can block everything.
         let mut wtxn = self.env.write_txn()?;
@@ -479,9 +475,13 @@ impl IndexScheduler {
                 KindDump::DeleteTasks { query, tasks } => {
                     KindWithContent::TaskDeletion { query, tasks }
                 }
-                KindDump::DumpExport { dump_uid } => KindWithContent::DumpExport {
+                KindDump::DumpExport {
                     dump_uid,
-                    keys: keys.to_vec(),
+                    keys,
+                    instance_uid,
+                } => KindWithContent::DumpExport {
+                    dump_uid,
+                    keys,
                     instance_uid,
                 },
                 KindDump::Snapshot => KindWithContent::Snapshot,

meilisearch-http/src/lib.rs

@@ -190,7 +190,7 @@ fn import_dump(
     // 2. Import the `Key`s.
     let mut keys = Vec::new();
     auth.raw_delete_all_keys()?;
-    for key in dump_reader.keys() {
+    for key in dump_reader.keys()? {
         let key = key?;
         auth.raw_insert_key(key.clone())?;
         keys.push(key);
@@ -259,9 +259,9 @@ fn import_dump(
     }
 
     // 4. Import the tasks.
-    for ret in dump_reader.tasks() {
+    for ret in dump_reader.tasks()? {
         let (task, file) = ret?;
-        index_scheduler.register_dumped_task(task, file, &keys, instance_uid)?;
+        index_scheduler.register_dumped_task(task, file)?;
     }
     Ok(())
 }