mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-10 23:18:55 +01:00
dump the content of the dump tasks instead of recreating at import time with wrong API keys
This commit is contained in:
parent
655705eb2b
commit
6bd6321226
@ -1,8 +1,10 @@
|
|||||||
use meilisearch_types::{
|
use meilisearch_types::{
|
||||||
error::ResponseError,
|
error::ResponseError,
|
||||||
|
keys::Key,
|
||||||
milli::update::IndexDocumentsMethod,
|
milli::update::IndexDocumentsMethod,
|
||||||
settings::Unchecked,
|
settings::Unchecked,
|
||||||
tasks::{Details, KindWithContent, Status, Task, TaskId},
|
tasks::{Details, KindWithContent, Status, Task, TaskId},
|
||||||
|
InstanceUid,
|
||||||
};
|
};
|
||||||
use roaring::RoaringBitmap;
|
use roaring::RoaringBitmap;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@ -120,6 +122,8 @@ pub enum KindDump {
|
|||||||
},
|
},
|
||||||
DumpExport {
|
DumpExport {
|
||||||
dump_uid: String,
|
dump_uid: String,
|
||||||
|
keys: Vec<Key>,
|
||||||
|
instance_uid: Option<InstanceUid>,
|
||||||
},
|
},
|
||||||
Snapshot,
|
Snapshot,
|
||||||
}
|
}
|
||||||
@ -181,7 +185,15 @@ impl From<KindWithContent> for KindDump {
|
|||||||
KindWithContent::TaskDeletion { query, tasks } => {
|
KindWithContent::TaskDeletion { query, tasks } => {
|
||||||
KindDump::DeleteTasks { query, tasks }
|
KindDump::DeleteTasks { query, tasks }
|
||||||
}
|
}
|
||||||
KindWithContent::DumpExport { dump_uid, .. } => KindDump::DumpExport { dump_uid },
|
KindWithContent::DumpExport {
|
||||||
|
dump_uid,
|
||||||
|
keys,
|
||||||
|
instance_uid,
|
||||||
|
} => KindDump::DumpExport {
|
||||||
|
dump_uid,
|
||||||
|
keys,
|
||||||
|
instance_uid,
|
||||||
|
},
|
||||||
KindWithContent::Snapshot => KindDump::Snapshot,
|
KindWithContent::Snapshot => KindDump::Snapshot,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -444,7 +456,7 @@ pub(crate) mod test {
|
|||||||
drop(indexes);
|
drop(indexes);
|
||||||
|
|
||||||
// ==== checking the task queue
|
// ==== checking the task queue
|
||||||
for (task, expected) in dump.tasks().zip(create_test_tasks()) {
|
for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) {
|
||||||
let (task, content_file) = task.unwrap();
|
let (task, content_file) = task.unwrap();
|
||||||
assert_eq!(task, expected.0);
|
assert_eq!(task, expected.0);
|
||||||
|
|
||||||
@ -463,7 +475,7 @@ pub(crate) mod test {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ==== checking the keys
|
// ==== checking the keys
|
||||||
for (key, expected) in dump.keys().zip(create_test_api_keys()) {
|
for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
|
||||||
assert_eq!(key.unwrap(), expected);
|
assert_eq!(key.unwrap(), expected);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -54,13 +54,16 @@ impl CompatV5ToV6 {
|
|||||||
|
|
||||||
pub fn tasks(
|
pub fn tasks(
|
||||||
&mut self,
|
&mut self,
|
||||||
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> {
|
) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
|
||||||
|
let instance_uid = self.instance_uid().ok().flatten().map(|uid| uid.clone());
|
||||||
|
let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
let tasks = match self {
|
let tasks = match self {
|
||||||
CompatV5ToV6::V5(v5) => v5.tasks(),
|
CompatV5ToV6::V5(v5) => v5.tasks(),
|
||||||
CompatV5ToV6::Compat(compat) => compat.tasks(),
|
CompatV5ToV6::Compat(compat) => compat.tasks(),
|
||||||
};
|
};
|
||||||
Box::new(tasks.map(|task| {
|
Ok(Box::new(tasks.map(move |task| {
|
||||||
task.map(|(task, content_file)| {
|
task.and_then(|(task, content_file)| {
|
||||||
let task_view: v5::tasks::TaskView = task.clone().into();
|
let task_view: v5::tasks::TaskView = task.clone().into();
|
||||||
|
|
||||||
let task = v6::Task {
|
let task = v6::Task {
|
||||||
@ -116,9 +119,11 @@ impl CompatV5ToV6 {
|
|||||||
allow_index_creation,
|
allow_index_creation,
|
||||||
settings: settings.into(),
|
settings: settings.into(),
|
||||||
},
|
},
|
||||||
v5::tasks::TaskContent::Dump { uid } => {
|
v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpExport {
|
||||||
v6::Kind::DumpExport { dump_uid: uid }
|
dump_uid: uid,
|
||||||
}
|
keys: keys.clone(),
|
||||||
|
instance_uid: instance_uid.clone(),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
details: task_view.details.map(|details| match details {
|
details: task_view.details.map(|details| match details {
|
||||||
v5::Details::DocumentAddition {
|
v5::Details::DocumentAddition {
|
||||||
@ -152,17 +157,18 @@ impl CompatV5ToV6 {
|
|||||||
finished_at: task_view.finished_at,
|
finished_at: task_view.finished_at,
|
||||||
};
|
};
|
||||||
|
|
||||||
(task, content_file)
|
Ok((task, content_file))
|
||||||
})
|
})
|
||||||
}))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> {
|
pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
|
||||||
let keys = match self {
|
let keys = match self {
|
||||||
CompatV5ToV6::V5(v5) => v5.keys(),
|
CompatV5ToV6::V5(v5) => v5.keys()?,
|
||||||
CompatV5ToV6::Compat(compat) => compat.keys(),
|
CompatV5ToV6::Compat(compat) => compat.keys(),
|
||||||
};
|
};
|
||||||
Box::new(keys.map(|key| {
|
|
||||||
|
Ok(Box::new(keys.map(|key| {
|
||||||
key.map(|key| v6::Key {
|
key.map(|key| v6::Key {
|
||||||
description: key.description,
|
description: key.description,
|
||||||
name: key.name,
|
name: key.name,
|
||||||
@ -186,7 +192,7 @@ impl CompatV5ToV6 {
|
|||||||
created_at: key.created_at,
|
created_at: key.created_at,
|
||||||
updated_at: key.updated_at,
|
updated_at: key.updated_at,
|
||||||
})
|
})
|
||||||
}))
|
})))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -412,7 +418,7 @@ pub(crate) mod test {
|
|||||||
insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
|
insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
|
||||||
|
|
||||||
// tasks
|
// tasks
|
||||||
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
|
let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||||
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
|
||||||
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac");
|
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac");
|
||||||
assert_eq!(update_files.len(), 22);
|
assert_eq!(update_files.len(), 22);
|
||||||
@ -421,7 +427,7 @@ pub(crate) mod test {
|
|||||||
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
|
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
|
||||||
|
|
||||||
// keys
|
// keys
|
||||||
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
|
let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
|
||||||
meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
|
meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
|
||||||
|
|
||||||
// indexes
|
// indexes
|
||||||
|
@ -98,16 +98,16 @@ impl DumpReader {
|
|||||||
|
|
||||||
pub fn tasks(
|
pub fn tasks(
|
||||||
&mut self,
|
&mut self,
|
||||||
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> {
|
) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
|
||||||
match self {
|
match self {
|
||||||
DumpReader::Current(current) => current.tasks(),
|
DumpReader::Current(current) => Ok(current.tasks()),
|
||||||
DumpReader::Compat(compat) => compat.tasks(),
|
DumpReader::Compat(compat) => compat.tasks(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> {
|
pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
|
||||||
match self {
|
match self {
|
||||||
DumpReader::Current(current) => current.keys(),
|
DumpReader::Current(current) => Ok(current.keys()),
|
||||||
DumpReader::Compat(compat) => compat.keys(),
|
DumpReader::Compat(compat) => compat.keys(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -34,7 +34,7 @@
|
|||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
fs::{self, File},
|
fs::{self, File},
|
||||||
io::{BufRead, BufReader},
|
io::{BufRead, BufReader, Seek, SeekFrom},
|
||||||
path::Path,
|
path::Path,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -176,12 +176,11 @@ impl V5Reader {
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
|
pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<Key>> + '_>> {
|
||||||
Box::new(
|
self.keys.seek(SeekFrom::Start(0))?;
|
||||||
(&mut self.keys)
|
Ok(Box::new((&mut self.keys).lines().map(
|
||||||
.lines()
|
|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
|
||||||
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
|
)))
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -11,10 +11,8 @@ pub type TaskId = u32;
|
|||||||
|
|
||||||
use dump::{KindDump, TaskDump, UpdateFile};
|
use dump::{KindDump, TaskDump, UpdateFile};
|
||||||
pub use error::Error;
|
pub use error::Error;
|
||||||
use meilisearch_types::keys::Key;
|
|
||||||
use meilisearch_types::milli::documents::DocumentsBatchBuilder;
|
use meilisearch_types::milli::documents::DocumentsBatchBuilder;
|
||||||
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
|
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
|
||||||
use meilisearch_types::InstanceUid;
|
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
@ -400,8 +398,6 @@ impl IndexScheduler {
|
|||||||
&mut self,
|
&mut self,
|
||||||
task: TaskDump,
|
task: TaskDump,
|
||||||
content_file: Option<Box<UpdateFile>>,
|
content_file: Option<Box<UpdateFile>>,
|
||||||
keys: &[Key],
|
|
||||||
instance_uid: Option<InstanceUid>,
|
|
||||||
) -> Result<Task> {
|
) -> Result<Task> {
|
||||||
// Currently we don't need to access the tasks queue while loading a dump thus I can block everything.
|
// Currently we don't need to access the tasks queue while loading a dump thus I can block everything.
|
||||||
let mut wtxn = self.env.write_txn()?;
|
let mut wtxn = self.env.write_txn()?;
|
||||||
@ -479,9 +475,13 @@ impl IndexScheduler {
|
|||||||
KindDump::DeleteTasks { query, tasks } => {
|
KindDump::DeleteTasks { query, tasks } => {
|
||||||
KindWithContent::TaskDeletion { query, tasks }
|
KindWithContent::TaskDeletion { query, tasks }
|
||||||
}
|
}
|
||||||
KindDump::DumpExport { dump_uid } => KindWithContent::DumpExport {
|
KindDump::DumpExport {
|
||||||
dump_uid,
|
dump_uid,
|
||||||
keys: keys.to_vec(),
|
keys,
|
||||||
|
instance_uid,
|
||||||
|
} => KindWithContent::DumpExport {
|
||||||
|
dump_uid,
|
||||||
|
keys,
|
||||||
instance_uid,
|
instance_uid,
|
||||||
},
|
},
|
||||||
KindDump::Snapshot => KindWithContent::Snapshot,
|
KindDump::Snapshot => KindWithContent::Snapshot,
|
||||||
|
@ -190,7 +190,7 @@ fn import_dump(
|
|||||||
// 2. Import the `Key`s.
|
// 2. Import the `Key`s.
|
||||||
let mut keys = Vec::new();
|
let mut keys = Vec::new();
|
||||||
auth.raw_delete_all_keys()?;
|
auth.raw_delete_all_keys()?;
|
||||||
for key in dump_reader.keys() {
|
for key in dump_reader.keys()? {
|
||||||
let key = key?;
|
let key = key?;
|
||||||
auth.raw_insert_key(key.clone())?;
|
auth.raw_insert_key(key.clone())?;
|
||||||
keys.push(key);
|
keys.push(key);
|
||||||
@ -259,9 +259,9 @@ fn import_dump(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 4. Import the tasks.
|
// 4. Import the tasks.
|
||||||
for ret in dump_reader.tasks() {
|
for ret in dump_reader.tasks()? {
|
||||||
let (task, file) = ret?;
|
let (task, file) = ret?;
|
||||||
index_scheduler.register_dumped_task(task, file, &keys, instance_uid)?;
|
index_scheduler.register_dumped_task(task, file)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user