searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 06:44:27 +01:00
Code Issues Releases Wiki Activity

fix the import of the dumpv4&v5 when there is no instance-uid + rename the Kind+KindWithContent+Details variant for the DocumentImport and the Setting

Browse Source
This commit is contained in:
Irevoire 2022-10-21 18:03:10 +02:00 committed by Clément Renault
parent 131fe30934
commit 8d1408c65e
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
14 changed files with 98 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dump/src/lib.rs
View File

@ -149,7 +149,7 @@ impl From<Task> for TaskDump {
impl From<KindWithContent> for KindDump {
fn from(kind: KindWithContent) -> Self {
match kind {
KindWithContent::DocumentImport {
KindWithContent::DocumentAdditionOrUpdate {
primary_key,
method,
documents_count,
@ -165,8 +165,11 @@ impl From<KindWithContent> for KindDump {
KindDump::DocumentDeletion { documents_ids }
}
KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
KindWithContent::Settings {
new_settings, is_deletion, allow_index_creation, ..
KindWithContent::SettingsUpdate {
new_settings,
is_deletion,
allow_index_creation,
..
} => KindDump::Settings { settings: new_settings, is_deletion, allow_index_creation },
KindWithContent::IndexDeletion { .. } => KindDump::IndexDeletion,
KindWithContent::IndexCreation { primary_key, .. } => {
@ -274,7 +277,7 @@ pub(crate) mod test {
documents_count: 12,
},
canceled_by: None,
details: Some(Details::DocumentAddition {
details: Some(Details::DocumentAdditionOrUpdate {
received_documents: 12,
indexed_documents: Some(10),
}),
@ -297,7 +300,7 @@ pub(crate) mod test {
documents_count: 2,
},
canceled_by: None,
details: Some(Details::DocumentAddition {
details: Some(Details::DocumentAdditionOrUpdate {
received_documents: 2,
indexed_documents: None,
}),

10
dump/src/reader/compat/v5_to_v6.rs
View File

@ -60,7 +60,11 @@ impl CompatV5ToV6 {
};
Ok(Box::new(tasks.map(move |task| {
task.and_then(|(task, content_file)| {
let task_view: v5::tasks::TaskView = task.clone().into();
let mut task_view: v5::tasks::TaskView = task.clone().into();
if task_view.status == v5::Status::Processing {
task_view.started_at = None;
}
let task = v6::Task {
uid: task_view.uid,
@ -124,13 +128,13 @@ impl CompatV5ToV6 {
canceled_by: None,
details: task_view.details.map(|details| match details {
v5::Details::DocumentAddition { received_documents, indexed_documents } => {
v6::Details::DocumentAddition {
v6::Details::DocumentAdditionOrUpdate {
received_documents: received_documents as u64,
indexed_documents: indexed_documents.map(|i| i as u64),
}
}
v5::Details::Settings { settings } => {
v6::Details::Settings { settings: settings.into() }
v6::Details::SettingsUpdate { settings: settings.into() }
}
v5::Details::IndexInfo { primary_key } => {
v6::Details::IndexInfo { primary_key }

9
dump/src/reader/v4/mod.rs
View File

@ -1,5 +1,5 @@
use std::fs::{self, File};
use std::io::{BufRead, BufReader};
use std::io::{BufRead, BufReader, ErrorKind};
use std::path::Path;
use serde::{Deserialize, Serialize};
@ -88,8 +88,11 @@ impl V4Reader {
}
pub fn instance_uid(&self) -> Result<Option<Uuid>> {
let uuid = fs::read_to_string(self.dump.path().join("instance-uid"))?;
Ok(Some(Uuid::parse_str(&uuid)?))
match fs::read_to_string(self.dump.path().join("instance-uid")) {
Ok(uuid) => Ok(Some(Uuid::parse_str(&uuid)?)),
Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V4IndexReader>> + '_> {

9
dump/src/reader/v5/mod.rs
View File

@ -33,7 +33,7 @@
//!
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Seek, SeekFrom};
use std::io::{BufRead, BufReader, ErrorKind, Seek, SeekFrom};
use std::path::Path;
use serde::{Deserialize, Serialize};
@ -129,8 +129,11 @@ impl V5Reader {
}
pub fn instance_uid(&self) -> Result<Option<Uuid>> {
let uuid = fs::read_to_string(self.dump.path().join("instance-uid"))?;
Ok(Some(Uuid::parse_str(&uuid)?))
match fs::read_to_string(self.dump.path().join("instance-uid")) {
Ok(uuid) => Ok(Some(Uuid::parse_str(&uuid)?)),
Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
}
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V5IndexReader>> + '_> {

2
dump/src/reader/v5/tasks.rs
View File

@ -326,7 +326,7 @@ impl From<TaskContent> for TaskType {
}
}
#[derive(Debug, Deserialize)]
#[derive(Debug, PartialEq, Eq, Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(rename_all = "camelCase")]
pub enum TaskStatus {

8
index-scheduler/src/autobatcher.rs
View File

@ -43,12 +43,12 @@ impl AutobatchKind {
impl From<KindWithContent> for AutobatchKind {
fn from(kind: KindWithContent) -> Self {
match kind {
KindWithContent::DocumentImport { method, allow_index_creation, .. } => {
KindWithContent::DocumentAdditionOrUpdate { method, allow_index_creation, .. } => {
AutobatchKind::DocumentImport { method, allow_index_creation }
}
KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
KindWithContent::Settings { allow_index_creation, is_deletion, .. } => {
KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
AutobatchKind::Settings {
allow_index_creation: allow_index_creation && !is_deletion,
}
@ -449,7 +449,7 @@ mod tests {
}
fn doc_imp(method: IndexDocumentsMethod, allow_index_creation: bool) -> KindWithContent {
KindWithContent::DocumentImport {
KindWithContent::DocumentAdditionOrUpdate {
index_uid: String::from("doggo"),
primary_key: None,
method,
@ -471,7 +471,7 @@ mod tests {
}
fn settings(allow_index_creation: bool) -> KindWithContent {
KindWithContent::Settings {
KindWithContent::SettingsUpdate {
index_uid: String::from("doggo"),
new_settings: Default::default(),
is_deletion: false,

22
index-scheduler/src/batch.rs
View File

@ -181,7 +181,9 @@ impl IndexScheduler {
BatchKind::DocumentImport { method, import_ids, .. } => {
let tasks = self.get_existing_tasks(rtxn, import_ids)?;
let primary_key = match &tasks[0].kind {
KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(),
KindWithContent::DocumentAdditionOrUpdate { primary_key, .. } => {
primary_key.clone()
}
_ => unreachable!(),
};
@ -189,8 +191,10 @@ impl IndexScheduler {
let mut content_files = Vec::new();
for task in &tasks {
match task.kind {
KindWithContent::DocumentImport {
content_file, documents_count, ..
KindWithContent::DocumentAdditionOrUpdate {
content_file,
documents_count,
..
} => {
documents_counts.push(documents_count);
content_files.push(content_file);
@ -235,9 +239,9 @@ impl IndexScheduler {
let mut settings = Vec::new();
for task in &tasks {
match task.kind {
KindWithContent::Settings { ref new_settings, is_deletion, .. } => {
settings.push((is_deletion, new_settings.clone()))
}
KindWithContent::SettingsUpdate {
ref new_settings, is_deletion, ..
} => settings.push((is_deletion, new_settings.clone())),
_ => unreachable!(),
}
}
@ -873,14 +877,14 @@ impl IndexScheduler {
match ret {
Ok(DocumentAdditionResult { indexed_documents, number_of_documents }) => {
task.status = Status::Succeeded;
task.details = Some(Details::DocumentAddition {
task.details = Some(Details::DocumentAdditionOrUpdate {
received_documents: number_of_documents,
indexed_documents: Some(indexed_documents),
});
}
Err(error) => {
task.status = Status::Failed;
task.details = Some(Details::DocumentAddition {
task.details = Some(Details::DocumentAdditionOrUpdate {
received_documents: count,
indexed_documents: Some(count),
});
@ -914,7 +918,7 @@ impl IndexScheduler {
// TODO merge the settings to only do *one* reindexation.
for (task, (_, settings)) in tasks.iter_mut().zip(settings) {
let checked_settings = settings.clone().check();
task.details = Some(Details::Settings { settings });
task.details = Some(Details::SettingsUpdate { settings });
let mut builder =
milli::update::Settings::new(index_wtxn, index, indexer_config);

12
index-scheduler/src/lib.rs
View File

@ -562,7 +562,7 @@ impl IndexScheduler {
method,
documents_count,
allow_index_creation,
} => KindWithContent::DocumentImport {
} => KindWithContent::DocumentAdditionOrUpdate {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
primary_key,
method,
@ -578,7 +578,7 @@ impl IndexScheduler {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
},
KindDump::Settings { settings, is_deletion, allow_index_creation } => {
KindWithContent::Settings {
KindWithContent::SettingsUpdate {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
new_settings: settings,
is_deletion,
@ -788,7 +788,7 @@ mod tests {
content_file_uuid: u128,
documents_count: u64,
) -> KindWithContent {
KindWithContent::DocumentImport {
KindWithContent::DocumentAdditionOrUpdate {
index_uid: S(index),
primary_key: primary_key.map(ToOwned::to_owned),
method: ReplaceDocuments,
@ -1102,7 +1102,7 @@ mod tests {
meilisearch_types::document_formats::read_json(content.as_bytes(), file.as_file_mut())
.unwrap() as u64;
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,
@ -1144,7 +1144,7 @@ mod tests {
.unwrap() as u64;
file.persist().unwrap();
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,
@ -1255,7 +1255,7 @@ mod tests {
.unwrap() as u64;
file.persist().unwrap();
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,

4
index-scheduler/src/snapshot.rs
View File

@ -147,13 +147,13 @@ fn snapshot_task(task: &Task) -> String {
fn snaphsot_details(d: &Details) -> String {
match d {
Details::DocumentAddition {
Details::DocumentAdditionOrUpdate {
received_documents,
indexed_documents,
} => {
format!("{{ received_documents: {received_documents}, indexed_documents: {indexed_documents:?} }}")
}
Details::Settings { settings } => {
Details::SettingsUpdate { settings } => {
format!("{{ settings: {settings:?} }}")
}
Details::IndexInfo { primary_key } => {

4
index-scheduler/src/utils.rs
View File

@ -236,10 +236,10 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
use KindWithContent as K;
let mut index_uids = vec![];
match &mut task.kind {
K::DocumentImport { index_uid, .. } => index_uids.push(index_uid),
K::DocumentAdditionOrUpdate { index_uid, .. } => index_uids.push(index_uid),
K::DocumentDeletion { index_uid, .. } => index_uids.push(index_uid),
K::DocumentClear { index_uid } => index_uids.push(index_uid),
K::Settings { index_uid, .. } => index_uids.push(index_uid),
K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid),
K::IndexDeletion { index_uid } => index_uids.push(index_uid),
K::IndexCreation { index_uid, .. } => index_uids.push(index_uid),
K::IndexUpdate { index_uid, .. } => index_uids.push(index_uid),

2
meilisearch-http/src/routes/indexes/documents.rs
View File

@ -255,7 +255,7 @@ async fn document_addition(
}
};
let task = KindWithContent::DocumentImport {
let task = KindWithContent::DocumentAdditionOrUpdate {
method,
content_file: uuid,
documents_count,

8
meilisearch-http/src/routes/indexes/settings.rs
View File

@ -40,7 +40,7 @@ macro_rules! make_setting_route {
let new_settings = Settings { $attr: Setting::Reset, ..Default::default() };
let allow_index_creation = index_scheduler.filters().allow_index_creation;
let task = KindWithContent::Settings {
let task = KindWithContent::SettingsUpdate {
index_uid: index_uid.into_inner(),
new_settings,
is_deletion: true,
@ -78,7 +78,7 @@ macro_rules! make_setting_route {
};
let allow_index_creation = index_scheduler.filters().allow_index_creation;
let task = KindWithContent::Settings {
let task = KindWithContent::SettingsUpdate {
index_uid: index_uid.into_inner(),
new_settings,
is_deletion: false,
@ -436,7 +436,7 @@ pub async fn update_all(
);
let allow_index_creation = index_scheduler.filters().allow_index_creation;
let task = KindWithContent::Settings {
let task = KindWithContent::SettingsUpdate {
index_uid: index_uid.into_inner(),
new_settings,
is_deletion: false,
@ -467,7 +467,7 @@ pub async fn delete_all(
let new_settings = Settings::cleared().into_unchecked();
let allow_index_creation = index_scheduler.filters().allow_index_creation;
let task = KindWithContent::Settings {
let task = KindWithContent::SettingsUpdate {
index_uid: index_uid.into_inner(),
new_settings,
is_deletion: true,

14
meilisearch-http/src/routes/tasks.rs
View File

@ -121,12 +121,14 @@ pub struct DetailsView {
impl From<Details> for DetailsView {
fn from(details: Details) -> Self {
match details.clone() {
Details::DocumentAddition { received_documents, indexed_documents } => DetailsView {
received_documents: Some(received_documents),
indexed_documents,
..DetailsView::default()
},
Details::Settings { settings } => {
Details::DocumentAdditionOrUpdate { received_documents, indexed_documents } => {
DetailsView {
received_documents: Some(received_documents),
indexed_documents,
..DetailsView::default()
}
}
Details::SettingsUpdate { settings } => {
DetailsView { settings: Some(settings), ..DetailsView::default() }
}
Details::IndexInfo { primary_key } => {

58
meilisearch-types/src/tasks.rs
View File

@ -46,10 +46,10 @@ impl Task {
| TaskCancelation { .. }
| TaskDeletion { .. }
| IndexSwap { .. } => None,
DocumentImport { index_uid, .. }
DocumentAdditionOrUpdate { index_uid, .. }
| DocumentDeletion { index_uid, .. }
| DocumentClear { index_uid }
| Settings { index_uid, .. }
| SettingsUpdate { index_uid, .. }
| IndexCreation { index_uid, .. }
| IndexUpdate { index_uid, .. }
| IndexDeletion { index_uid } => Some(index_uid),
@ -64,10 +64,12 @@ impl Task {
/// Return the content-uuid if there is one
pub fn content_uuid(&self) -> Option<&Uuid> {
match self.kind {
KindWithContent::DocumentImport { ref content_file, .. } => Some(content_file),
KindWithContent::DocumentAdditionOrUpdate { ref content_file, .. } => {
Some(content_file)
}
KindWithContent::DocumentDeletion { .. }
| KindWithContent::DocumentClear { .. }
| KindWithContent::Settings { .. }
| KindWithContent::SettingsUpdate { .. }
| KindWithContent::IndexDeletion { .. }
| KindWithContent::IndexCreation { .. }
| KindWithContent::IndexUpdate { .. }
@ -83,7 +85,7 @@ impl Task {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum KindWithContent {
DocumentImport {
DocumentAdditionOrUpdate {
index_uid: String,
primary_key: Option<String>,
method: IndexDocumentsMethod,
@ -98,7 +100,7 @@ pub enum KindWithContent {
DocumentClear {
index_uid: String,
},
Settings {
SettingsUpdate {
index_uid: String,
new_settings: Settings<Unchecked>,
is_deletion: bool,
@ -137,10 +139,10 @@ pub enum KindWithContent {
impl KindWithContent {
pub fn as_kind(&self) -> Kind {
match self {
KindWithContent::DocumentImport { .. } => Kind::DocumentImport,
KindWithContent::DocumentAdditionOrUpdate { .. } => Kind::DocumentAdditionOrUpdate,
KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion,
KindWithContent::DocumentClear { .. } => Kind::DocumentClear,
KindWithContent::Settings { .. } => Kind::Settings,
KindWithContent::SettingsUpdate { .. } => Kind::SettingsUpdate,
KindWithContent::IndexCreation { .. } => Kind::IndexCreation,
KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion,
KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate,
@ -157,10 +159,10 @@ impl KindWithContent {
match self {
DumpExport { .. } | Snapshot | TaskCancelation { .. } | TaskDeletion { .. } => None,
DocumentImport { index_uid, .. }
DocumentAdditionOrUpdate { index_uid, .. }
| DocumentDeletion { index_uid, .. }
| DocumentClear { index_uid }
| Settings { index_uid, .. }
| SettingsUpdate { index_uid, .. }
| IndexCreation { index_uid, .. }
| IndexUpdate { index_uid, .. }
| IndexDeletion { index_uid } => Some(vec![index_uid]),
@ -179,8 +181,8 @@ impl KindWithContent {
/// `None` if it cannot be generated.
pub fn default_details(&self) -> Option<Details> {
match self {
KindWithContent::DocumentImport { documents_count, .. } => {
Some(Details::DocumentAddition {
KindWithContent::DocumentAdditionOrUpdate { documents_count, .. } => {
Some(Details::DocumentAdditionOrUpdate {
received_documents: *documents_count,
indexed_documents: None,
})
@ -194,8 +196,8 @@ impl KindWithContent {
KindWithContent::DocumentClear { .. } => {
Some(Details::ClearAll { deleted_documents: None })
}
KindWithContent::Settings { new_settings, .. } => {
Some(Details::Settings { settings: new_settings.clone() })
KindWithContent::SettingsUpdate { new_settings, .. } => {
Some(Details::SettingsUpdate { settings: new_settings.clone() })
}
KindWithContent::IndexDeletion { .. } => None,
KindWithContent::IndexCreation { primary_key, .. }
@ -222,8 +224,8 @@ impl KindWithContent {
pub fn default_finished_details(&self) -> Option<Details> {
match self {
KindWithContent::DocumentImport { documents_count, .. } => {
Some(Details::DocumentAddition {
KindWithContent::DocumentAdditionOrUpdate { documents_count, .. } => {
Some(Details::DocumentAdditionOrUpdate {
received_documents: *documents_count,
indexed_documents: Some(0),
})
@ -237,8 +239,8 @@ impl KindWithContent {
KindWithContent::DocumentClear { .. } => {
Some(Details::ClearAll { deleted_documents: None })
}
KindWithContent::Settings { new_settings, .. } => {
Some(Details::Settings { settings: new_settings.clone() })
KindWithContent::SettingsUpdate { new_settings, .. } => {
Some(Details::SettingsUpdate { settings: new_settings.clone() })
}
KindWithContent::IndexDeletion { .. } => None,
KindWithContent::IndexCreation { primary_key, .. }
@ -267,16 +269,16 @@ impl KindWithContent {
impl From<&KindWithContent> for Option<Details> {
fn from(kind: &KindWithContent) -> Self {
match kind {
KindWithContent::DocumentImport { documents_count, .. } => {
Some(Details::DocumentAddition {
KindWithContent::DocumentAdditionOrUpdate { documents_count, .. } => {
Some(Details::DocumentAdditionOrUpdate {
received_documents: *documents_count,
indexed_documents: None,
})
}
KindWithContent::DocumentDeletion { .. } => None,
KindWithContent::DocumentClear { .. } => None,
KindWithContent::Settings { new_settings, .. } => {
Some(Details::Settings { settings: new_settings.clone() })
KindWithContent::SettingsUpdate { new_settings, .. } => {
Some(Details::SettingsUpdate { settings: new_settings.clone() })
}
KindWithContent::IndexDeletion { .. } => None,
KindWithContent::IndexCreation { primary_key, .. } => {
@ -359,10 +361,10 @@ impl FromStr for Status {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Sequence)]
#[serde(rename_all = "camelCase")]
pub enum Kind {
DocumentImport,
DocumentAdditionOrUpdate,
DocumentDeletion,
DocumentClear,
Settings,
SettingsUpdate,
IndexCreation,
IndexDeletion,
IndexUpdate,
@ -384,11 +386,11 @@ impl FromStr for Kind {
} else if kind.eq_ignore_ascii_case("indexDeletion") {
Ok(Kind::IndexDeletion)
} else if kind.eq_ignore_ascii_case("documentAdditionOrUpdate") {
Ok(Kind::DocumentImport)
Ok(Kind::DocumentAdditionOrUpdate)
} else if kind.eq_ignore_ascii_case("documentDeletion") {
Ok(Kind::DocumentDeletion)
} else if kind.eq_ignore_ascii_case("settingsUpdate") {
Ok(Kind::Settings)
Ok(Kind::SettingsUpdate)
} else if kind.eq_ignore_ascii_case("taskCancelation") {
Ok(Kind::TaskCancelation)
} else if kind.eq_ignore_ascii_case("taskDeletion") {
@ -418,11 +420,11 @@ impl FromStr for Kind {
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
#[allow(clippy::large_enum_variant)]
pub enum Details {
DocumentAddition {
DocumentAdditionOrUpdate {
received_documents: u64,
indexed_documents: Option<u64>,
},
Settings {
SettingsUpdate {
settings: Settings<Unchecked>,
},
IndexInfo {