fix the import of the dumpv4&v5 when there is no instance-uid + rename the Kind+KindWithContent+Details variant for the DocumentImport and the Setting

This commit is contained in:
Irevoire 2022-10-21 18:03:10 +02:00 committed by Tamo
parent 5bb9023722
commit d065a3e4f0
14 changed files with 98 additions and 77 deletions

View file

@ -43,12 +43,12 @@ impl AutobatchKind {
impl From<KindWithContent> for AutobatchKind {
fn from(kind: KindWithContent) -> Self {
match kind {
KindWithContent::DocumentImport { method, allow_index_creation, .. } => {
KindWithContent::DocumentAdditionOrUpdate { method, allow_index_creation, .. } => {
AutobatchKind::DocumentImport { method, allow_index_creation }
}
KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
KindWithContent::Settings { allow_index_creation, is_deletion, .. } => {
KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
AutobatchKind::Settings {
allow_index_creation: allow_index_creation && !is_deletion,
}
@ -449,7 +449,7 @@ mod tests {
}
fn doc_imp(method: IndexDocumentsMethod, allow_index_creation: bool) -> KindWithContent {
KindWithContent::DocumentImport {
KindWithContent::DocumentAdditionOrUpdate {
index_uid: String::from("doggo"),
primary_key: None,
method,
@ -471,7 +471,7 @@ mod tests {
}
fn settings(allow_index_creation: bool) -> KindWithContent {
KindWithContent::Settings {
KindWithContent::SettingsUpdate {
index_uid: String::from("doggo"),
new_settings: Default::default(),
is_deletion: false,

View file

@ -181,7 +181,9 @@ impl IndexScheduler {
BatchKind::DocumentImport { method, import_ids, .. } => {
let tasks = self.get_existing_tasks(rtxn, import_ids)?;
let primary_key = match &tasks[0].kind {
KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(),
KindWithContent::DocumentAdditionOrUpdate { primary_key, .. } => {
primary_key.clone()
}
_ => unreachable!(),
};
@ -189,8 +191,10 @@ impl IndexScheduler {
let mut content_files = Vec::new();
for task in &tasks {
match task.kind {
KindWithContent::DocumentImport {
content_file, documents_count, ..
KindWithContent::DocumentAdditionOrUpdate {
content_file,
documents_count,
..
} => {
documents_counts.push(documents_count);
content_files.push(content_file);
@ -235,9 +239,9 @@ impl IndexScheduler {
let mut settings = Vec::new();
for task in &tasks {
match task.kind {
KindWithContent::Settings { ref new_settings, is_deletion, .. } => {
settings.push((is_deletion, new_settings.clone()))
}
KindWithContent::SettingsUpdate {
ref new_settings, is_deletion, ..
} => settings.push((is_deletion, new_settings.clone())),
_ => unreachable!(),
}
}
@ -873,14 +877,14 @@ impl IndexScheduler {
match ret {
Ok(DocumentAdditionResult { indexed_documents, number_of_documents }) => {
task.status = Status::Succeeded;
task.details = Some(Details::DocumentAddition {
task.details = Some(Details::DocumentAdditionOrUpdate {
received_documents: number_of_documents,
indexed_documents: Some(indexed_documents),
});
}
Err(error) => {
task.status = Status::Failed;
task.details = Some(Details::DocumentAddition {
task.details = Some(Details::DocumentAdditionOrUpdate {
received_documents: count,
indexed_documents: Some(count),
});
@ -914,7 +918,7 @@ impl IndexScheduler {
// TODO merge the settings to only do *one* reindexation.
for (task, (_, settings)) in tasks.iter_mut().zip(settings) {
let checked_settings = settings.clone().check();
task.details = Some(Details::Settings { settings });
task.details = Some(Details::SettingsUpdate { settings });
let mut builder =
milli::update::Settings::new(index_wtxn, index, indexer_config);

View file

@ -562,7 +562,7 @@ impl IndexScheduler {
method,
documents_count,
allow_index_creation,
} => KindWithContent::DocumentImport {
} => KindWithContent::DocumentAdditionOrUpdate {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
primary_key,
method,
@ -578,7 +578,7 @@ impl IndexScheduler {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
},
KindDump::Settings { settings, is_deletion, allow_index_creation } => {
KindWithContent::Settings {
KindWithContent::SettingsUpdate {
index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
new_settings: settings,
is_deletion,
@ -788,7 +788,7 @@ mod tests {
content_file_uuid: u128,
documents_count: u64,
) -> KindWithContent {
KindWithContent::DocumentImport {
KindWithContent::DocumentAdditionOrUpdate {
index_uid: S(index),
primary_key: primary_key.map(ToOwned::to_owned),
method: ReplaceDocuments,
@ -1102,7 +1102,7 @@ mod tests {
meilisearch_types::document_formats::read_json(content.as_bytes(), file.as_file_mut())
.unwrap() as u64;
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,
@ -1144,7 +1144,7 @@ mod tests {
.unwrap() as u64;
file.persist().unwrap();
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,
@ -1255,7 +1255,7 @@ mod tests {
.unwrap() as u64;
file.persist().unwrap();
index_scheduler
.register(KindWithContent::DocumentImport {
.register(KindWithContent::DocumentAdditionOrUpdate {
index_uid: S("doggos"),
primary_key: Some(S("id")),
method: ReplaceDocuments,

View file

@ -147,13 +147,13 @@ fn snapshot_task(task: &Task) -> String {
fn snaphsot_details(d: &Details) -> String {
match d {
Details::DocumentAddition {
Details::DocumentAdditionOrUpdate {
received_documents,
indexed_documents,
} => {
format!("{{ received_documents: {received_documents}, indexed_documents: {indexed_documents:?} }}")
}
Details::Settings { settings } => {
Details::SettingsUpdate { settings } => {
format!("{{ settings: {settings:?} }}")
}
Details::IndexInfo { primary_key } => {

View file

@ -236,10 +236,10 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
use KindWithContent as K;
let mut index_uids = vec![];
match &mut task.kind {
K::DocumentImport { index_uid, .. } => index_uids.push(index_uid),
K::DocumentAdditionOrUpdate { index_uid, .. } => index_uids.push(index_uid),
K::DocumentDeletion { index_uid, .. } => index_uids.push(index_uid),
K::DocumentClear { index_uid } => index_uids.push(index_uid),
K::Settings { index_uid, .. } => index_uids.push(index_uid),
K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid),
K::IndexDeletion { index_uid } => index_uids.push(index_uid),
K::IndexCreation { index_uid, .. } => index_uids.push(index_uid),
K::IndexUpdate { index_uid, .. } => index_uids.push(index_uid),