mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-06-08 22:07:48 +02:00
Correctly batch tasks with different index creation rights
This commit is contained in:
Clément Renault
parent
87212cfd20
commit
068a4b2884
|
|
@ -10,6 +10,7 @@ pub enum BatchKind {
|
|||
},
|
||||
DocumentImport {
|
||||
method: IndexDocumentsMethod,
|
||||
allow_index_creation: bool,
|
||||
import_ids: Vec<TaskId>,
|
||||
},
|
||||
DocumentDeletion {
|
||||
|
|
@ -17,14 +18,17 @@ pub enum BatchKind {
|
|||
},
|
||||
ClearAndSettings {
|
||||
other: Vec<TaskId>,
|
||||
allow_index_creation: bool,
|
||||
settings_ids: Vec<TaskId>,
|
||||
},
|
||||
SettingsAndDocumentImport {
|
||||
settings_ids: Vec<TaskId>,
|
||||
method: IndexDocumentsMethod,
|
||||
allow_index_creation: bool,
|
||||
import_ids: Vec<TaskId>,
|
||||
},
|
||||
Settings {
|
||||
allow_index_creation: bool,
|
||||
settings_ids: Vec<TaskId>,
|
||||
},
|
||||
IndexDeletion {
|
||||
|
|
@ -50,18 +54,21 @@ impl BatchKind {
|
|||
Kind::IndexUpdate => Break(BatchKind::IndexUpdate { id: task_id }),
|
||||
Kind::IndexSwap => Break(BatchKind::IndexSwap { id: task_id }),
|
||||
Kind::DocumentClear => Continue(BatchKind::DocumentClear { ids: vec![task_id] }),
|
||||
Kind::DocumentAddition => Continue(BatchKind::DocumentImport {
|
||||
method: ReplaceDocuments,
|
||||
import_ids: vec![task_id],
|
||||
}),
|
||||
Kind::DocumentUpdate => Continue(BatchKind::DocumentImport {
|
||||
method: UpdateDocuments,
|
||||
Kind::DocumentImport {
|
||||
method,
|
||||
allow_index_creation,
|
||||
} => Continue(BatchKind::DocumentImport {
|
||||
method,
|
||||
allow_index_creation,
|
||||
import_ids: vec![task_id],
|
||||
}),
|
||||
Kind::DocumentDeletion => Continue(BatchKind::DocumentDeletion {
|
||||
deletion_ids: vec![task_id],
|
||||
}),
|
||||
Kind::Settings => Continue(BatchKind::Settings {
|
||||
Kind::Settings {
|
||||
allow_index_creation,
|
||||
} => Continue(BatchKind::Settings {
|
||||
allow_index_creation,
|
||||
settings_ids: vec![task_id],
|
||||
}),
|
||||
Kind::DumpExport | Kind::Snapshot | Kind::CancelTask => unreachable!(),
|
||||
|
|
@ -69,6 +76,7 @@ impl BatchKind {
|
|||
}
|
||||
|
||||
/// Returns a `ControlFlow::Break` if you must stop right now.
|
||||
#[rustfmt::skip]
|
||||
fn accumulate(self, id: TaskId, kind: Kind) -> ControlFlow<BatchKind, BatchKind> {
|
||||
match (self, kind) {
|
||||
// We don't batch any of these operations
|
||||
|
|
@ -76,31 +84,17 @@ impl BatchKind {
|
|||
// The index deletion can batch with everything but must stop after
|
||||
(
|
||||
BatchKind::DocumentClear { mut ids }
|
||||
| BatchKind::DocumentImport {
|
||||
method: _,
|
||||
import_ids: mut ids,
|
||||
}
|
||||
| BatchKind::DocumentDeletion {
|
||||
deletion_ids: mut ids,
|
||||
}
|
||||
| BatchKind::Settings {
|
||||
settings_ids: mut ids,
|
||||
},
|
||||
| BatchKind::DocumentDeletion { deletion_ids: mut ids }
|
||||
| BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids }
|
||||
| BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
|
||||
Kind::IndexDeletion,
|
||||
) => {
|
||||
ids.push(id);
|
||||
Break(BatchKind::IndexDeletion { ids })
|
||||
}
|
||||
(
|
||||
BatchKind::ClearAndSettings {
|
||||
settings_ids: mut ids,
|
||||
mut other,
|
||||
}
|
||||
| BatchKind::SettingsAndDocumentImport {
|
||||
import_ids: mut ids,
|
||||
method: _,
|
||||
settings_ids: mut other,
|
||||
},
|
||||
BatchKind::ClearAndSettings { settings_ids: mut ids, allow_index_creation: _, mut other }
|
||||
| BatchKind::SettingsAndDocumentImport { import_ids: mut ids, method: _, allow_index_creation: _, settings_ids: mut other },
|
||||
Kind::IndexDeletion,
|
||||
) => {
|
||||
ids.push(id);
|
||||
|
|
@ -117,110 +111,125 @@ impl BatchKind {
|
|||
}
|
||||
(
|
||||
this @ BatchKind::DocumentClear { .. },
|
||||
Kind::DocumentAddition | Kind::DocumentUpdate | Kind::Settings,
|
||||
Kind::DocumentImport { .. } | Kind::Settings { .. },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::DocumentImport {
|
||||
method: _,
|
||||
import_ids: mut ids,
|
||||
},
|
||||
BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids },
|
||||
Kind::DocumentClear,
|
||||
) => {
|
||||
ids.push(id);
|
||||
Continue(BatchKind::DocumentClear { ids })
|
||||
}
|
||||
|
||||
// We only want to batch together document imports that are allowed to create the index
|
||||
// or document imports not allowed to create an index if the first operation can.
|
||||
(
|
||||
this @ BatchKind::DocumentImport { method: _, allow_index_creation: false, .. },
|
||||
Kind::DocumentImport { method: _, allow_index_creation: true },
|
||||
) => Break(this),
|
||||
|
||||
// we can autobatch the same kind of document additions / updates
|
||||
(
|
||||
BatchKind::DocumentImport {
|
||||
method: ReplaceDocuments,
|
||||
mut import_ids,
|
||||
},
|
||||
Kind::DocumentAddition,
|
||||
BatchKind::DocumentImport { method: ReplaceDocuments, allow_index_creation, mut import_ids },
|
||||
Kind::DocumentImport { method: ReplaceDocuments, .. },
|
||||
) => {
|
||||
import_ids.push(id);
|
||||
Continue(BatchKind::DocumentImport {
|
||||
method: ReplaceDocuments,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
(
|
||||
BatchKind::DocumentImport {
|
||||
method: UpdateDocuments,
|
||||
mut import_ids,
|
||||
},
|
||||
Kind::DocumentUpdate,
|
||||
BatchKind::DocumentImport { method: UpdateDocuments, allow_index_creation, mut import_ids },
|
||||
Kind::DocumentImport { method: UpdateDocuments, .. },
|
||||
) => {
|
||||
import_ids.push(id);
|
||||
Continue(BatchKind::DocumentImport {
|
||||
method: UpdateDocuments,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
|
||||
// but we can't autobatch documents if it's not the same kind
|
||||
// this match branch MUST be AFTER the previous one
|
||||
(
|
||||
this @ BatchKind::DocumentImport { .. },
|
||||
Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate,
|
||||
Kind::DocumentDeletion | Kind::DocumentImport { .. },
|
||||
) => Break(this),
|
||||
(BatchKind::DocumentImport { method, import_ids }, Kind::Settings) => {
|
||||
Continue(BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids: vec![id],
|
||||
method,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
|
||||
// We only want to batch together document imports that are allowed to create the index
|
||||
// or document imports not allowed to create an index if the first operation can.
|
||||
(
|
||||
this @ BatchKind::DocumentImport { allow_index_creation: false, .. },
|
||||
Kind::Settings { allow_index_creation: true },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::DocumentImport { method, allow_index_creation, import_ids },
|
||||
Kind::Settings { .. },
|
||||
) => Continue(BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids: vec![id],
|
||||
method,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
}),
|
||||
|
||||
(BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentClear) => {
|
||||
deletion_ids.push(id);
|
||||
Continue(BatchKind::DocumentClear { ids: deletion_ids })
|
||||
}
|
||||
(
|
||||
this @ BatchKind::DocumentDeletion { .. },
|
||||
Kind::DocumentAddition | Kind::DocumentUpdate,
|
||||
) => Break(this),
|
||||
(this @ BatchKind::DocumentDeletion { .. }, Kind::DocumentImport { .. }) => Break(this),
|
||||
(BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentDeletion) => {
|
||||
deletion_ids.push(id);
|
||||
Continue(BatchKind::DocumentDeletion { deletion_ids })
|
||||
}
|
||||
(this @ BatchKind::DocumentDeletion { .. }, Kind::Settings) => Break(this),
|
||||
(this @ BatchKind::DocumentDeletion { .. }, Kind::Settings { .. }) => Break(this),
|
||||
|
||||
(BatchKind::Settings { settings_ids }, Kind::DocumentClear) => {
|
||||
Continue(BatchKind::ClearAndSettings {
|
||||
settings_ids: settings_ids,
|
||||
other: vec![id],
|
||||
})
|
||||
}
|
||||
(
|
||||
BatchKind::Settings { settings_ids, allow_index_creation },
|
||||
Kind::DocumentClear,
|
||||
) => Continue(BatchKind::ClearAndSettings {
|
||||
settings_ids: settings_ids,
|
||||
allow_index_creation,
|
||||
other: vec![id],
|
||||
}),
|
||||
(
|
||||
this @ BatchKind::Settings { .. },
|
||||
Kind::DocumentAddition | Kind::DocumentUpdate | Kind::DocumentDeletion,
|
||||
Kind::DocumentImport { .. } | Kind::DocumentDeletion,
|
||||
) => Break(this),
|
||||
(BatchKind::Settings { mut settings_ids }, Kind::Settings) => {
|
||||
(
|
||||
this @ BatchKind::Settings { allow_index_creation: false, .. },
|
||||
Kind::Settings { allow_index_creation: true },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::Settings { mut settings_ids, allow_index_creation },
|
||||
Kind::Settings { .. },
|
||||
) => {
|
||||
settings_ids.push(id);
|
||||
Continue(BatchKind::Settings { settings_ids })
|
||||
Continue(BatchKind::Settings {
|
||||
allow_index_creation,
|
||||
settings_ids,
|
||||
})
|
||||
}
|
||||
|
||||
(
|
||||
BatchKind::ClearAndSettings {
|
||||
mut other,
|
||||
settings_ids,
|
||||
},
|
||||
BatchKind::ClearAndSettings { mut other, settings_ids, allow_index_creation },
|
||||
Kind::DocumentClear,
|
||||
) => {
|
||||
other.push(id);
|
||||
Continue(BatchKind::ClearAndSettings {
|
||||
other,
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
})
|
||||
}
|
||||
(
|
||||
this @ BatchKind::ClearAndSettings { .. },
|
||||
Kind::DocumentAddition | Kind::DocumentUpdate,
|
||||
) => Break(this),
|
||||
(this @ BatchKind::ClearAndSettings { .. }, Kind::DocumentImport { .. }) => Break(this),
|
||||
(
|
||||
BatchKind::ClearAndSettings {
|
||||
mut other,
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
},
|
||||
Kind::DocumentDeletion,
|
||||
) => {
|
||||
|
|
@ -228,64 +237,64 @@ impl BatchKind {
|
|||
Continue(BatchKind::ClearAndSettings {
|
||||
other,
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
})
|
||||
}
|
||||
(
|
||||
BatchKind::ClearAndSettings {
|
||||
mut settings_ids,
|
||||
other,
|
||||
this @ BatchKind::ClearAndSettings { allow_index_creation: false, .. },
|
||||
Kind::Settings {
|
||||
allow_index_creation: true,
|
||||
},
|
||||
Kind::Settings,
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::ClearAndSettings { mut settings_ids, other, allow_index_creation },
|
||||
Kind::Settings { .. },
|
||||
) => {
|
||||
settings_ids.push(id);
|
||||
Continue(BatchKind::ClearAndSettings {
|
||||
other,
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
})
|
||||
}
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method: _,
|
||||
import_ids: mut other,
|
||||
},
|
||||
BatchKind::SettingsAndDocumentImport { settings_ids, method: _, import_ids: mut other, allow_index_creation },
|
||||
Kind::DocumentClear,
|
||||
) => {
|
||||
other.push(id);
|
||||
Continue(BatchKind::ClearAndSettings {
|
||||
settings_ids,
|
||||
other,
|
||||
allow_index_creation,
|
||||
})
|
||||
}
|
||||
|
||||
// we can batch the settings with a kind of document operation with the same kind of document operation
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method: ReplaceDocuments,
|
||||
mut import_ids,
|
||||
},
|
||||
Kind::DocumentAddition,
|
||||
this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. },
|
||||
Kind::DocumentImport { allow_index_creation: true, .. },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport { settings_ids, method: ReplaceDocuments, mut import_ids, allow_index_creation },
|
||||
Kind::DocumentImport { method: ReplaceDocuments, .. },
|
||||
) => {
|
||||
import_ids.push(id);
|
||||
Continue(BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method: ReplaceDocuments,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method: UpdateDocuments,
|
||||
mut import_ids,
|
||||
},
|
||||
Kind::DocumentUpdate,
|
||||
BatchKind::SettingsAndDocumentImport { settings_ids, method: UpdateDocuments, allow_index_creation, mut import_ids },
|
||||
Kind::DocumentImport { method: UpdateDocuments, .. },
|
||||
) => {
|
||||
import_ids.push(id);
|
||||
Continue(BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method: UpdateDocuments,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
|
|
@ -293,20 +302,21 @@ impl BatchKind {
|
|||
// this MUST be AFTER the two previous branch
|
||||
(
|
||||
this @ BatchKind::SettingsAndDocumentImport { .. },
|
||||
Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate,
|
||||
Kind::DocumentDeletion | Kind::DocumentImport { .. },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport {
|
||||
mut settings_ids,
|
||||
method,
|
||||
import_ids,
|
||||
},
|
||||
Kind::Settings,
|
||||
this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. },
|
||||
Kind::Settings { allow_index_creation: true },
|
||||
) => Break(this),
|
||||
(
|
||||
BatchKind::SettingsAndDocumentImport { mut settings_ids, method, allow_index_creation, import_ids },
|
||||
Kind::Settings { .. },
|
||||
) => {
|
||||
settings_ids.push(id);
|
||||
Continue(BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
})
|
||||
}
|
||||
|
|
@ -362,119 +372,129 @@ mod tests {
|
|||
#[test]
|
||||
fn autobatch_simple_operation_together() {
|
||||
// we can autobatch one or multiple DocumentAddition together
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple DocumentUpdate together
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple DocumentDeletion together
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentDeletion, DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple Settings together
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings]), @"Some(Settings { settings_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings, Settings, Settings]), @"Some(Settings { settings_ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_document_operation_dont_autobatch_with_other() {
|
||||
// addition, updates and deletion can't batch together
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentAddition]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentAddition]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexCreation]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexSwap]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_addition_batch_with_settings() {
|
||||
// simple case
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
|
||||
// multiple settings and doc addition
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
|
||||
// addition and setting unordered
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, import_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, import_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 2] })");
|
||||
|
||||
// We ensure this kind of batch doesn't batch with forbidden operations
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentAddition]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_and_additions() {
|
||||
// these two doesn't need to batch
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
|
||||
|
||||
// Basic use case
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
|
||||
// This batch kind doesn't mix with other document addition
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
|
||||
// But you can batch multiple clear together
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_and_additions_and_settings() {
|
||||
// A clear don't need to autobatch the settings that happens AFTER there is no documents
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings]), @"Some(DocumentClear { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings { allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings, DocumentClear, Settings]), @"Some(ClearAndSettings { other: [1], settings_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, DocumentClear, Settings { allow_index_creation: true }]), @"Some(ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anything_and_index_deletion() {
|
||||
// The indexdeletion doesn't batch with anything that happens AFTER
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentAddition]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentUpdate]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentDeletion]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentClear]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings]), @"Some(IndexDeletion { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings { allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
|
||||
|
||||
// The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion
|
||||
// First, the basic cases
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
|
||||
// Then the mixed cases
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allowed_and_disallowed_index_creation() {
|
||||
// DocumentImport that can create indexes can't be mixed with those disallowed to do so
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: false }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, Settings { allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use crate::{
|
|||
Error, IndexScheduler, Result, TaskId,
|
||||
};
|
||||
use index::apply_settings_to_builder;
|
||||
use index::error::{IndexError, MilliError};
|
||||
use index::error::IndexError;
|
||||
use index::{Settings, Unchecked};
|
||||
use log::{debug, info};
|
||||
use milli::documents::DocumentsBatchReader;
|
||||
|
|
@ -39,6 +39,7 @@ pub(crate) enum IndexOperation {
|
|||
index_uid: String,
|
||||
primary_key: Option<String>,
|
||||
method: IndexDocumentsMethod,
|
||||
allow_index_creation: bool,
|
||||
documents_counts: Vec<u64>,
|
||||
content_files: Vec<Uuid>,
|
||||
tasks: Vec<Task>,
|
||||
|
|
@ -56,6 +57,7 @@ pub(crate) enum IndexOperation {
|
|||
index_uid: String,
|
||||
// TODO what's that boolean, does it mean that it removes things or what?
|
||||
settings: Vec<(bool, Settings<Unchecked>)>,
|
||||
allow_index_creation: bool,
|
||||
tasks: Vec<Task>,
|
||||
},
|
||||
DocumentClearAndSetting {
|
||||
|
|
@ -64,6 +66,7 @@ pub(crate) enum IndexOperation {
|
|||
|
||||
// TODO what's that boolean, does it mean that it removes things or what?
|
||||
settings: Vec<(bool, Settings<Unchecked>)>,
|
||||
allow_index_creation: bool,
|
||||
settings_tasks: Vec<Task>,
|
||||
},
|
||||
SettingsAndDocumentImport {
|
||||
|
|
@ -71,6 +74,7 @@ pub(crate) enum IndexOperation {
|
|||
|
||||
primary_key: Option<String>,
|
||||
method: IndexDocumentsMethod,
|
||||
allow_index_creation: bool,
|
||||
documents_counts: Vec<u64>,
|
||||
content_files: Vec<Uuid>,
|
||||
document_import_tasks: Vec<Task>,
|
||||
|
|
@ -126,7 +130,11 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
})))
|
||||
}
|
||||
BatchKind::DocumentImport { method, import_ids } => {
|
||||
BatchKind::DocumentImport {
|
||||
method,
|
||||
import_ids,
|
||||
allow_index_creation,
|
||||
} => {
|
||||
let tasks = self.get_existing_tasks(rtxn, import_ids)?;
|
||||
let primary_key = match &tasks[0].kind {
|
||||
KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(),
|
||||
|
|
@ -154,6 +162,7 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
primary_key,
|
||||
method,
|
||||
allow_index_creation,
|
||||
documents_counts,
|
||||
content_files,
|
||||
tasks,
|
||||
|
|
@ -181,7 +190,10 @@ impl IndexScheduler {
|
|||
},
|
||||
)))
|
||||
}
|
||||
BatchKind::Settings { settings_ids } => {
|
||||
BatchKind::Settings {
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
} => {
|
||||
let tasks = self.get_existing_tasks(rtxn, settings_ids)?;
|
||||
|
||||
let mut settings = Vec::new();
|
||||
|
|
@ -199,20 +211,30 @@ impl IndexScheduler {
|
|||
Ok(Some(Batch::IndexOperation(IndexOperation::Settings {
|
||||
index_uid,
|
||||
settings,
|
||||
allow_index_creation,
|
||||
tasks,
|
||||
})))
|
||||
}
|
||||
BatchKind::ClearAndSettings {
|
||||
other,
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
} => {
|
||||
let (index_uid, settings, settings_tasks) = match self
|
||||
.create_next_batch_index(rtxn, index_uid, BatchKind::Settings { settings_ids })?
|
||||
.create_next_batch_index(
|
||||
rtxn,
|
||||
index_uid,
|
||||
BatchKind::Settings {
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
},
|
||||
)?
|
||||
.unwrap()
|
||||
{
|
||||
Batch::IndexOperation(IndexOperation::Settings {
|
||||
index_uid,
|
||||
settings,
|
||||
allow_index_creation,
|
||||
tasks,
|
||||
}) => (index_uid, settings, tasks),
|
||||
_ => unreachable!(),
|
||||
|
|
@ -235,6 +257,7 @@ impl IndexScheduler {
|
|||
IndexOperation::DocumentClearAndSetting {
|
||||
index_uid,
|
||||
cleared_tasks,
|
||||
allow_index_creation,
|
||||
settings,
|
||||
settings_tasks,
|
||||
},
|
||||
|
|
@ -243,18 +266,26 @@ impl IndexScheduler {
|
|||
BatchKind::SettingsAndDocumentImport {
|
||||
settings_ids,
|
||||
method,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
} => {
|
||||
let settings = self.create_next_batch_index(
|
||||
rtxn,
|
||||
index_uid.clone(),
|
||||
BatchKind::Settings { settings_ids },
|
||||
BatchKind::Settings {
|
||||
settings_ids,
|
||||
allow_index_creation,
|
||||
},
|
||||
)?;
|
||||
|
||||
let document_import = self.create_next_batch_index(
|
||||
rtxn,
|
||||
index_uid.clone(),
|
||||
BatchKind::DocumentImport { method, import_ids },
|
||||
BatchKind::DocumentImport {
|
||||
method,
|
||||
allow_index_creation,
|
||||
import_ids,
|
||||
},
|
||||
)?;
|
||||
|
||||
match (document_import, settings) {
|
||||
|
|
@ -276,6 +307,7 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
primary_key,
|
||||
method,
|
||||
allow_index_creation,
|
||||
documents_counts,
|
||||
content_files,
|
||||
document_import_tasks,
|
||||
|
|
@ -521,6 +553,7 @@ impl IndexScheduler {
|
|||
index_uid: _,
|
||||
primary_key,
|
||||
method,
|
||||
allow_index_creation,
|
||||
documents_counts,
|
||||
content_files,
|
||||
mut tasks,
|
||||
|
|
@ -628,6 +661,7 @@ impl IndexScheduler {
|
|||
IndexOperation::Settings {
|
||||
index_uid: _,
|
||||
settings,
|
||||
allow_index_creation,
|
||||
mut tasks,
|
||||
} => {
|
||||
let indexer_config = self.index_mapper.indexer_config();
|
||||
|
|
@ -652,6 +686,7 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
primary_key,
|
||||
method,
|
||||
allow_index_creation,
|
||||
documents_counts,
|
||||
content_files,
|
||||
document_import_tasks,
|
||||
|
|
@ -664,6 +699,7 @@ impl IndexScheduler {
|
|||
IndexOperation::Settings {
|
||||
index_uid: index_uid.clone(),
|
||||
settings,
|
||||
allow_index_creation,
|
||||
tasks: settings_tasks,
|
||||
},
|
||||
)?;
|
||||
|
|
@ -675,6 +711,7 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
primary_key,
|
||||
method,
|
||||
allow_index_creation,
|
||||
documents_counts,
|
||||
content_files,
|
||||
tasks: document_import_tasks,
|
||||
|
|
@ -689,6 +726,7 @@ impl IndexScheduler {
|
|||
index_uid,
|
||||
cleared_tasks,
|
||||
settings,
|
||||
allow_index_creation,
|
||||
settings_tasks,
|
||||
} => {
|
||||
let mut import_tasks = self.apply_index_operation(
|
||||
|
|
@ -706,6 +744,7 @@ impl IndexScheduler {
|
|||
IndexOperation::Settings {
|
||||
index_uid,
|
||||
settings,
|
||||
allow_index_creation,
|
||||
tasks: settings_tasks,
|
||||
},
|
||||
)?;
|
||||
|
|
|
|||
|
|
@ -446,7 +446,7 @@ impl IndexScheduler {
|
|||
mod tests {
|
||||
use big_s::S;
|
||||
use insta::*;
|
||||
use milli::update::IndexDocumentsMethod::{self, ReplaceDocuments, UpdateDocuments};
|
||||
use milli::update::IndexDocumentsMethod::ReplaceDocuments;
|
||||
use tempfile::TempDir;
|
||||
use uuid::Uuid;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ pub struct TaskView {
|
|||
pub uid: TaskId,
|
||||
pub index_uid: Option<String>,
|
||||
pub status: Status,
|
||||
// TODO use our own Kind for the user
|
||||
#[serde(rename = "type")]
|
||||
pub kind: Kind,
|
||||
|
||||
|
|
@ -175,17 +176,21 @@ impl KindWithContent {
|
|||
pub fn as_kind(&self) -> Kind {
|
||||
match self {
|
||||
KindWithContent::DocumentImport {
|
||||
method: IndexDocumentsMethod::ReplaceDocuments,
|
||||
method,
|
||||
allow_index_creation,
|
||||
..
|
||||
} => Kind::DocumentAddition,
|
||||
KindWithContent::DocumentImport {
|
||||
method: IndexDocumentsMethod::UpdateDocuments,
|
||||
..
|
||||
} => Kind::DocumentUpdate,
|
||||
KindWithContent::DocumentImport { .. } => unreachable!(),
|
||||
} => Kind::DocumentImport {
|
||||
method: *method,
|
||||
allow_index_creation: *allow_index_creation,
|
||||
},
|
||||
KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion,
|
||||
KindWithContent::DocumentClear { .. } => Kind::DocumentClear,
|
||||
KindWithContent::Settings { .. } => Kind::Settings,
|
||||
KindWithContent::Settings {
|
||||
allow_index_creation,
|
||||
..
|
||||
} => Kind::Settings {
|
||||
allow_index_creation: *allow_index_creation,
|
||||
},
|
||||
KindWithContent::IndexCreation { .. } => Kind::IndexCreation,
|
||||
KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion,
|
||||
KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate,
|
||||
|
|
@ -299,11 +304,15 @@ impl KindWithContent {
|
|||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Kind {
|
||||
DocumentAddition,
|
||||
DocumentUpdate,
|
||||
DocumentImport {
|
||||
method: IndexDocumentsMethod,
|
||||
allow_index_creation: bool,
|
||||
},
|
||||
DocumentDeletion,
|
||||
DocumentClear,
|
||||
Settings,
|
||||
Settings {
|
||||
allow_index_creation: bool,
|
||||
},
|
||||
IndexCreation,
|
||||
IndexDeletion,
|
||||
IndexUpdate,
|
||||
|
|
@ -318,11 +327,22 @@ impl FromStr for Kind {
|
|||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"document_addition" => Ok(Kind::DocumentAddition),
|
||||
"document_update" => Ok(Kind::DocumentUpdate),
|
||||
"document_addition" => Ok(Kind::DocumentImport {
|
||||
method: IndexDocumentsMethod::ReplaceDocuments,
|
||||
// TODO this doesn't make sense
|
||||
allow_index_creation: false,
|
||||
}),
|
||||
"document_update" => Ok(Kind::DocumentImport {
|
||||
method: IndexDocumentsMethod::UpdateDocuments,
|
||||
// TODO this doesn't make sense
|
||||
allow_index_creation: false,
|
||||
}),
|
||||
"document_deletion" => Ok(Kind::DocumentDeletion),
|
||||
"document_clear" => Ok(Kind::DocumentClear),
|
||||
"settings" => Ok(Kind::Settings),
|
||||
"settings" => Ok(Kind::Settings {
|
||||
// TODO this doesn't make sense
|
||||
allow_index_creation: false,
|
||||
}),
|
||||
"index_creation" => Ok(Kind::IndexCreation),
|
||||
"index_deletion" => Ok(Kind::IndexDeletion),
|
||||
"index_update" => Ok(Kind::IndexUpdate),
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user