mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-03 03:47:02 +02:00
Continue implementation of task deletion
1. Matched tasks are a roaring bitmap 2. Start implementation in meilisearch-http 3. Snapshots use meili-snap 4. Rename to TaskDeletion
This commit is contained in:
Loïc Lecrenier
Clément Renault
parent
e4d461ecba
commit
9522b75454
27 changed files with 290 additions and 456 deletions
|
|
@ -12,7 +12,7 @@ csv = "1.1.6"
|
|||
file-store = { path = "../file-store" }
|
||||
log = "0.4.14"
|
||||
meilisearch-types = { path = "../meilisearch-types" }
|
||||
roaring = "0.9.0"
|
||||
roaring = { version = "0.10.0", features = ["serde"] }
|
||||
serde = { version = "1.0.136", features = ["derive"] }
|
||||
serde_json = { version = "1.0.85", features = ["preserve_order"] }
|
||||
tempfile = "3.3.0"
|
||||
|
|
@ -27,3 +27,4 @@ crossbeam = "0.8.2"
|
|||
nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}
|
||||
insta = { version = "1.19.1", features = ["json", "redactions"] }
|
||||
big_s = "1.0.2"
|
||||
meili-snap = { path = "../meili-snap" }
|
||||
|
|
@ -23,7 +23,7 @@ enum AutobatchKind {
|
|||
IndexUpdate,
|
||||
IndexSwap,
|
||||
CancelTask,
|
||||
DeleteTasks,
|
||||
TaskDeletion,
|
||||
DumpExport,
|
||||
Snapshot,
|
||||
}
|
||||
|
|
@ -63,7 +63,7 @@ impl From<KindWithContent> for AutobatchKind {
|
|||
KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate,
|
||||
KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap,
|
||||
KindWithContent::CancelTask { .. } => AutobatchKind::CancelTask,
|
||||
KindWithContent::DeleteTasks { .. } => AutobatchKind::DeleteTasks,
|
||||
KindWithContent::TaskDeletion { .. } => AutobatchKind::TaskDeletion,
|
||||
KindWithContent::DumpExport { .. } => AutobatchKind::DumpExport,
|
||||
KindWithContent::Snapshot => AutobatchKind::Snapshot,
|
||||
}
|
||||
|
|
@ -153,7 +153,7 @@ impl BatchKind {
|
|||
allow_index_creation,
|
||||
settings_ids: vec![task_id],
|
||||
}),
|
||||
K::DumpExport | K::Snapshot | K::CancelTask | K::DeleteTasks => {
|
||||
K::DumpExport | K::Snapshot | K::CancelTask | K::TaskDeletion => {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
|
@ -378,7 +378,7 @@ impl BatchKind {
|
|||
import_ids,
|
||||
})
|
||||
}
|
||||
(_, K::CancelTask | K::DeleteTasks | K::DumpExport | K::Snapshot) => {
|
||||
(_, K::CancelTask | K::TaskDeletion | K::DumpExport | K::Snapshot) => {
|
||||
unreachable!()
|
||||
}
|
||||
(
|
||||
|
|
@ -414,7 +414,7 @@ pub fn autobatch(enqueued: Vec<(TaskId, KindWithContent)>) -> Option<BatchKind>
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::assert_smol_debug_snapshot;
|
||||
use crate::debug_snapshot;
|
||||
|
||||
use super::*;
|
||||
use uuid::Uuid;
|
||||
|
|
@ -492,129 +492,129 @@ mod tests {
|
|||
#[test]
|
||||
fn autobatch_simple_operation_together() {
|
||||
// we can autobatch one or multiple DocumentAddition together
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp( ReplaceDocuments, true ), doc_imp(ReplaceDocuments, true )]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp( ReplaceDocuments, true ), doc_imp(ReplaceDocuments, true )]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple DocumentUpdate together
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple DocumentDeletion together
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_del(), doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), doc_del(), doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })");
|
||||
// we can autobatch one or multiple Settings together
|
||||
assert_smol_debug_snapshot!(autobatch_from([settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([settings(true), settings(true), settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([settings(true), settings(true), settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_document_operation_dont_autobatch_with_other() {
|
||||
// addition, updates and deletion can't batch together
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_del()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_del()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_del()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_del()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_create()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_create()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_create()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_create()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_create()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), idx_create()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_update()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_update()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_update()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_update()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_update()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), idx_update()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_swap()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_swap()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_swap()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_swap()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_swap()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), idx_swap()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_addition_batch_with_settings() {
|
||||
// simple case
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
|
||||
// multiple settings and doc addition
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
|
||||
// addition and setting unordered
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(UpdateDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(UpdateDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 2] })");
|
||||
|
||||
// We ensure this kind of batch doesn't batch with forbidden operations
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(UpdateDocuments, true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(ReplaceDocuments, true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_del()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_del()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_create()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_create()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_update()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_update()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_swap()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_swap()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(UpdateDocuments, true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(ReplaceDocuments, true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_del()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_del()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_create()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_create()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_update()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_update()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_swap()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_swap()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_and_additions() {
|
||||
// these two doesn't need to batch
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(doc_clr() { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(doc_clr() { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentClear { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentClear { ids: [0] })");
|
||||
|
||||
// Basic use case
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
|
||||
// This batch kind doesn't mix with other document addition
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(doc_clr() { ids: [0, 1, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(doc_clr() { ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentClear { ids: [0, 1, 2] })");
|
||||
|
||||
// But you can batch multiple clear together
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2, 3, 4] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2, 3, 4] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_and_additions_and_settings() {
|
||||
// A clear don't need to autobatch the settings that happens AFTER there is no documents
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), settings(true)]), @"Some(doc_clr() { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_clr(), settings(true)]), @"Some(DocumentClear { ids: [0] })");
|
||||
|
||||
assert_smol_debug_snapshot!(autobatch_from([settings(true), doc_clr(), settings(true)]), @"Some(clearAndSettings([1) allow_index_creation: true, settings_ids: [0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr()]), @"Some(clearAndSettings([0)2], allow_index_creation: true, settings_ids: [1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr()]), @"Some(clearAndSettings([0)2], allow_index_creation: true, settings_ids: [1] })");
|
||||
debug_snapshot!(autobatch_from([settings(true), doc_clr(), settings(true)]), @"Some(ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr()]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr()]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anything_and_index_deletion() {
|
||||
// The indexdeletion doesn't batch with anything that happens AFTER
|
||||
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_imp(ReplaceDocuments, true)]), @"Some(idx_del() { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_imp(UpdateDocuments, true)]), @"Some(idx_del() { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_del()]), @"Some(idx_del() { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_clr()]), @"Some(idx_del() { ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([idx_del(), settings(true)]), @"Some(idx_del() { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([idx_del(), doc_imp(ReplaceDocuments, true)]), @"Some(IndexDeletion { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([idx_del(), doc_imp(UpdateDocuments, true)]), @"Some(IndexDeletion { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([idx_del(), doc_del()]), @"Some(IndexDeletion { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([idx_del(), doc_clr()]), @"Some(IndexDeletion { ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([idx_del(), settings(true)]), @"Some(IndexDeletion { ids: [0] })");
|
||||
|
||||
// The index deletion can accept almost any type of BatchKind and transform it to an idx_del()
|
||||
// First, the basic cases
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_del()]), @"Some(idx_del() { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), idx_del()]), @"Some(idx_del() { ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_del(), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
|
||||
|
||||
// Then the mixed cases
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 2, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(idx_del() { ids: [1, 3, 0, 2] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(idx_del() { ids: [1, 3, 0, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allowed_and_disallowed_index_creation() {
|
||||
// doc_imp(indexes canbe)ixed with those disallowed to do so
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, true)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(doc_imp(ReplaceDocuments, true)import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, false)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0, 1] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_imp(: ReplaceDocuments: true)import_ids: [0] })");
|
||||
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), settings(true)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, false)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0, 1] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
|
||||
debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), settings(true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use std::collections::HashSet;
|
||||
|
||||
use crate::{autobatcher::BatchKind, Error, IndexScheduler, Result, TaskId};
|
||||
|
||||
use meilisearch_types::tasks::{Details, Kind, KindWithContent, Status, Task};
|
||||
|
|
@ -20,7 +22,7 @@ use uuid::Uuid;
|
|||
|
||||
pub(crate) enum Batch {
|
||||
Cancel(Task),
|
||||
DeleteTasks(Task),
|
||||
TaskDeletion(Task),
|
||||
Snapshot(Vec<Task>),
|
||||
Dump(Vec<Task>),
|
||||
IndexOperation(IndexOperation),
|
||||
|
|
@ -95,7 +97,7 @@ impl Batch {
|
|||
pub fn ids(&self) -> Vec<TaskId> {
|
||||
match self {
|
||||
Batch::Cancel(task)
|
||||
| Batch::DeleteTasks(task)
|
||||
| Batch::TaskDeletion(task)
|
||||
| Batch::IndexCreation { task, .. }
|
||||
| Batch::IndexUpdate { task, .. } => vec![task.uid],
|
||||
Batch::Snapshot(tasks) | Batch::Dump(tasks) | Batch::IndexDeletion { tasks, .. } => {
|
||||
|
|
@ -379,13 +381,13 @@ impl IndexScheduler {
|
|||
}
|
||||
|
||||
// 2. we get the next task to delete
|
||||
let to_delete = self.get_kind(rtxn, Kind::DeleteTasks)?;
|
||||
let to_delete = self.get_kind(rtxn, Kind::TaskDeletion)?;
|
||||
if let Some(task_id) = to_delete.min() {
|
||||
let task = self
|
||||
.get_task(rtxn, task_id)?
|
||||
.ok_or(Error::CorruptedTaskQueue)?;
|
||||
|
||||
return Ok(Some(Batch::DeleteTasks(task)));
|
||||
return Ok(Some(Batch::TaskDeletion(task)));
|
||||
}
|
||||
|
||||
// 3. we batch the snapshot.
|
||||
|
|
@ -445,10 +447,10 @@ impl IndexScheduler {
|
|||
pub(crate) fn process_batch(&self, batch: Batch) -> Result<Vec<Task>> {
|
||||
match batch {
|
||||
Batch::Cancel(_) => todo!(),
|
||||
Batch::DeleteTasks(mut task) => {
|
||||
Batch::TaskDeletion(mut task) => {
|
||||
// 1. Retrieve the tasks that matched the query at enqueue-time.
|
||||
let matched_tasks =
|
||||
if let KindWithContent::DeleteTasks { tasks, query: _ } = &task.kind {
|
||||
if let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind {
|
||||
tasks
|
||||
} else {
|
||||
unreachable!()
|
||||
|
|
@ -459,7 +461,7 @@ impl IndexScheduler {
|
|||
|
||||
task.status = Status::Succeeded;
|
||||
match &mut task.details {
|
||||
Some(Details::DeleteTasks {
|
||||
Some(Details::TaskDeletion {
|
||||
matched_tasks: _,
|
||||
deleted_tasks,
|
||||
original_query: _,
|
||||
|
|
@ -816,49 +818,52 @@ impl IndexScheduler {
|
|||
/// Delete each given task from all the databases (if it is deleteable).
|
||||
///
|
||||
/// Return the number of tasks that were actually deleted
|
||||
fn delete_matched_tasks(&self, wtxn: &mut RwTxn, matched_tasks: &[u32]) -> Result<usize> {
|
||||
fn delete_matched_tasks(
|
||||
&self,
|
||||
wtxn: &mut RwTxn,
|
||||
matched_tasks: &RoaringBitmap,
|
||||
) -> Result<usize> {
|
||||
// 1. Remove from this list the tasks that we are not allowed to delete
|
||||
let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?;
|
||||
|
||||
let processing_tasks = &self.processing_tasks.read().unwrap().1;
|
||||
let to_delete_tasks = matched_tasks
|
||||
.iter()
|
||||
.filter(|&&task_id| {
|
||||
!processing_tasks.contains(task_id) && !enqueued_tasks.contains(task_id)
|
||||
})
|
||||
.copied();
|
||||
let to_delete_tasks = RoaringBitmap::from_iter(to_delete_tasks);
|
||||
|
||||
let mut to_delete_tasks = matched_tasks - processing_tasks;
|
||||
to_delete_tasks -= enqueued_tasks;
|
||||
|
||||
// 2. We now have a list of tasks to delete, delete them
|
||||
|
||||
let mut affected_indexes = HashSet::new();
|
||||
let mut affected_statuses = HashSet::new();
|
||||
let mut affected_kinds = HashSet::new();
|
||||
|
||||
for task_id in to_delete_tasks.iter() {
|
||||
let task = self.all_tasks.get(wtxn, &BEU32::new(task_id))?.unwrap();
|
||||
self.delete_task(wtxn, &task)?;
|
||||
}
|
||||
|
||||
Ok(to_delete_tasks.len() as usize)
|
||||
}
|
||||
|
||||
/// Delete the given task from all the databases
|
||||
fn delete_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> {
|
||||
let task_id = BEU32::new(task.uid);
|
||||
|
||||
if let Some(indexes) = task.indexes() {
|
||||
for index in indexes {
|
||||
self.update_index(wtxn, index, |bitmap| {
|
||||
bitmap.remove(task.uid);
|
||||
})?;
|
||||
if let Some(task) = self.all_tasks.get(wtxn, &BEU32::new(task_id))? {
|
||||
if let Some(task_indexes) = task.indexes() {
|
||||
affected_indexes.extend(task_indexes.into_iter().map(|x| x.to_owned()));
|
||||
}
|
||||
affected_statuses.insert(task.status);
|
||||
affected_kinds.insert(task.kind.as_kind());
|
||||
}
|
||||
}
|
||||
|
||||
self.update_status(wtxn, task.status, |bitmap| {
|
||||
bitmap.remove(task.uid);
|
||||
})?;
|
||||
|
||||
self.update_kind(wtxn, task.kind.as_kind(), |bitmap| {
|
||||
(bitmap.remove(task.uid));
|
||||
})?;
|
||||
|
||||
self.all_tasks.delete(wtxn, &task_id)?;
|
||||
Ok(())
|
||||
for index in affected_indexes {
|
||||
self.update_index(wtxn, &index, |bitmap| {
|
||||
*bitmap -= &to_delete_tasks;
|
||||
})?;
|
||||
}
|
||||
for status in affected_statuses {
|
||||
self.update_status(wtxn, status, |bitmap| {
|
||||
*bitmap -= &to_delete_tasks;
|
||||
})?;
|
||||
}
|
||||
for kind in affected_kinds {
|
||||
self.update_kind(wtxn, kind, |bitmap| {
|
||||
*bitmap -= &to_delete_tasks;
|
||||
})?;
|
||||
}
|
||||
for task in to_delete_tasks.iter() {
|
||||
self.all_tasks.delete(wtxn, &BEU32::new(task))?;
|
||||
}
|
||||
Ok(to_delete_tasks.len() as usize)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -242,44 +242,52 @@ impl IndexScheduler {
|
|||
self.index_mapper.indexes(&rtxn)
|
||||
}
|
||||
|
||||
/// Returns the tasks corresponding to the query.
|
||||
pub fn get_tasks(&self, query: Query) -> Result<Vec<Task>> {
|
||||
/// Return the task ids corresponding to the query
|
||||
pub fn get_task_ids(&self, query: &Query) -> Result<RoaringBitmap> {
|
||||
let rtxn = self.env.read_txn()?;
|
||||
let last_task_id = match self.last_task_id(&rtxn)? {
|
||||
Some(tid) => query.from.map(|from| from.min(tid)).unwrap_or(tid),
|
||||
None => return Ok(Vec::new()),
|
||||
None => return Ok(RoaringBitmap::new()),
|
||||
};
|
||||
|
||||
// This is the list of all the tasks.
|
||||
let mut tasks = RoaringBitmap::from_sorted_iter(0..last_task_id).unwrap();
|
||||
|
||||
if let Some(uids) = query.uid {
|
||||
if let Some(uids) = &query.uid {
|
||||
tasks &= RoaringBitmap::from_iter(uids);
|
||||
}
|
||||
|
||||
if let Some(status) = query.status {
|
||||
if let Some(status) = &query.status {
|
||||
let mut status_tasks = RoaringBitmap::new();
|
||||
for status in status {
|
||||
status_tasks |= self.get_status(&rtxn, status)?;
|
||||
status_tasks |= self.get_status(&rtxn, *status)?;
|
||||
}
|
||||
tasks &= status_tasks;
|
||||
}
|
||||
|
||||
if let Some(kind) = query.kind {
|
||||
if let Some(kind) = &query.kind {
|
||||
let mut kind_tasks = RoaringBitmap::new();
|
||||
for kind in kind {
|
||||
kind_tasks |= self.get_kind(&rtxn, kind)?;
|
||||
kind_tasks |= self.get_kind(&rtxn, *kind)?;
|
||||
}
|
||||
tasks &= kind_tasks;
|
||||
}
|
||||
|
||||
if let Some(index) = query.index_uid {
|
||||
if let Some(index) = &query.index_uid {
|
||||
let mut index_tasks = RoaringBitmap::new();
|
||||
for index in index {
|
||||
index_tasks |= self.index_tasks(&rtxn, &index)?;
|
||||
}
|
||||
tasks &= index_tasks;
|
||||
}
|
||||
rtxn.commit().unwrap();
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
/// Returns the tasks corresponding to the query.
|
||||
pub fn get_tasks(&self, query: Query) -> Result<Vec<Task>> {
|
||||
let rtxn = self.env.read_txn()?;
|
||||
let tasks = self.get_task_ids(&query)?;
|
||||
|
||||
let tasks =
|
||||
self.get_existing_tasks(&rtxn, tasks.into_iter().rev().take(query.limit as usize))?;
|
||||
|
|
@ -442,7 +450,7 @@ impl IndexScheduler {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use big_s::S;
|
||||
use insta::*;
|
||||
use meili_snap::snapshot;
|
||||
use meilisearch_types::milli::update::IndexDocumentsMethod::ReplaceDocuments;
|
||||
use tempfile::TempDir;
|
||||
use uuid::Uuid;
|
||||
|
|
@ -580,7 +588,7 @@ mod tests {
|
|||
assert_eq!(task.kind.as_kind(), k);
|
||||
}
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -597,7 +605,7 @@ mod tests {
|
|||
})
|
||||
.unwrap();
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
}
|
||||
|
||||
/// We send a lot of tasks but notify the tasks scheduler only once as
|
||||
|
|
@ -692,23 +700,23 @@ mod tests {
|
|||
|
||||
// here we have registered all the tasks, but the index scheduler
|
||||
// has not progressed at all
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued");
|
||||
|
||||
index_scheduler
|
||||
.register(KindWithContent::DeleteTasks {
|
||||
.register(KindWithContent::TaskDeletion {
|
||||
query: "test_query".to_owned(),
|
||||
tasks: vec![0, 1],
|
||||
tasks: RoaringBitmap::from_iter(&[0, 1]),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// again, no progress made at all, but one more task is registered
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_enqueued");
|
||||
|
||||
// now we create the first batch
|
||||
handle.wait_till(Breakpoint::BatchCreated);
|
||||
|
||||
// the task deletion should now be "processing"
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processing");
|
||||
|
||||
handle.wait_till(Breakpoint::AfterProcessing);
|
||||
|
||||
|
|
@ -716,7 +724,7 @@ mod tests {
|
|||
// because the tasks with ids 0 and 1 were still "enqueued", and thus undeleteable
|
||||
// the "task deletion" task should be marked as "succeeded" and, in its details, the
|
||||
// number of deleted tasks should be 0
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_done");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -737,25 +745,25 @@ mod tests {
|
|||
file0.persist().unwrap();
|
||||
file1.persist().unwrap();
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued");
|
||||
|
||||
handle.wait_till(Breakpoint::AfterProcessing);
|
||||
// first addition of documents should be successful
|
||||
// TODO: currently the result of this operation is incorrect!
|
||||
// only the first task should be successful, because it should not be batched with
|
||||
// the second task, that operates on a different index!
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_processed");
|
||||
|
||||
// Now we delete the first task
|
||||
index_scheduler
|
||||
.register(KindWithContent::DeleteTasks {
|
||||
.register(KindWithContent::TaskDeletion {
|
||||
query: "test_query".to_owned(),
|
||||
tasks: vec![0],
|
||||
tasks: RoaringBitmap::from_iter(&[0]),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
handle.wait_till(Breakpoint::AfterProcessing);
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -784,15 +792,15 @@ mod tests {
|
|||
.unwrap();
|
||||
file.persist().unwrap();
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
|
||||
handle.wait_till(Breakpoint::BatchCreated);
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
|
||||
handle.wait_till(Breakpoint::AfterProcessing);
|
||||
|
||||
assert_snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
snapshot!(snapshot_index_scheduler(&index_scheduler));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -833,18 +841,10 @@ mod tests {
|
|||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! assert_smol_debug_snapshot {
|
||||
macro_rules! debug_snapshot {
|
||||
($value:expr, @$snapshot:literal) => {{
|
||||
let value = format!("{:?}", $value);
|
||||
insta::assert_snapshot!(value, stringify!($value), @$snapshot);
|
||||
}};
|
||||
($name:expr, $value:expr) => {{
|
||||
let value = format!("{:?}", $value);
|
||||
insta::assert_snapshot!(Some($name), value, stringify!($value));
|
||||
}};
|
||||
($value:expr) => {{
|
||||
let value = format!("{:?}", $value);
|
||||
insta::assert_snapshot!($crate::_macro_support::AutoName, value, stringify!($value));
|
||||
meili_snap::snapshot!(value, @$snapshot);
|
||||
}};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
use meilisearch_types::heed::{
|
||||
types::{OwnedType, SerdeBincode, SerdeJson, Str},
|
||||
Database, RoTxn,
|
||||
};
|
||||
use meilisearch_types::milli::{RoaringBitmapCodec, BEU32};
|
||||
use meilisearch_types::tasks::{Details, Task};
|
||||
use meilisearch_types::tasks::Details;
|
||||
use meilisearch_types::{
|
||||
heed::{
|
||||
types::{OwnedType, SerdeBincode, SerdeJson, Str},
|
||||
Database, RoTxn,
|
||||
},
|
||||
tasks::Task,
|
||||
};
|
||||
use roaring::RoaringBitmap;
|
||||
|
||||
use crate::{index_mapper::IndexMapper, IndexScheduler, Kind, Status};
|
||||
|
|
@ -127,7 +130,7 @@ fn snaphsot_details(d: &Details) -> String {
|
|||
Details::ClearAll { deleted_documents } => {
|
||||
format!("{{ deleted_documents: {deleted_documents:?} }}")
|
||||
},
|
||||
Details::DeleteTasks {
|
||||
Details::TaskDeletion {
|
||||
matched_tasks,
|
||||
deleted_tasks,
|
||||
original_query,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -13,7 +12,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
enqueued [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,]
|
||||
"documentImport" [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
doggos [0,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -13,7 +12,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
enqueued [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,]
|
||||
"documentImport" [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
doggos [0,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -14,7 +13,7 @@ enqueued []
|
|||
succeeded [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,]
|
||||
"documentImport" [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
doggos [0,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -17,7 +16,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
enqueued [0,1,2,3,4,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,3,4,]
|
||||
"documentImport" [1,3,4,]
|
||||
"indexCreation" [0,]
|
||||
"cancelTask" [2,]
|
||||
----------------------------------------------------------------------
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -14,7 +13,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
enqueued [0,1,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,1,]
|
||||
"documentImport" [0,1,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto [0,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -15,7 +14,7 @@ enqueued []
|
|||
succeeded [0,1,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,1,]
|
||||
"documentImport" [0,1,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto [0,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -8,15 +7,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
----------------------------------------------------------------------
|
||||
### All Tasks:
|
||||
1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: 1 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
|
||||
2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0] }}
|
||||
2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
|
||||
----------------------------------------------------------------------
|
||||
### Status:
|
||||
enqueued []
|
||||
succeeded [1,2,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,]
|
||||
"deleteTasks" [2,]
|
||||
"documentImport" [1,]
|
||||
"taskDeletion" [2,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto []
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -15,7 +14,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
enqueued [0,1,2,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,]
|
||||
"documentImport" [1,2,]
|
||||
"indexCreation" [0,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -10,16 +9,16 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
|
||||
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
|
||||
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
|
||||
3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }}
|
||||
3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
|
||||
----------------------------------------------------------------------
|
||||
### Status:
|
||||
enqueued [0,1,2,]
|
||||
succeeded [3,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,]
|
||||
"documentImport" [1,2,]
|
||||
"indexCreation" [0,]
|
||||
"deleteTasks" [3,]
|
||||
"taskDeletion" [3,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto [0,1,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -10,15 +9,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
|
||||
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
|
||||
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
|
||||
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }}
|
||||
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
|
||||
----------------------------------------------------------------------
|
||||
### Status:
|
||||
enqueued [0,1,2,3,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,]
|
||||
"documentImport" [1,2,]
|
||||
"indexCreation" [0,]
|
||||
"deleteTasks" [3,]
|
||||
"taskDeletion" [3,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto [0,1,]
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
source: index-scheduler/src/lib.rs
|
||||
expression: snapshot_index_scheduler(&index_scheduler)
|
||||
---
|
||||
### Autobatching Enabled = true
|
||||
### Processing Tasks:
|
||||
|
|
@ -10,15 +9,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
|
|||
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
|
||||
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
|
||||
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
|
||||
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }}
|
||||
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
|
||||
----------------------------------------------------------------------
|
||||
### Status:
|
||||
enqueued [0,1,2,3,]
|
||||
----------------------------------------------------------------------
|
||||
### Kind:
|
||||
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,]
|
||||
"documentImport" [1,2,]
|
||||
"indexCreation" [0,]
|
||||
"deleteTasks" [3,]
|
||||
"taskDeletion" [3,]
|
||||
----------------------------------------------------------------------
|
||||
### Index Tasks:
|
||||
catto [0,1,]
|
||||
Loading…
Add table
Add a link
Reference in a new issue