From e3848b5f28e4d2349a46cdebb9f0f299e4eae721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 20 Oct 2022 13:11:50 +0200 Subject: [PATCH] Add assert method to verify validity of index scheduler state --- index-scheduler/src/lib.rs | 91 ++++++++- .../src/snapshots/lib.rs/main/1.snap | 34 ---- .../src/snapshots/lib.rs/main/2.snap | 34 ---- .../src/snapshots/lib.rs/main/3.snap | 36 ---- .../src/snapshots/lib.rs/main/4.snap | 40 ---- .../src/snapshots/lib.rs/main/5.snap | 44 ----- .../src/snapshots/lib.rs/main/6.snap | 37 ---- .../src/snapshots/lib.rs/main/7.snap | 39 ---- .../src/snapshots/lib.rs/main/8.snap | 39 ---- .../src/snapshots/lib.rs/main/9.snap | 41 ---- .../lib.rs/main/first_swap_processed.snap | 56 ------ .../lib.rs/main/initial_tasks_enqueued.snap | 39 ---- .../lib.rs/main/initial_tasks_processed.snap | 51 ----- .../lib.rs/main/second_swap_processed.snap | 60 ------ .../lib.rs/main/task_deletion_done.snap | 45 ----- .../lib.rs/main/task_deletion_enqueued.snap | 42 ---- .../lib.rs/main/task_deletion_processing.snap | 42 ---- .../src/snapshots/lib.rs/register/1.snap | 3 + .../initial_tasks_enqueued.snap | 6 +- .../task_deletion_done.snap | 6 +- .../task_deletion_enqueued.snap | 6 +- .../task_deletion_processing.snap | 6 +- index-scheduler/src/utils.rs | 180 ++++++++++++++++++ 23 files changed, 282 insertions(+), 695 deletions(-) delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/1.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/2.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/3.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/4.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/5.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/6.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/7.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/8.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/9.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/first_swap_processed.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/initial_tasks_enqueued.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/initial_tasks_processed.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/second_swap_processed.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/task_deletion_done.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/task_deletion_enqueued.snap delete mode 100644 index-scheduler/src/snapshots/lib.rs/main/task_deletion_processing.snap diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index c54558bab..d02201606 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -53,7 +53,8 @@ use uuid::Uuid; use crate::index_mapper::IndexMapper; -type BEI128 = meilisearch_types::heed::zerocopy::I128; +pub(crate) type BEI128 = + meilisearch_types::heed::zerocopy::I128; /// Defines a subset of tasks to be retrieved from the [`IndexScheduler`]. /// @@ -879,9 +880,17 @@ mod tests { replace_document_import_task("catto", None, 1, 50), replace_document_import_task("doggo", Some("bone"), 2, 5000), ]; + let (_, file) = index_scheduler.create_update_file_with_uuid(0).unwrap(); + file.persist().unwrap(); + let (_, file) = index_scheduler.create_update_file_with_uuid(1).unwrap(); + file.persist().unwrap(); + let (_, file) = index_scheduler.create_update_file_with_uuid(2).unwrap(); + file.persist().unwrap(); + for (idx, kind) in kinds.into_iter().enumerate() { let k = kind.as_kind(); let task = index_scheduler.register(kind).unwrap(); + index_scheduler.assert_internally_consistent(); assert_eq!(task.uid, idx as u32); assert_eq!(task.status, Status::Enqueued); @@ -896,12 +905,19 @@ mod tests { let (index_scheduler, handle) = IndexScheduler::test(true); index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::BatchCreated); + index_scheduler.assert_internally_consistent(); + // while the task is processing can we register another task? index_scheduler.register(index_creation_task("index_b", "id")).unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::IndexDeletion { index_uid: S("index_a") }) .unwrap(); + index_scheduler.assert_internally_consistent(); snapshot!(snapshot_index_scheduler(&index_scheduler)); } @@ -915,17 +931,29 @@ mod tests { index_scheduler .register(KindWithContent::IndexCreation { index_uid: S("doggos"), primary_key: None }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::IndexCreation { index_uid: S("cattos"), primary_key: None }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::IndexDeletion { index_uid: S("doggos") }) .unwrap(); + index_scheduler.assert_internally_consistent(); handle.wait_till(Breakpoint::Start); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); tasks.reverse(); @@ -942,20 +970,34 @@ mod tests { index_scheduler .register(KindWithContent::IndexCreation { index_uid: S("doggos"), primary_key: None }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::DocumentClear { index_uid: S("doggos") }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::DocumentClear { index_uid: S("doggos") }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::DocumentClear { index_uid: S("doggos") }) .unwrap(); + index_scheduler.assert_internally_consistent(); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); tasks.reverse(); @@ -970,13 +1012,20 @@ mod tests { fn task_deletion_undeleteable() { let (index_scheduler, handle) = IndexScheduler::test(true); + let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); + let (file1, documents_count1) = sample_documents(&index_scheduler, 1, 1); + file0.persist().unwrap(); + file1.persist().unwrap(); + let to_enqueue = [ index_creation_task("catto", "mouse"), - replace_document_import_task("catto", None, 0, 12), - replace_document_import_task("doggo", Some("bone"), 1, 5000), + replace_document_import_task("catto", None, 0, documents_count0), + replace_document_import_task("doggo", Some("bone"), 1, documents_count1), ]; + for task in to_enqueue { let _ = index_scheduler.register(task).unwrap(); + index_scheduler.assert_internally_consistent(); } // here we have registered all the tasks, but the index scheduler @@ -989,17 +1038,20 @@ mod tests { tasks: RoaringBitmap::from_iter(&[0, 1]), }) .unwrap(); + index_scheduler.assert_internally_consistent(); // again, no progress made at all, but one more task is registered snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_enqueued"); // now we create the first batch handle.wait_till(Breakpoint::BatchCreated); + index_scheduler.assert_internally_consistent(); // the task deletion should now be "processing" snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processing"); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); // after the task deletion is processed, no task should actually have been deleted, // because the tasks with ids 0 and 1 were still "enqueued", and thus undeleteable @@ -1014,6 +1066,8 @@ mod tests { let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); let (file1, documents_count1) = sample_documents(&index_scheduler, 1, 1); + file0.persist().unwrap(); + file1.persist().unwrap(); let to_enqueue = [ replace_document_import_task("catto", None, 0, documents_count0), @@ -1022,13 +1076,14 @@ mod tests { for task in to_enqueue { let _ = index_scheduler.register(task).unwrap(); + index_scheduler.assert_internally_consistent(); } - file0.persist().unwrap(); - file1.persist().unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued"); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + // first addition of documents should be successful snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_processed"); @@ -1039,8 +1094,11 @@ mod tests { tasks: RoaringBitmap::from_iter(&[0]), }) .unwrap(); + index_scheduler.assert_internally_consistent(); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processed"); } @@ -1050,6 +1108,8 @@ mod tests { let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); let (file1, documents_count1) = sample_documents(&index_scheduler, 1, 1); + file0.persist().unwrap(); + file1.persist().unwrap(); let to_enqueue = [ replace_document_import_task("catto", None, 0, documents_count0), @@ -1058,13 +1118,14 @@ mod tests { for task in to_enqueue { let _ = index_scheduler.register(task).unwrap(); + index_scheduler.assert_internally_consistent(); } - file0.persist().unwrap(); - file1.persist().unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued"); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); + // first addition of documents should be successful snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_processed"); @@ -1076,10 +1137,13 @@ mod tests { tasks: RoaringBitmap::from_iter(&[0]), }) .unwrap(); + index_scheduler.assert_internally_consistent(); } for _ in 0..2 { handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); } + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processed"); } @@ -1097,6 +1161,7 @@ mod tests { let documents_count = meilisearch_types::document_formats::read_json(content.as_bytes(), file.as_file_mut()) .unwrap() as u64; + file.persist().unwrap(); index_scheduler .register(KindWithContent::DocumentAdditionOrUpdate { index_uid: S("doggos"), @@ -1107,15 +1172,18 @@ mod tests { allow_index_creation: true, }) .unwrap(); - file.persist().unwrap(); + + index_scheduler.assert_internally_consistent(); snapshot!(snapshot_index_scheduler(&index_scheduler)); handle.wait_till(Breakpoint::BatchCreated); + index_scheduler.assert_internally_consistent(); snapshot!(snapshot_index_scheduler(&index_scheduler)); handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); snapshot!(snapshot_index_scheduler(&index_scheduler)); } @@ -1133,6 +1201,7 @@ mod tests { index_scheduler .register(KindWithContent::IndexCreation { index_uid: S("doggos"), primary_key: None }) .unwrap(); + index_scheduler.assert_internally_consistent(); let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap(); let documents_count = @@ -1149,9 +1218,12 @@ mod tests { allow_index_creation: true, }) .unwrap(); + index_scheduler.assert_internally_consistent(); + index_scheduler .register(KindWithContent::IndexDeletion { index_uid: S("doggos") }) .unwrap(); + index_scheduler.assert_internally_consistent(); snapshot!(snapshot_index_scheduler(&index_scheduler)); @@ -1174,16 +1246,19 @@ mod tests { primary_key: None, }) .unwrap(); + index_scheduler.assert_internally_consistent(); } for name in index_names { index_scheduler .register(KindWithContent::DocumentClear { index_uid: name.to_string() }) .unwrap(); + index_scheduler.assert_internally_consistent(); } for _ in 0..(index_names.len() * 2) { handle.wait_till(Breakpoint::AfterProcessing); + index_scheduler.assert_internally_consistent(); } let mut tasks = index_scheduler.get_tasks(Query::default()).unwrap(); diff --git a/index-scheduler/src/snapshots/lib.rs/main/1.snap b/index-scheduler/src/snapshots/lib.rs/main/1.snap deleted file mode 100644 index 6abb00f81..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/1.snap +++ /dev/null @@ -1,34 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ----------------------------------------------------------------------- -### Status: -enqueued [0,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [0,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: -00000000-0000-0000-0000-000000000000 - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/2.snap b/index-scheduler/src/snapshots/lib.rs/main/2.snap deleted file mode 100644 index b9e745cf0..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/2.snap +++ /dev/null @@ -1,34 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[0,] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ----------------------------------------------------------------------- -### Status: -enqueued [0,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [0,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: -00000000-0000-0000-0000-000000000000 - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/3.snap b/index-scheduler/src/snapshots/lib.rs/main/3.snap deleted file mode 100644 index 2bcc9368d..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/3.snap +++ /dev/null @@ -1,36 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [0,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,] ----------------------------------------------------------------------- -### Index Mapper: -["doggos"] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [0,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [0,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/4.snap b/index-scheduler/src/snapshots/lib.rs/main/4.snap deleted file mode 100644 index 448988c8c..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/4.snap +++ /dev/null @@ -1,40 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} -2 {uid: 2, status: enqueued, kind: IndexDeletion { index_uid: "doggos" }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,] -"indexCreation" [0,] -"indexDeletion" [2,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,1,2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: -00000000-0000-0000-0000-000000000000 - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/5.snap b/index-scheduler/src/snapshots/lib.rs/main/5.snap deleted file mode 100644 index 800b5a9b4..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/5.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} -2 {uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,1,2,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,] -"indexCreation" [0,] -"indexDeletion" [2,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,1,2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [0,] -[timestamp] [2,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [0,] -[timestamp] [2,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/6.snap b/index-scheduler/src/snapshots/lib.rs/main/6.snap deleted file mode 100644 index 3f921934d..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/6.snap +++ /dev/null @@ -1,37 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} -1 {uid: 1, status: enqueued, kind: IndexDeletion { index_uid: "doggos" }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [0,] -"indexDeletion" [1,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,1,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: -00000000-0000-0000-0000-000000000000 - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/7.snap b/index-scheduler/src/snapshots/lib.rs/main/7.snap deleted file mode 100644 index e29b7216e..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/7.snap +++ /dev/null @@ -1,39 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} -1 {uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,1,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [0,] -"indexDeletion" [1,] ----------------------------------------------------------------------- -### Index Tasks: -doggos [0,1,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [1,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [1,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/8.snap b/index-scheduler/src/snapshots/lib.rs/main/8.snap deleted file mode 100644 index ddac65249..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/8.snap +++ /dev/null @@ -1,39 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[0,] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, kind: IndexDeletion { index_uid: "index_a" }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,] ----------------------------------------------------------------------- -### Kind: -"indexCreation" [0,1,] -"indexDeletion" [2,] ----------------------------------------------------------------------- -### Index Tasks: -index_a [0,2,] -index_b [1,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/9.snap b/index-scheduler/src/snapshots/lib.rs/main/9.snap deleted file mode 100644 index 31034057a..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/9.snap +++ /dev/null @@ -1,41 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} -3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,3,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,2,3,] -"indexCreation" [0,] ----------------------------------------------------------------------- -### Index Tasks: -catto [0,1,2,] -doggo [3,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/first_swap_processed.snap b/index-scheduler/src/snapshots/lib.rs/main/first_swap_processed.snap deleted file mode 100644 index 6744367c3..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/first_swap_processed.snap +++ /dev/null @@ -1,56 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -2 {uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -4 {uid: 4, status: succeeded, details: { indexes: [("a", "b"), ("c", "d")] }, kind: IndexSwap { swaps: [("a", "b"), ("c", "d")] }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,1,2,3,4,] ----------------------------------------------------------------------- -### Kind: -"indexCreation" [0,1,2,3,] -"indexSwap" [4,] ----------------------------------------------------------------------- -### Index Tasks: -a [1,4,] -b [0,4,] -c [3,4,] -d [2,4,] ----------------------------------------------------------------------- -### Index Mapper: -["a", "b", "c", "d"] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_enqueued.snap b/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_enqueued.snap deleted file mode 100644 index 86318909e..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_enqueued.snap +++ /dev/null @@ -1,39 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,2,] -"indexCreation" [0,] ----------------------------------------------------------------------- -### Index Tasks: -catto [0,1,] -doggo [2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_processed.snap b/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_processed.snap deleted file mode 100644 index 073f280f3..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/initial_tasks_processed.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,1,2,3,] ----------------------------------------------------------------------- -### Kind: -"indexCreation" [0,1,2,3,] ----------------------------------------------------------------------- -### Index Tasks: -a [0,] -b [1,] -c [2,] -d [3,] ----------------------------------------------------------------------- -### Index Mapper: -["a", "b", "c", "d"] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/second_swap_processed.snap b/index-scheduler/src/snapshots/lib.rs/main/second_swap_processed.snap deleted file mode 100644 index 543a0afa4..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/second_swap_processed.snap +++ /dev/null @@ -1,60 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, status: succeeded, details: { indexes: [("a", "b"), ("c", "d")] }, kind: IndexSwap { swaps: [("c", "b"), ("a", "d")] }} -5 {uid: 5, status: succeeded, details: { indexes: [("a", "c")] }, kind: IndexSwap { swaps: [("a", "c")] }} ----------------------------------------------------------------------- -### Status: -enqueued [] -succeeded [0,1,2,3,4,5,] ----------------------------------------------------------------------- -### Kind: -"indexCreation" [0,1,2,3,] -"indexSwap" [4,5,] ----------------------------------------------------------------------- -### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] ----------------------------------------------------------------------- -### Index Mapper: -["a", "b", "c", "d"] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] -[timestamp] [5,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] -[timestamp] [5,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] -[timestamp] [4,] -[timestamp] [5,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_done.snap b/index-scheduler/src/snapshots/lib.rs/main/task_deletion_done.snap deleted file mode 100644 index 792756094..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_done.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} -3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,] -succeeded [3,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,2,] -"indexCreation" [0,] -"taskDeletion" [3,] ----------------------------------------------------------------------- -### Index Tasks: -catto [0,1,] -doggo [2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Started At: -[timestamp] [3,] ----------------------------------------------------------------------- -### Finished At: -[timestamp] [3,] ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_enqueued.snap b/index-scheduler/src/snapshots/lib.rs/main/task_deletion_enqueued.snap deleted file mode 100644 index 6fdb8b423..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_enqueued.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} -3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,3,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,2,] -"indexCreation" [0,] -"taskDeletion" [3,] ----------------------------------------------------------------------- -### Index Tasks: -catto [0,1,] -doggo [2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_processing.snap b/index-scheduler/src/snapshots/lib.rs/main/task_deletion_processing.snap deleted file mode 100644 index 8a04e70e8..000000000 --- a/index-scheduler/src/snapshots/lib.rs/main/task_deletion_processing.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: index-scheduler/src/lib.rs ---- -### Autobatching Enabled = true -### Processing Tasks: -[3,] ----------------------------------------------------------------------- -### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} -3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ----------------------------------------------------------------------- -### Status: -enqueued [0,1,2,3,] ----------------------------------------------------------------------- -### Kind: -"documentAdditionOrUpdate" [1,2,] -"indexCreation" [0,] -"taskDeletion" [3,] ----------------------------------------------------------------------- -### Index Tasks: -catto [0,1,] -doggo [2,] ----------------------------------------------------------------------- -### Index Mapper: -[] ----------------------------------------------------------------------- -### Enqueued At: -[timestamp] [0,] -[timestamp] [1,] -[timestamp] [2,] -[timestamp] [3,] ----------------------------------------------------------------------- -### Started At: ----------------------------------------------------------------------- -### Finished At: ----------------------------------------------------------------------- -### File Store: - ----------------------------------------------------------------------- - diff --git a/index-scheduler/src/snapshots/lib.rs/register/1.snap b/index-scheduler/src/snapshots/lib.rs/register/1.snap index 31034057a..95eaa11c5 100644 --- a/index-scheduler/src/snapshots/lib.rs/register/1.snap +++ b/index-scheduler/src/snapshots/lib.rs/register/1.snap @@ -36,6 +36,9 @@ doggo [3,] ### Finished At: ---------------------------------------------------------------------- ### File Store: +00000000-0000-0000-0000-000000000000 +00000000-0000-0000-0000-000000000001 +00000000-0000-0000-0000-000000000002 ---------------------------------------------------------------------- diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index 86318909e..b22cad0ca 100644 --- a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -7,8 +7,8 @@ source: index-scheduler/src/lib.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} +1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} +2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] @@ -34,6 +34,8 @@ doggo [2,] ### Finished At: ---------------------------------------------------------------------- ### File Store: +00000000-0000-0000-0000-000000000000 +00000000-0000-0000-0000-000000000001 ---------------------------------------------------------------------- diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap index 792756094..acf3b752c 100644 --- a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -7,8 +7,8 @@ source: index-scheduler/src/lib.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} +1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} +2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ---------------------------------------------------------------------- ### Status: @@ -40,6 +40,8 @@ doggo [2,] [timestamp] [3,] ---------------------------------------------------------------------- ### File Store: +00000000-0000-0000-0000-000000000000 +00000000-0000-0000-0000-000000000001 ---------------------------------------------------------------------- diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index 6fdb8b423..f41fae458 100644 --- a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -7,8 +7,8 @@ source: index-scheduler/src/lib.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} +1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} +2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ---------------------------------------------------------------------- ### Status: @@ -37,6 +37,8 @@ doggo [2,] ### Finished At: ---------------------------------------------------------------------- ### File Store: +00000000-0000-0000-0000-000000000000 +00000000-0000-0000-0000-000000000001 ---------------------------------------------------------------------- diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap index 8a04e70e8..15638b4b4 100644 --- a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,8 +7,8 @@ source: index-scheduler/src/lib.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} -2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} +1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} +2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} ---------------------------------------------------------------------- ### Status: @@ -37,6 +37,8 @@ doggo [2,] ### Finished At: ---------------------------------------------------------------------- ### File Store: +00000000-0000-0000-0000-000000000000 +00000000-0000-0000-0000-000000000001 ---------------------------------------------------------------------- diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index 45e26edeb..3611f3a17 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -266,3 +266,183 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { } } } +#[cfg(test)] +use meilisearch_types::tasks::Details; +#[cfg(test)] +impl IndexScheduler { + /// Asserts that the index scheduler's content is internally consistent. + pub fn assert_internally_consistent(&self) { + let rtxn = self.env.read_txn().unwrap(); + for task in self.all_tasks.iter(&rtxn).unwrap() { + let (task_id, task) = task.unwrap(); + let task_id = task_id.get(); + + let task_index_uid = task.index_uid().map(ToOwned::to_owned); + + let Task { + uid, + enqueued_at, + started_at, + finished_at, + error: _, + canceled_by, + details, + status, + kind, + } = task; + assert_eq!(uid, task.uid); + if let Some(task_index_uid) = &task_index_uid { + assert!(self + .index_tasks + .get(&rtxn, task_index_uid.as_str()) + .unwrap() + .unwrap() + .contains(task.uid)); + } + let db_enqueued_at = self + .enqueued_at + .get(&rtxn, &BEI128::new(enqueued_at.unix_timestamp_nanos())) + .unwrap() + .unwrap(); + assert!(db_enqueued_at.contains(task_id)); + if let Some(started_at) = started_at { + let db_started_at = self + .started_at + .get(&rtxn, &BEI128::new(started_at.unix_timestamp_nanos())) + .unwrap() + .unwrap(); + assert!(db_started_at.contains(task_id)); + } + if let Some(finished_at) = finished_at { + let db_finished_at = self + .finished_at + .get(&rtxn, &BEI128::new(finished_at.unix_timestamp_nanos())) + .unwrap() + .unwrap(); + assert!(db_finished_at.contains(task_id)); + } + if let Some(canceled_by) = canceled_by { + let db_canceled_tasks = self.get_status(&rtxn, Status::Canceled).unwrap(); + assert!(db_canceled_tasks.contains(canceled_by)); + let db_canceling_task = self.get_task(&rtxn, canceled_by).unwrap().unwrap(); + assert_eq!(db_canceling_task.status, Status::Succeeded); + match db_canceling_task.kind { + KindWithContent::TaskCancelation { query: _, tasks } => { + assert!(tasks.contains(uid)); + } + _ => panic!(), + } + } + match details { + Some(details) => match details { + Details::IndexSwap { swaps } => { + todo!() + } + Details::DocumentAdditionOrUpdate { received_documents, indexed_documents } => { + assert_eq!(kind.as_kind(), Kind::DocumentAdditionOrUpdate); + if let Some(indexed_documents) = indexed_documents { + assert_eq!(status, Status::Succeeded); + assert!(indexed_documents <= received_documents); + } else { + assert_ne!(status, Status::Succeeded); + } + } + Details::SettingsUpdate { settings: _ } => { + assert_eq!(kind.as_kind(), Kind::SettingsUpdate); + } + Details::IndexInfo { primary_key: pk1 } => match &kind { + KindWithContent::IndexCreation { index_uid, primary_key: pk2 } + | KindWithContent::IndexUpdate { index_uid, primary_key: pk2 } => { + self.index_tasks + .get(&rtxn, index_uid.as_str()) + .unwrap() + .unwrap() + .contains(uid); + assert_eq!(&pk1, pk2); + } + _ => panic!(), + }, + Details::DocumentDeletion { received_document_ids, deleted_documents } => { + if let Some(deleted_documents) = deleted_documents { + assert_eq!(status, Status::Succeeded); + assert!(deleted_documents <= received_document_ids as u64); + assert_eq!(kind.as_kind(), Kind::DocumentDeletion); + + match &kind { + KindWithContent::DocumentDeletion { index_uid, documents_ids } => { + assert_eq!(&task_index_uid.unwrap(), index_uid); + assert!(documents_ids.len() >= received_document_ids); + } + _ => panic!(), + } + } else { + assert_ne!(status, Status::Succeeded); + } + } + Details::ClearAll { deleted_documents } => { + assert!(matches!( + kind.as_kind(), + Kind::DocumentDeletion | Kind::IndexDeletion + )); + if deleted_documents.is_some() { + assert_eq!(status, Status::Succeeded); + } else { + assert_ne!(status, Status::Succeeded); + } + } + Details::TaskCancelation { matched_tasks, canceled_tasks, original_query } => { + if let Some(canceled_tasks) = canceled_tasks { + assert_eq!(status, Status::Succeeded); + assert!(canceled_tasks <= matched_tasks); + match &kind { + KindWithContent::TaskCancelation { query, tasks } => { + assert_eq!(query, &original_query); + assert_eq!(tasks.len(), matched_tasks); + } + _ => panic!(), + } + } else { + assert_ne!(status, Status::Succeeded); + } + } + Details::TaskDeletion { matched_tasks, deleted_tasks, original_query } => { + if let Some(deleted_tasks) = deleted_tasks { + assert_eq!(status, Status::Succeeded); + assert!(deleted_tasks <= matched_tasks); + match &kind { + KindWithContent::TaskDeletion { query, tasks } => { + assert_eq!(query, &original_query); + assert_eq!(tasks.len(), matched_tasks); + } + _ => panic!(), + } + } else { + assert_ne!(status, Status::Succeeded); + } + } + Details::Dump { dump_uid: d1 } => { + assert!( + matches!(&kind, KindWithContent::DumpExport { dump_uid: d2, keys: _, instance_uid: _ } if &d1 == d2 ) + ); + } + }, + None => (), + } + + assert!(self.get_status(&rtxn, status).unwrap().contains(uid)); + assert!(self.get_kind(&rtxn, kind.as_kind()).unwrap().contains(uid)); + + match kind { + KindWithContent::DocumentAdditionOrUpdate { content_file, .. } => match status { + Status::Enqueued | Status::Processing => { + assert!(self.file_store.__all_uuids().contains(&content_file)); + } + Status::Succeeded | Status::Failed | Status::Canceled => { + assert!(!self.file_store.__all_uuids().contains(&content_file)); + } + }, + _ => (), + } + } + } +}