From d7ddf4925e7ca5b4a444a7f14fe5f3035242f522 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 17 May 2023 14:25:50 +0200 Subject: [PATCH 01/14] Revert "Disable autobatching of additions and deletions" This reverts commit a94e78ffb051193ece752a9dd19858a05922f706. --- index-scheduler/src/autobatcher.rs | 103 ++++++++++++++++++++--------- index-scheduler/src/lib.rs | 99 +++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 33 deletions(-) diff --git a/index-scheduler/src/autobatcher.rs b/index-scheduler/src/autobatcher.rs index d738cc5e4..096bcce14 100644 --- a/index-scheduler/src/autobatcher.rs +++ b/index-scheduler/src/autobatcher.rs @@ -321,9 +321,18 @@ impl BatchKind { }) } ( - this @ BatchKind::DocumentOperation { .. }, + BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids }, K::DocumentDeletion, - ) => Break(this), + ) => { + operation_ids.push(id); + + Continue(BatchKind::DocumentOperation { + method, + allow_index_creation, + primary_key, + operation_ids, + }) + } // but we can't autobatch documents if it's not the same kind // this match branch MUST be AFTER the previous one ( @@ -346,7 +355,35 @@ impl BatchKind { deletion_ids.push(id); Continue(BatchKind::DocumentClear { ids: deletion_ids }) } - // we can't autobatch a deletion and an import + // we can autobatch the deletion and import if the index already exists + ( + BatchKind::DocumentDeletion { mut deletion_ids }, + K::DocumentImport { method, allow_index_creation, primary_key } + ) if index_already_exists => { + deletion_ids.push(id); + + Continue(BatchKind::DocumentOperation { + method, + allow_index_creation, + primary_key, + operation_ids: deletion_ids, + }) + } + // we can autobatch the deletion and import if both can't create an index + ( + BatchKind::DocumentDeletion { mut deletion_ids }, + K::DocumentImport { method, allow_index_creation, primary_key } + ) if !allow_index_creation => { + deletion_ids.push(id); + + Continue(BatchKind::DocumentOperation { + method, + allow_index_creation, + primary_key, + operation_ids: deletion_ids, + }) + } + // we can't autobatch a deletion and an import if the index does not exists but would be created by an addition ( this @ BatchKind::DocumentDeletion { .. }, K::DocumentImport { .. } @@ -648,36 +685,36 @@ mod tests { debug_snapshot!(autobatch_from(false,None, [settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0] }, false))"); debug_snapshot!(autobatch_from(false,None, [settings(false), settings(false), settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0, 1, 2] }, false))"); - // We can't autobatch document addition with document deletion - debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))"); - debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))"); - debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###); - debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###); - debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###); - debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###); - debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))"); - debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))"); - debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###); - debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###); - debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###); - debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###); - // we also can't do the only way around - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); - debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))"); + // We can autobatch document addition with document deletion + debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))"); + debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))"); + debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###); + debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###); + debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))"); + debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))"); + debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###); + debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###); + debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + // And the other way around + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))"); + debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); + debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###); } #[test] diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index af20ba1ae..f245bb186 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -2017,6 +2017,105 @@ mod tests { snapshot!(snapshot_index_scheduler(&index_scheduler), name: "both_task_succeeded"); } + #[test] + fn document_addition_and_document_deletion() { + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + + let content = r#"[ + { "id": 1, "doggo": "jean bob" }, + { "id": 2, "catto": "jorts" }, + { "id": 3, "doggo": "bork" } + ]"#; + + let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap(); + let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap(); + file.persist().unwrap(); + index_scheduler + .register(KindWithContent::DocumentAdditionOrUpdate { + index_uid: S("doggos"), + primary_key: Some(S("id")), + method: ReplaceDocuments, + content_file: uuid, + documents_count, + allow_index_creation: true, + }) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task"); + index_scheduler + .register(KindWithContent::DocumentDeletion { + index_uid: S("doggos"), + documents_ids: vec![S("1"), S("2")], + }) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task"); + + handle.advance_one_successful_batch(); // The addition AND deletion should've been batched together + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_the_batch"); + + let index = index_scheduler.index("doggos").unwrap(); + let rtxn = index.read_txn().unwrap(); + let field_ids_map = index.fields_ids_map(&rtxn).unwrap(); + let field_ids = field_ids_map.ids().collect::>(); + let documents = index + .all_documents(&rtxn) + .unwrap() + .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap()) + .collect::>(); + snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents"); + } + + #[test] + fn document_deletion_and_document_addition() { + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + index_scheduler + .register(KindWithContent::DocumentDeletion { + index_uid: S("doggos"), + documents_ids: vec![S("1"), S("2")], + }) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task"); + + let content = r#"[ + { "id": 1, "doggo": "jean bob" }, + { "id": 2, "catto": "jorts" }, + { "id": 3, "doggo": "bork" } + ]"#; + + let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap(); + let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap(); + file.persist().unwrap(); + index_scheduler + .register(KindWithContent::DocumentAdditionOrUpdate { + index_uid: S("doggos"), + primary_key: Some(S("id")), + method: ReplaceDocuments, + content_file: uuid, + documents_count, + allow_index_creation: true, + }) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task"); + + // The deletion should have failed because it can't create an index + handle.advance_one_failed_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_failing_the_deletion"); + + // The addition should works + handle.advance_one_successful_batch(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_last_successful_addition"); + + let index = index_scheduler.index("doggos").unwrap(); + let rtxn = index.read_txn().unwrap(); + let field_ids_map = index.fields_ids_map(&rtxn).unwrap(); + let field_ids = field_ids_map.ids().collect::>(); + let documents = index + .all_documents(&rtxn) + .unwrap() + .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap()) + .collect::>(); + snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents"); + } + #[test] fn do_not_batch_task_of_different_indexes() { let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); From 4391cba6ca10a60cce5be068f61634ee02e7a3aa Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 17 May 2023 18:19:43 +0200 Subject: [PATCH 02/14] fix the addition + deletion bug --- Cargo.lock | 22 +++ .../after_processing_the_batch.snap | 43 +++++ .../documents.snap | 9 + .../registered_the_first_task.snap | 37 ++++ .../registered_the_second_task.snap | 40 ++++ .../after_failing_the_deletion.snap | 43 +++++ .../after_last_successful_addition.snap | 46 +++++ .../documents.snap | 17 ++ .../registered_the_first_task.snap | 36 ++++ .../registered_the_second_task.snap | 40 ++++ milli/Cargo.toml | 4 +- milli/src/documents/mod.rs | 2 - milli/src/update/index_documents/mod.rs | 176 +++++++++++++++++- milli/src/update/index_documents/transform.rs | 21 +-- 14 files changed, 518 insertions(+), 18 deletions(-) create mode 100644 index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap create mode 100644 index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap diff --git a/Cargo.lock b/Cargo.lock index 5f192b6d1..a432908a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,6 +359,15 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arbitrary" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -1096,6 +1105,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdeb9ec472d588e539a818b2dee436825730da08ad0017c4b1a17676bdc8b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -2711,6 +2731,7 @@ dependencies = [ name = "milli" version = "1.2.0" dependencies = [ + "arbitrary", "big_s", "bimap", "bincode", @@ -2722,6 +2743,7 @@ dependencies = [ "csv", "deserr", "either", + "fastrand", "filter-parser", "flatten-serde-json", "fst", diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap new file mode 100644 index 000000000..b27288a0f --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap @@ -0,0 +1,43 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +1 {uid: 1, status: succeeded, details: { received_document_ids: 2, deleted_documents: Some(2) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [0,1,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [0,] +"documentDeletion" [1,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,] +---------------------------------------------------------------------- +### Index Mapper: +doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} } + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,1,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,1,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap new file mode 100644 index 000000000..2b56b71d1 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap @@ -0,0 +1,9 @@ +--- +source: index-scheduler/src/lib.rs +--- +[ + { + "id": 3, + "doggo": "bork" + } +] diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap new file mode 100644 index 000000000..d26e62bff --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap @@ -0,0 +1,37 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: +00000000-0000-0000-0000-000000000000 + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap new file mode 100644 index 000000000..e0f371120 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap @@ -0,0 +1,40 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +1 {uid: 1, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +---------------------------------------------------------------------- +### Status: +enqueued [0,1,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [0,] +"documentDeletion" [1,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: +00000000-0000-0000-0000-000000000000 + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap new file mode 100644 index 000000000..1d4aa24e2 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap @@ -0,0 +1,43 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +---------------------------------------------------------------------- +### Status: +enqueued [1,] +failed [0,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [1,] +"documentDeletion" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +---------------------------------------------------------------------- +### File Store: +00000000-0000-0000-0000-000000000000 + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap new file mode 100644 index 000000000..0f9dfd3e6 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap @@ -0,0 +1,46 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +---------------------------------------------------------------------- +### Status: +enqueued [] +succeeded [1,] +failed [0,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [1,] +"documentDeletion" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,] +---------------------------------------------------------------------- +### Index Mapper: +doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} } + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Finished At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap new file mode 100644 index 000000000..8204d059b --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap @@ -0,0 +1,17 @@ +--- +source: index-scheduler/src/lib.rs +--- +[ + { + "id": 1, + "doggo": "jean bob" + }, + { + "id": 2, + "catto": "jorts" + }, + { + "id": 3, + "doggo": "bork" + } +] diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap new file mode 100644 index 000000000..5753db7e6 --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap @@ -0,0 +1,36 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"documentDeletion" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap new file mode 100644 index 000000000..0b6191f9e --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap @@ -0,0 +1,40 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }} +1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} +---------------------------------------------------------------------- +### Status: +enqueued [0,1,] +---------------------------------------------------------------------- +### Kind: +"documentAdditionOrUpdate" [1,] +"documentDeletion" [0,] +---------------------------------------------------------------------- +### Index Tasks: +doggos [0,1,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +[timestamp] [1,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: +00000000-0000-0000-0000-000000000000 + +---------------------------------------------------------------------- + diff --git a/milli/Cargo.toml b/milli/Cargo.toml index de0f4e31d..ea48e008c 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -56,6 +56,7 @@ itertools = "0.10.5" log = "0.4.17" logging_timer = "1.1.0" csv = "1.2.1" +fastrand = "1.9.0" [dev-dependencies] mimalloc = { version = "0.1.29", default-features = false } @@ -64,12 +65,13 @@ insta = "1.29.0" maplit = "1.0.2" md5 = "0.7.0" rand = {version = "0.8.5", features = ["small_rng"] } +arbitrary = { version = "1.3.0", features = ["derive"] } [target.'cfg(fuzzing)'.dev-dependencies] fuzzcheck = "0.12.1" [features] -all-tokenizations = [ "charabia/default" ] +all-tokenizations = ["charabia/default"] # Use POSIX semaphores instead of SysV semaphores in LMDB # For more information on this feature, see heed's Cargo.toml diff --git a/milli/src/documents/mod.rs b/milli/src/documents/mod.rs index 43b31187d..7c037b3bf 100644 --- a/milli/src/documents/mod.rs +++ b/milli/src/documents/mod.rs @@ -111,7 +111,6 @@ pub enum Error { Io(#[from] io::Error), } -#[cfg(test)] pub fn objects_from_json_value(json: serde_json::Value) -> Vec { let documents = match json { object @ serde_json::Value::Object(_) => vec![object], @@ -141,7 +140,6 @@ macro_rules! documents { }}; } -#[cfg(test)] pub fn documents_batch_reader_from_objects( objects: impl IntoIterator, ) -> DocumentsBatchReader>> { diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index bbfa1d00c..406bfb0c9 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -198,6 +198,7 @@ where let number_of_documents = self.index.number_of_documents(self.wtxn)?; return Ok(DocumentAdditionResult { indexed_documents: 0, number_of_documents }); } + let output = self .transform .take() @@ -220,6 +221,7 @@ where } let indexed_documents = output.documents_count as u64; + let number_of_documents = self.execute_raw(output)?; Ok(DocumentAdditionResult { indexed_documents, number_of_documents }) @@ -236,7 +238,7 @@ where primary_key, fields_ids_map, field_distribution, - mut external_documents_ids, + new_external_documents_ids, new_documents_ids, replaced_documents_ids, documents_count, @@ -363,9 +365,6 @@ where deletion_builder.delete_documents(&replaced_documents_ids); let deleted_documents_result = deletion_builder.execute_inner()?; debug!("{} documents actually deleted", deleted_documents_result.deleted_documents); - if !deleted_documents_result.soft_deletion_used { - external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; - } } let index_documents_ids = self.index.documents_ids(self.wtxn)?; @@ -445,6 +444,9 @@ where self.index.put_primary_key(self.wtxn, &primary_key)?; // We write the external documents ids into the main database. + let mut external_documents_ids = self.index.external_documents_ids(self.wtxn)?; + external_documents_ids.insert_ids(&new_external_documents_ids)?; + let external_documents_ids = external_documents_ids.into_static(); self.index.put_external_documents_ids(self.wtxn, &external_documents_ids)?; let all_documents_ids = index_documents_ids | new_documents_ids; @@ -2515,4 +2517,170 @@ mod tests { db_snap!(index, word_position_docids, 3, @"74f556b91d161d997a89468b4da1cb8f"); db_snap!(index, docid_word_positions, 3, @"5287245332627675740b28bd46e1cde1"); } + + #[test] + fn reproduce_the_bug() { + /* + [milli/examples/fuzz.rs:69] &batches = [ + Batch( + [ + AddDoc( + { "id": 1, "doggo": "bernese" }, => internal 0 + ), + ], + ), + Batch( + [ + DeleteDoc( + 1, => delete internal 0 + ), + AddDoc( + { "id": 0, "catto": "jorts" }, => internal 1 + ), + ], + ), + Batch( + [ + AddDoc( + { "id": 1, "catto": "jorts" }, => internal 2 + ), + ], + ), + ] + */ + let mut index = TempIndex::new(); + index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard; + + // START OF BATCH + + println!("--- ENTERING BATCH 1"); + + let mut wtxn = index.write_txn().unwrap(); + + let builder = IndexDocuments::new( + &mut wtxn, + &index, + &index.indexer_config, + index.index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); + + // OP + + let documents = documents!([ + { "id": 1, "doggo": "bernese" }, + ]); + let (builder, added) = builder.add_documents(documents).unwrap(); + insta::assert_display_snapshot!(added.unwrap(), @"1"); + + // FINISHING + let addition = builder.execute().unwrap(); + insta::assert_debug_snapshot!(addition, @r###" + DocumentAdditionResult { + indexed_documents: 1, + number_of_documents: 1, + } + "###); + wtxn.commit().unwrap(); + + db_snap!(index, documents, @r###" + {"id":1,"doggo":"bernese"} + "###); + db_snap!(index, external_documents_ids, @r###" + soft: + hard: + 1 0 + "###); + + // A first batch of documents has been inserted + + // BATCH 2 + + println!("--- ENTERING BATCH 2"); + + let mut wtxn = index.write_txn().unwrap(); + + let builder = IndexDocuments::new( + &mut wtxn, + &index, + &index.indexer_config, + index.index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); + + let (builder, removed) = builder.remove_documents(vec![S("1")]).unwrap(); + insta::assert_display_snapshot!(removed.unwrap(), @"1"); + + let documents = documents!([ + { "id": 0, "catto": "jorts" }, + ]); + let (builder, added) = builder.add_documents(documents).unwrap(); + insta::assert_display_snapshot!(added.unwrap(), @"1"); + + let addition = builder.execute().unwrap(); + insta::assert_debug_snapshot!(addition, @r###" + DocumentAdditionResult { + indexed_documents: 1, + number_of_documents: 1, + } + "###); + wtxn.commit().unwrap(); + + db_snap!(index, documents, @r###" + {"id":0,"catto":"jorts"} + "###); + + db_snap!(index, external_documents_ids, @r###" + soft: + hard: + 0 1 + "###); + + db_snap!(index, soft_deleted_documents_ids, @"[]"); + + // BATCH 3 + + println!("--- ENTERING BATCH 3"); + + let mut wtxn = index.write_txn().unwrap(); + + let builder = IndexDocuments::new( + &mut wtxn, + &index, + &index.indexer_config, + index.index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); + + let documents = documents!([ + { "id": 1, "catto": "jorts" }, + ]); + let (builder, added) = builder.add_documents(documents).unwrap(); + insta::assert_display_snapshot!(added.unwrap(), @"1"); + + let addition = builder.execute().unwrap(); + insta::assert_debug_snapshot!(addition, @r###" + DocumentAdditionResult { + indexed_documents: 1, + number_of_documents: 2, + } + "###); + wtxn.commit().unwrap(); + + db_snap!(index, documents, @r###" + {"id":1,"catto":"jorts"} + {"id":0,"catto":"jorts"} + "###); + + // Ensuring all the returned IDs actually exists + let rtxn = index.read_txn().unwrap(); + let res = index.search(&rtxn).execute().unwrap(); + index.documents(&rtxn, res.documents_ids).unwrap(); + } } diff --git a/milli/src/update/index_documents/transform.rs b/milli/src/update/index_documents/transform.rs index 6097278a7..e2a260391 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/milli/src/update/index_documents/transform.rs @@ -21,15 +21,14 @@ use crate::error::{Error, InternalError, UserError}; use crate::index::{db_name, main_key}; use crate::update::{AvailableDocumentsIds, ClearDocuments, UpdateIndexingStep}; use crate::{ - ExternalDocumentsIds, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldsIdsMap, Index, - Result, BEU32, + FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldsIdsMap, Index, Result, BEU32, }; pub struct TransformOutput { pub primary_key: String, pub fields_ids_map: FieldsIdsMap, pub field_distribution: FieldDistribution, - pub external_documents_ids: ExternalDocumentsIds<'static>, + pub new_external_documents_ids: fst::Map>, pub new_documents_ids: RoaringBitmap, pub replaced_documents_ids: RoaringBitmap, pub documents_count: usize, @@ -58,8 +57,8 @@ pub struct Transform<'a, 'i> { original_sorter: grenad::Sorter, flattened_sorter: grenad::Sorter, - replaced_documents_ids: RoaringBitmap, - new_documents_ids: RoaringBitmap, + pub replaced_documents_ids: RoaringBitmap, + pub new_documents_ids: RoaringBitmap, // To increase the cache locality and decrease the heap usage we use compact smartstring. new_external_documents_ids_builder: FxHashMap, u64>, documents_count: usize, @@ -568,8 +567,6 @@ impl<'a, 'i> Transform<'a, 'i> { }))? .to_string(); - let mut external_documents_ids = self.index.external_documents_ids(wtxn)?; - // We create a final writer to write the new documents in order from the sorter. let mut writer = create_writer( self.indexer_settings.chunk_compression_type, @@ -651,13 +648,14 @@ impl<'a, 'i> Transform<'a, 'i> { fst_new_external_documents_ids_builder.insert(key, value) })?; let new_external_documents_ids = fst_new_external_documents_ids_builder.into_map(); - external_documents_ids.insert_ids(&new_external_documents_ids)?; Ok(TransformOutput { primary_key, fields_ids_map: self.fields_ids_map, field_distribution, - external_documents_ids: external_documents_ids.into_static(), + new_external_documents_ids: new_external_documents_ids + .map_data(|c| Cow::Owned(c)) + .unwrap(), new_documents_ids: self.new_documents_ids, replaced_documents_ids: self.replaced_documents_ids, documents_count: self.documents_count, @@ -691,7 +689,8 @@ impl<'a, 'i> Transform<'a, 'i> { let new_external_documents_ids = { let mut external_documents_ids = self.index.external_documents_ids(wtxn)?; external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; - external_documents_ids + // it is safe to get the hard document IDs + external_documents_ids.into_static().hard }; let documents_ids = self.index.documents_ids(wtxn)?; @@ -776,7 +775,7 @@ impl<'a, 'i> Transform<'a, 'i> { primary_key, fields_ids_map: new_fields_ids_map, field_distribution, - external_documents_ids: new_external_documents_ids.into_static(), + new_external_documents_ids, new_documents_ids: documents_ids, replaced_documents_ids: RoaringBitmap::default(), documents_count, From 7f619ff0e43708610fd00bb953ecef723c7bb16c Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 22 May 2023 10:33:49 +0200 Subject: [PATCH 03/14] get rids of the now unused soft_deletion_used parameter --- milli/src/update/delete_documents.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index 311f93f8f..10649b1bb 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -72,7 +72,6 @@ impl std::fmt::Display for DeletionStrategy { pub(crate) struct DetailedDocumentDeletionResult { pub deleted_documents: u64, pub remaining_documents: u64, - pub soft_deletion_used: bool, } impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { @@ -109,11 +108,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { Some(docid) } pub fn execute(self) -> Result { - let DetailedDocumentDeletionResult { - deleted_documents, - remaining_documents, - soft_deletion_used: _, - } = self.execute_inner()?; + let DetailedDocumentDeletionResult { deleted_documents, remaining_documents } = + self.execute_inner()?; Ok(DocumentDeletionResult { deleted_documents, remaining_documents }) } @@ -134,7 +130,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { return Ok(DetailedDocumentDeletionResult { deleted_documents: 0, remaining_documents: 0, - soft_deletion_used: false, }); } @@ -150,7 +145,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { return Ok(DetailedDocumentDeletionResult { deleted_documents: current_documents_ids_len, remaining_documents, - soft_deletion_used: false, }); } @@ -219,7 +213,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { return Ok(DetailedDocumentDeletionResult { deleted_documents: self.to_delete_docids.len(), remaining_documents: documents_ids.len(), - soft_deletion_used: true, }); } @@ -472,7 +465,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { Ok(DetailedDocumentDeletionResult { deleted_documents: self.to_delete_docids.len(), remaining_documents: documents_ids.len(), - soft_deletion_used: false, }) } From 602ad98cb8ceeb90a13067f94821381434722655 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 22 May 2023 11:15:14 +0200 Subject: [PATCH 04/14] improve the way we handle the fsts --- milli/Cargo.toml | 4 ++- milli/src/external_documents_ids.rs | 36 +++++++++++-------- milli/src/update/index_documents/mod.rs | 2 -- milli/src/update/index_documents/transform.rs | 12 +++---- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/milli/Cargo.toml b/milli/Cargo.toml index ea48e008c..f708edc73 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -56,7 +56,6 @@ itertools = "0.10.5" log = "0.4.17" logging_timer = "1.1.0" csv = "1.2.1" -fastrand = "1.9.0" [dev-dependencies] mimalloc = { version = "0.1.29", default-features = false } @@ -65,7 +64,10 @@ insta = "1.29.0" maplit = "1.0.2" md5 = "0.7.0" rand = {version = "0.8.5", features = ["small_rng"] } + +# fuzzing arbitrary = { version = "1.3.0", features = ["derive"] } +fastrand = "1.9.0" [target.'cfg(fuzzing)'.dev-dependencies] fuzzcheck = "0.12.1" diff --git a/milli/src/external_documents_ids.rs b/milli/src/external_documents_ids.rs index 2cecd1abe..36b147336 100644 --- a/milli/src/external_documents_ids.rs +++ b/milli/src/external_documents_ids.rs @@ -106,22 +106,30 @@ impl<'a> ExternalDocumentsIds<'a> { map } + /// Return an fst of the combined hard and soft deleted ID. + pub fn to_fst<'b>(&'b self) -> fst::Result>>> { + if self.soft.is_empty() { + return Ok(Cow::Borrowed(&self.hard)); + } + let union_op = self.hard.op().add(&self.soft).r#union(); + + let mut iter = union_op.into_stream(); + let mut new_hard_builder = fst::MapBuilder::memory(); + while let Some((external_id, marked_docids)) = iter.next() { + let value = indexed_last_value(marked_docids).unwrap(); + if value != DELETED_ID { + new_hard_builder.insert(external_id, value)?; + } + } + + drop(iter); + + Ok(Cow::Owned(new_hard_builder.into_map().map_data(Cow::Owned)?)) + } + fn merge_soft_into_hard(&mut self) -> fst::Result<()> { if self.soft.len() >= self.hard.len() / 2 { - let union_op = self.hard.op().add(&self.soft).r#union(); - - let mut iter = union_op.into_stream(); - let mut new_hard_builder = fst::MapBuilder::memory(); - while let Some((external_id, marked_docids)) = iter.next() { - let value = indexed_last_value(marked_docids).unwrap(); - if value != DELETED_ID { - new_hard_builder.insert(external_id, value)?; - } - } - - drop(iter); - - self.hard = new_hard_builder.into_map().map_data(Cow::Owned)?; + self.hard = self.to_fst()?.into_owned(); self.soft = fst::Map::default().map_data(Cow::Owned)?; } diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 406bfb0c9..70ec377aa 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -198,7 +198,6 @@ where let number_of_documents = self.index.number_of_documents(self.wtxn)?; return Ok(DocumentAdditionResult { indexed_documents: 0, number_of_documents }); } - let output = self .transform .take() @@ -221,7 +220,6 @@ where } let indexed_documents = output.documents_count as u64; - let number_of_documents = self.execute_raw(output)?; Ok(DocumentAdditionResult { indexed_documents, number_of_documents }) diff --git a/milli/src/update/index_documents/transform.rs b/milli/src/update/index_documents/transform.rs index e2a260391..ee6831be5 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/milli/src/update/index_documents/transform.rs @@ -57,8 +57,8 @@ pub struct Transform<'a, 'i> { original_sorter: grenad::Sorter, flattened_sorter: grenad::Sorter, - pub replaced_documents_ids: RoaringBitmap, - pub new_documents_ids: RoaringBitmap, + replaced_documents_ids: RoaringBitmap, + new_documents_ids: RoaringBitmap, // To increase the cache locality and decrease the heap usage we use compact smartstring. new_external_documents_ids_builder: FxHashMap, u64>, documents_count: usize, @@ -653,9 +653,7 @@ impl<'a, 'i> Transform<'a, 'i> { primary_key, fields_ids_map: self.fields_ids_map, field_distribution, - new_external_documents_ids: new_external_documents_ids - .map_data(|c| Cow::Owned(c)) - .unwrap(), + new_external_documents_ids: new_external_documents_ids.map_data(Cow::Owned).unwrap(), new_documents_ids: self.new_documents_ids, replaced_documents_ids: self.replaced_documents_ids, documents_count: self.documents_count, @@ -689,8 +687,8 @@ impl<'a, 'i> Transform<'a, 'i> { let new_external_documents_ids = { let mut external_documents_ids = self.index.external_documents_ids(wtxn)?; external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; - // it is safe to get the hard document IDs - external_documents_ids.into_static().hard + // This call should be free and can't fail since the previous method merged both fsts. + external_documents_ids.into_static().to_fst()?.into_owned() }; let documents_ids = self.index.documents_ids(wtxn)?; From 22213dc6044c32101a6e1b4d15640af71466f469 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 23 May 2023 09:14:26 +0200 Subject: [PATCH 05/14] push the fuzzer --- milli/examples/fuzz.rs | 116 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 milli/examples/fuzz.rs diff --git a/milli/examples/fuzz.rs b/milli/examples/fuzz.rs new file mode 100644 index 000000000..9536f3811 --- /dev/null +++ b/milli/examples/fuzz.rs @@ -0,0 +1,116 @@ +use arbitrary::{Arbitrary, Unstructured}; +use milli::heed::EnvOpenOptions; +use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; +use milli::Index; +use serde_json::{json, Value}; +use tempfile::TempDir; + +#[derive(Debug, Arbitrary)] +enum Document { + One, + Two, + Three, + Four, + Five, + Six, +} + +impl Document { + pub fn to_d(&self) -> Value { + match self { + Document::One => json!({ "id": 0, "doggo": "bernese" }), + Document::Two => json!({ "id": 0, "doggo": "golden" }), + Document::Three => json!({ "id": 0, "catto": "jorts" }), + Document::Four => json!({ "id": 1, "doggo": "bernese" }), + Document::Five => json!({ "id": 1, "doggo": "golden" }), + Document::Six => json!({ "id": 1, "catto": "jorts" }), + } + } +} + +#[derive(Debug, Arbitrary)] +enum DocId { + Zero, + One, +} + +impl DocId { + pub fn to_s(&self) -> String { + match self { + DocId::Zero => "0".to_string(), + DocId::One => "1".to_string(), + } + } +} + +#[derive(Debug, Arbitrary)] +enum Operation { + AddDoc(Document), + DeleteDoc(DocId), +} + +#[derive(Debug, Arbitrary)] +struct Batch([Operation; 2]); + +fn main() { + let mut options = EnvOpenOptions::new(); + options.map_size(1024 * 1024 * 1024 * 1024); + let _tempdir = TempDir::new_in("ramdisk").unwrap(); + let index = Index::new(options, _tempdir.path()).unwrap(); + let indexer_config = IndexerConfig::default(); + let index_documents_config = IndexDocumentsConfig::default(); + + loop { + // let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); + + // let data = Unstructured::new(&v); + // let batches = <[Batch; 3]>::arbitrary(&mut data).unwrap(); + let batches = [ + Batch([Operation::AddDoc(Document::Five), Operation::AddDoc(Document::Three)]), + Batch([Operation::DeleteDoc(DocId::One), Operation::AddDoc(Document::Two)]), + Batch([Operation::DeleteDoc(DocId::Zero), Operation::AddDoc(Document::Five)]), + ]; + + dbg!(&batches); + + let mut wtxn = index.write_txn().unwrap(); + + for batch in batches { + dbg!(&batch); + + let mut builder = IndexDocuments::new( + &mut wtxn, + &index, + &indexer_config, + index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); + + for op in batch.0 { + match op { + Operation::AddDoc(doc) => { + let documents = milli::documents::objects_from_json_value(doc.to_d()); + let documents = + milli::documents::documents_batch_reader_from_objects(documents); + let (b, _added) = builder.add_documents(documents).unwrap(); + builder = b; + } + Operation::DeleteDoc(id) => { + let (b, _removed) = builder.remove_documents(vec![id.to_s()]).unwrap(); + builder = b; + } + } + } + builder.execute().unwrap(); + // wtxn.commit().unwrap(); + + // after executing a batch we check if the database is corrupted + // let rtxn = index.read_txn().unwrap(); + let res = index.search(&wtxn).execute().unwrap(); + index.documents(&wtxn, res.documents_ids).unwrap(); + } + wtxn.abort().unwrap(); + } +} From 002f42875f1f852da092ecdce0a5ae9700ec08ca Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 23 May 2023 11:42:40 +0200 Subject: [PATCH 06/14] fix the fuzzer --- milli/examples/fuzz.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/milli/examples/fuzz.rs b/milli/examples/fuzz.rs index 9536f3811..dc79e37b7 100644 --- a/milli/examples/fuzz.rs +++ b/milli/examples/fuzz.rs @@ -50,34 +50,27 @@ enum Operation { } #[derive(Debug, Arbitrary)] -struct Batch([Operation; 2]); +struct Batch([Operation; 5]); fn main() { let mut options = EnvOpenOptions::new(); options.map_size(1024 * 1024 * 1024 * 1024); - let _tempdir = TempDir::new_in("ramdisk").unwrap(); + let _tempdir = TempDir::new().unwrap(); let index = Index::new(options, _tempdir.path()).unwrap(); let indexer_config = IndexerConfig::default(); let index_documents_config = IndexDocumentsConfig::default(); loop { - // let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); + let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); - // let data = Unstructured::new(&v); - // let batches = <[Batch; 3]>::arbitrary(&mut data).unwrap(); - let batches = [ - Batch([Operation::AddDoc(Document::Five), Operation::AddDoc(Document::Three)]), - Batch([Operation::DeleteDoc(DocId::One), Operation::AddDoc(Document::Two)]), - Batch([Operation::DeleteDoc(DocId::Zero), Operation::AddDoc(Document::Five)]), - ]; + let mut data = Unstructured::new(&v); + let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap(); dbg!(&batches); let mut wtxn = index.write_txn().unwrap(); for batch in batches { - dbg!(&batch); - let mut builder = IndexDocuments::new( &mut wtxn, &index, From 6c6387d05e1fe8a700b274e1703c621cef09f2b4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 12:27:39 +0200 Subject: [PATCH 07/14] move the fuzzer to its own crate --- Cargo.lock | 708 ++++++++++++++---------------- Cargo.toml | 3 +- fuzzers/Cargo.toml | 20 + fuzzers/README.md | 3 + fuzzers/src/bin/fuzz.rs | 136 ++++++ fuzzers/src/lib.rs | 46 ++ milli/Cargo.toml | 7 - milli/{examples => tests}/fuzz.rs | 4 +- 8 files changed, 531 insertions(+), 396 deletions(-) create mode 100644 fuzzers/Cargo.toml create mode 100644 fuzzers/README.md create mode 100644 fuzzers/src/bin/fuzz.rs create mode 100644 fuzzers/src/lib.rs rename milli/{examples => tests}/fuzz.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index a432908a2..83950615a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags", "bytes", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", "tokio", "tokio-util", + "tracing", ] [[package]] @@ -46,7 +46,7 @@ dependencies = [ "actix-tls", "actix-utils", "ahash 0.8.3", - "base64 0.21.0", + "base64 0.21.2", "bitflags", "brotli", "bytes", @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" dependencies = [ "actix-rt", "actix-service", @@ -150,7 +150,7 @@ dependencies = [ "futures-core", "log", "pin-project-lite", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util", "webpki-roots", ] @@ -247,14 +247,13 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ "cfg-if", "cipher", "cpufeatures", - "opaque-debug", ] [[package]] @@ -282,9 +281,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -312,49 +311,58 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", + "colorchoice", "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" dependencies = [ "utf8parse", ] [[package]] -name = "anstyle-wincon" -version = "0.2.0" +name = "anstyle-query" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" dependencies = [ "backtrace", ] @@ -397,7 +405,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -408,7 +416,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -447,8 +455,8 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", - "object 0.30.2", + "miniz_oxide 0.6.2", + "object", "rustc-demangle", ] @@ -460,15 +468,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "base64ct" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "benchmarks" @@ -513,12 +521,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -557,9 +559,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" dependencies = [ "memchr", "once_cell", @@ -569,9 +571,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-unit" @@ -606,7 +608,7 @@ checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -623,9 +625,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bytestring" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ "bytes", ] @@ -727,9 +729,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" dependencies = [ "ciborium-io", "ciborium-ll", @@ -738,15 +740,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" dependencies = [ "ciborium-io", "half", @@ -754,18 +756,19 @@ dependencies = [ [[package]] name = "cipher" -version = "0.3.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "generic-array", + "crypto-common", + "inout", ] [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags", "clap_lex 0.2.4", @@ -775,9 +778,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" dependencies = [ "clap_builder", "clap_derive", @@ -786,27 +789,27 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", "bitflags", - "clap_lex 0.4.1", + "clap_lex 0.5.0", "strsim", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -820,9 +823,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concat-arrays" @@ -835,31 +844,16 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "concolor-override" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] - [[package]] name = "console" -version = "0.15.5" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" dependencies = [ "encode_unicode", "lazy_static", "libc", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -914,9 +908,9 @@ checksum = "79bb3adfaf5f75d24b01aee375f7555907840fa2800e5ec8fa3b9e2031830173" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -940,7 +934,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -1197,9 +1191,9 @@ checksum = "8c1bba4f227a4a53d12b653f50ca7bf10c9119ae2aba56aff9e0338b5c98f36a" [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -1249,7 +1243,7 @@ dependencies = [ "tempfile", "thiserror", "time", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] @@ -1351,22 +1345,22 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "706d9e7cf1c7664859d79cd524e4e53ea2b67ea03c98cc2870c5e539695d597e" +checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355f93763ef7b0ae1c43c4d8eccc9d5848d84ad1a1d8ce61c421d1ac85a19d05" +checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -1382,17 +1376,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - [[package]] name = "errno" version = "0.3.1" @@ -1453,19 +1436,19 @@ dependencies = [ "faux", "tempfile", "thiserror", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] name = "filetime" -version = "0.2.19" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", "redox_syscall 0.2.16", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -1485,13 +1468,12 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "libz-sys", - "miniz_oxide", + "miniz_oxide 0.7.1", ] [[package]] @@ -1579,7 +1561,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -1613,47 +1595,16 @@ dependencies = [ ] [[package]] -name = "fuzzcheck" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee76e8096c3fcd82ab23177edddcc9b81b72c123caab54bb1e2dc19fd09d2dec" +name = "fuzzers" +version = "1.2.0" dependencies = [ - "ahash 0.7.6", - "bit-vec", - "cc", - "cfg-if", + "arbitrary", + "clap 4.3.0", "fastrand", - "flate2", - "fuzzcheck_common", - "fuzzcheck_mutators_derive", - "getopts", - "libc", - "md5", - "nu-ansi-term", - "object 0.27.1", - "regex-syntax", + "milli", "serde", "serde_json", -] - -[[package]] -name = "fuzzcheck_common" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde06f8d25b14a35d43eb2d3dbace3b9193424460b10ad4ccf1b3d542d48f06f" -dependencies = [ - "getopts", -] - -[[package]] -name = "fuzzcheck_mutators_derive" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ce01e8bbb3e7e0758dcf907fe799f5998a54368963f766ae94b84624ba60c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "tempfile", ] [[package]] @@ -1681,20 +1632,11 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36d244a08113319b5ebcabad2b8b7925732d15eec46d7e7ac3c11734f3b7a6ad" -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -1715,9 +1657,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "git2" @@ -1751,9 +1693,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1926,9 +1868,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -1950,15 +1892,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", - "rustls", + "rustls 0.21.1", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", ] [[package]] @@ -2004,7 +1946,7 @@ dependencies = [ "tempfile", "thiserror", "time", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] @@ -2018,6 +1960,15 @@ dependencies = [ "serde", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "insta" version = "1.29.0" @@ -2045,9 +1996,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -2079,7 +2030,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -2124,9 +2075,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -2145,7 +2096,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "pem", "ring", "serde", @@ -2185,9 +2136,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.141" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libgit2-sys" @@ -2203,15 +2154,15 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libmimalloc-sys" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a558e3d911bc3c7bfc8c78bc580b404d6e51c1cefbf656e176a94b49b0df40" +checksum = "f4ac0e912c8ef1b735e92369695618dc5b1819f5a7bf3f167301a3ba1cea515e" dependencies = [ "cc", "libc", @@ -2219,9 +2170,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", "libc", @@ -2460,9 +2411,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lmdb-rkv-sys" @@ -2504,12 +2455,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "logging_timer" @@ -2535,14 +2483,14 @@ dependencies = [ [[package]] name = "manifest-dir-macros" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08150cf2bab1fc47c2196f4f41173a27fcd0f684165e5458c0046b53a472e2f" +checksum = "450e5ef583bc05177c4975b9ea907047091a9f62e74e81fcafb99dbffac51e7e" dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -2586,7 +2534,7 @@ dependencies = [ "byte-unit", "bytes", "cargo_toml", - "clap 4.2.1", + "clap 4.3.0", "crossbeam-channel", "deserr", "dump", @@ -2625,7 +2573,7 @@ dependencies = [ "rayon", "regex", "reqwest", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "segment", "serde", @@ -2647,7 +2595,7 @@ dependencies = [ "tokio-stream", "toml", "urlencoding", - "uuid 1.3.1", + "uuid 1.3.3", "vergen", "walkdir", "yaup", @@ -2658,7 +2606,7 @@ dependencies = [ name = "meilisearch-auth" version = "1.2.0" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "enum-iterator", "hmac", "maplit", @@ -2670,7 +2618,7 @@ dependencies = [ "sha2", "thiserror", "time", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] @@ -2700,7 +2648,7 @@ dependencies = [ "thiserror", "time", "tokio", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] @@ -2731,7 +2679,6 @@ dependencies = [ name = "milli" version = "1.2.0" dependencies = [ - "arbitrary", "big_s", "bimap", "bincode", @@ -2743,11 +2690,9 @@ dependencies = [ "csv", "deserr", "either", - "fastrand", "filter-parser", "flatten-serde-json", "fst", - "fuzzcheck", "fxhash", "geoutils", "grenad", @@ -2778,14 +2723,14 @@ dependencies = [ "tempfile", "thiserror", "time", - "uuid 1.3.1", + "uuid 1.3.3", ] [[package]] name = "mimalloc" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d88dad3f985ec267a3fcb7a1726f5cb1a7e8cad8b646e70a84f967210df23da" +checksum = "4e2894987a3459f3ffb755608bd82188f8ed00d0ae077f1edea29c068d639d98" dependencies = [ "libmimalloc-sys", ] @@ -2822,15 +2767,24 @@ dependencies = [ ] [[package]] -name = "mio" -version = "0.8.6" +name = "miniz_oxide" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2861,23 +2815,13 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e2187cbceeafee9fb7b5e5e182623e0628ebf430a479df4487beb8f92fd7a" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.3" @@ -2921,18 +2865,9 @@ dependencies = [ [[package]] name = "object" -version = "0.27.1" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" -dependencies = [ - "memchr", -] - -[[package]] -name = "object" -version = "0.30.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8c786513eb403643f2a88c244c2aaa270ef2153f55094587d0c48a3cf22a83" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "memchr", ] @@ -2955,17 +2890,11 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "ordered-float" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" +checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" dependencies = [ "num-traits", ] @@ -2976,12 +2905,6 @@ version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "page_size" version = "0.4.2" @@ -3071,9 +2994,9 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" dependencies = [ "base64 0.13.1", ] @@ -3094,9 +3017,9 @@ dependencies = [ [[package]] name = "pest" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1403e8401ad5dedea73c626b99758535b342502f8d1e361f4a2dd952749122" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" dependencies = [ "thiserror", "ucd-trie", @@ -3104,9 +3027,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be99c4c1d2fc2769b1d00239431d711d08f6efedcecb8b6e30707160aee99c15" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" dependencies = [ "pest", "pest_generator", @@ -3114,22 +3037,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56094789873daa36164de2e822b3888c6ae4b4f9da555a1103587658c805b1e" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "pest_meta" -version = "2.5.7" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6733073c7cff3d8459fda0e42f13a047870242aed8b509fe98000928975f359e" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" dependencies = [ "once_cell", "pest", @@ -3194,9 +3117,9 @@ checksum = "3bd12336e3afa34152e002f57df37a7056778daa59ea542b3473b87f5fb260c4" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platform-dirs" @@ -3267,9 +3190,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -3284,7 +3207,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.11", + "rustix 0.36.14", ] [[package]] @@ -3312,9 +3235,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -3402,9 +3325,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -3419,17 +3342,17 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -3446,13 +3369,13 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.1", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", "tower-service", "url", "wasm-bindgen", @@ -3509,9 +3432,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustc_version" @@ -3524,12 +3447,12 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.11" +version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ "bitflags", - "errno 0.2.8", + "errno", "io-lifetimes", "libc", "linux-raw-sys 0.1.4", @@ -3538,15 +3461,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", - "errno 0.3.1", + "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.3.1", + "linux-raw-sys 0.3.8", "windows-sys 0.48.0", ] @@ -3562,13 +3485,35 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -3630,9 +3575,9 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] @@ -3648,20 +3593,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "indexmap", "itoa", @@ -3671,9 +3616,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" dependencies = [ "serde", ] @@ -3767,9 +3712,9 @@ dependencies = [ [[package]] name = "slice-group-by" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallstr" @@ -3854,9 +3799,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -3871,9 +3816,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.14" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -3929,9 +3874,9 @@ dependencies = [ [[package]] name = "temp-env" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95b343d943e5a0d2221fb73029e8040f3c91d6d06afec86c664682a361681" +checksum = "9547444bfe52cbd79515c6c8087d8ae6ca8d64d2d31a27746320f5cb81d1a15c" dependencies = [ "parking_lot", ] @@ -3945,7 +3890,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.45.0", ] @@ -3981,14 +3926,14 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "time" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "serde", @@ -3998,15 +3943,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -4038,9 +3983,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -4052,18 +3997,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -4072,16 +4017,26 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] [[package]] -name = "tokio-stream" -version = "0.1.12" +name = "tokio-rustls" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +dependencies = [ + "rustls 0.21.1", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -4090,9 +4045,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -4104,9 +4059,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", "serde_spanned", @@ -4116,18 +4071,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ "indexmap", "serde", @@ -4156,9 +4111,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -4198,15 +4153,15 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-blocks" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de2be6bad6f56ce8373d377e611cbb2265de3a656138065609ce82e217aad70" +checksum = "943e3f1f50cc455d072e0801ccb71ff893b0c88060b1169f92e35fb5bb881cc6" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -4223,12 +4178,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - [[package]] name = "unicode-xid" version = "0.2.4" @@ -4281,9 +4230,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ "getrandom", "serde", @@ -4356,9 +4305,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4366,24 +4315,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if", "js-sys", @@ -4393,9 +4342,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4403,28 +4352,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -4490,21 +4439,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.45.0" @@ -4639,9 +4573,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] @@ -4712,9 +4646,9 @@ dependencies = [ [[package]] name = "zip" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "aes", "byteorder", @@ -4745,7 +4679,7 @@ version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe 6.0.4+zstd.1.5.4", + "zstd-safe 6.0.5+zstd.1.5.4", ] [[package]] @@ -4760,9 +4694,9 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "6.0.5+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" dependencies = [ "libc", "zstd-sys", @@ -4770,9 +4704,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index b69831b9c..f7e5758d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,8 @@ members = [ "filter-parser", "flatten-serde-json", "json-depth-checker", - "benchmarks" + "benchmarks", + "fuzzers", ] [workspace.package] diff --git a/fuzzers/Cargo.toml b/fuzzers/Cargo.toml new file mode 100644 index 000000000..cbc27a55c --- /dev/null +++ b/fuzzers/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fuzzers" +publish = false + +version.workspace = true +authors.workspace = true +description.workspace = true +homepage.workspace = true +readme.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +arbitrary = { version = "1.3.0", features = ["derive"] } +clap = { version = "4.3.0", features = ["derive"] } +fastrand = "1.9.0" +milli = { path = "../milli" } +serde = { version = "1.0.160", features = ["derive"] } +serde_json = { version = "1.0.95", features = ["preserve_order"] } +tempfile = "3.5.0" diff --git a/fuzzers/README.md b/fuzzers/README.md new file mode 100644 index 000000000..d9d02de0e --- /dev/null +++ b/fuzzers/README.md @@ -0,0 +1,3 @@ +# Fuzzers + +The purpose of this crate is to contains all the handmade "fuzzer" we may need. diff --git a/fuzzers/src/bin/fuzz.rs b/fuzzers/src/bin/fuzz.rs new file mode 100644 index 000000000..880c7c452 --- /dev/null +++ b/fuzzers/src/bin/fuzz.rs @@ -0,0 +1,136 @@ +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use arbitrary::{Arbitrary, Unstructured}; +use clap::Parser; +use fuzzers::Operation; +use milli::heed::EnvOpenOptions; +use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; +use milli::Index; +use tempfile::TempDir; + +#[derive(Debug, Arbitrary)] +struct Batch([Operation; 5]); + +#[derive(Debug, Clone, Parser)] +struct Opt { + /// The number of fuzzer to run in parallel. + #[clap(long)] + par: Option, + // We need to put a lot of newlines in the following documentation or else everything gets collapsed on one line + /// The path in which the databases will be created. + /// Using a ramdisk is recommended. + /// + /// Linux: + /// + /// sudo mount -t tmpfs -o size=2g tmpfs ramdisk # to create it + /// + /// sudo umount ramdisk # to remove it + /// + /// MacOS: + /// + /// diskutil erasevolume HFS+ 'RAM Disk' `hdiutil attach -nobrowse -nomount ram://4194304 # create it + /// + /// hdiutil detach /dev/:the_disk + /// + #[clap(long)] + path: Option, +} + +fn main() { + let opt = Opt::parse(); + let progression: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0))); + + let par = opt.par.unwrap_or_else(|| std::thread::available_parallelism().unwrap()).get(); + let mut handles = Vec::with_capacity(par); + + for _ in 0..par { + let opt = opt.clone(); + + let handle = std::thread::spawn(move || { + let mut options = EnvOpenOptions::new(); + options.map_size(1024 * 1024 * 1024 * 1024); + let tempdir = match opt.path { + Some(path) => TempDir::new_in(path).unwrap(), + None => TempDir::new().unwrap(), + }; + let index = Index::new(options, tempdir.path()).unwrap(); + let indexer_config = IndexerConfig::default(); + let index_documents_config = IndexDocumentsConfig::default(); + + loop { + let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); + + let mut data = Unstructured::new(&v); + let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap(); + // will be used to display the error once a thread crashes + let dbg_input = format!("{:#?}", batches); + + let mut wtxn = index.write_txn().unwrap(); + + for batch in batches { + let mut builder = IndexDocuments::new( + &mut wtxn, + &index, + &indexer_config, + index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); + + for op in batch.0 { + match op { + Operation::AddDoc(doc) => { + let documents = + milli::documents::objects_from_json_value(doc.to_d()); + let documents = + milli::documents::documents_batch_reader_from_objects( + documents, + ); + let (b, _added) = + builder.add_documents(documents).expect(&dbg_input); + builder = b; + } + Operation::DeleteDoc(id) => { + let (b, _removed) = + builder.remove_documents(vec![id.to_s()]).unwrap(); + builder = b; + } + } + } + builder.execute().expect(&dbg_input); + + // after executing a batch we check if the database is corrupted + let res = index.search(&wtxn).execute().expect(&dbg_input); + index.documents(&wtxn, res.documents_ids).expect(&dbg_input); + progression.fetch_add(1, Ordering::Relaxed); + } + wtxn.abort().unwrap(); + } + }); + handles.push(handle); + } + + std::thread::spawn(|| { + let mut last_value = 0; + let start = std::time::Instant::now(); + loop { + let total = progression.load(Ordering::Relaxed); + println!( + "Has been running for {:?}. Tested {} new values for a total of {}.", + start.elapsed(), + total - last_value, + total + ); + last_value = total; + std::thread::sleep(Duration::from_secs(1)); + } + }); + + for handle in handles { + handle.join().unwrap(); + } +} diff --git a/fuzzers/src/lib.rs b/fuzzers/src/lib.rs new file mode 100644 index 000000000..c0eef38e8 --- /dev/null +++ b/fuzzers/src/lib.rs @@ -0,0 +1,46 @@ +use arbitrary::Arbitrary; +use serde_json::{json, Value}; + +#[derive(Debug, Arbitrary)] +pub enum Document { + One, + Two, + Three, + Four, + Five, + Six, +} + +impl Document { + pub fn to_d(&self) -> Value { + match self { + Document::One => json!({ "id": 0, "doggo": "bernese" }), + Document::Two => json!({ "id": 0, "doggo": "golden" }), + Document::Three => json!({ "id": 0, "catto": "jorts" }), + Document::Four => json!({ "id": 1, "doggo": "bernese" }), + Document::Five => json!({ "id": 1, "doggo": "golden" }), + Document::Six => json!({ "id": 1, "catto": "jorts" }), + } + } +} + +#[derive(Debug, Arbitrary)] +pub enum DocId { + Zero, + One, +} + +impl DocId { + pub fn to_s(&self) -> String { + match self { + DocId::Zero => "0".to_string(), + DocId::One => "1".to_string(), + } + } +} + +#[derive(Debug, Arbitrary)] +pub enum Operation { + AddDoc(Document), + DeleteDoc(DocId), +} diff --git a/milli/Cargo.toml b/milli/Cargo.toml index f708edc73..acd1d35c4 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -65,13 +65,6 @@ maplit = "1.0.2" md5 = "0.7.0" rand = {version = "0.8.5", features = ["small_rng"] } -# fuzzing -arbitrary = { version = "1.3.0", features = ["derive"] } -fastrand = "1.9.0" - -[target.'cfg(fuzzing)'.dev-dependencies] -fuzzcheck = "0.12.1" - [features] all-tokenizations = ["charabia/default"] diff --git a/milli/examples/fuzz.rs b/milli/tests/fuzz.rs similarity index 99% rename from milli/examples/fuzz.rs rename to milli/tests/fuzz.rs index dc79e37b7..3ecc151ad 100644 --- a/milli/examples/fuzz.rs +++ b/milli/tests/fuzz.rs @@ -52,7 +52,9 @@ enum Operation { #[derive(Debug, Arbitrary)] struct Batch([Operation; 5]); -fn main() { +#[test] +#[ignore] +fn fuzz() { let mut options = EnvOpenOptions::new(); options.map_size(1024 * 1024 * 1024 * 1024); let _tempdir = TempDir::new().unwrap(); From 8d40d300a5dc4aac21f0f847ff379cd2ad9a595b Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 12:37:24 +0200 Subject: [PATCH 08/14] rename the fuzzer to indexing --- fuzzers/src/bin/{fuzz.rs => indexing.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fuzzers/src/bin/{fuzz.rs => indexing.rs} (100%) diff --git a/fuzzers/src/bin/fuzz.rs b/fuzzers/src/bin/indexing.rs similarity index 100% rename from fuzzers/src/bin/fuzz.rs rename to fuzzers/src/bin/indexing.rs From 99e9057684853302f85707a5838fd7c85f22cf86 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 13:07:06 +0200 Subject: [PATCH 09/14] rename the indexing fuzzer to fuzz-indexing so it doesn't collide with other binary name when being called from the root of the workspace --- fuzzers/src/bin/{indexing.rs => fuzz-indexing.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fuzzers/src/bin/{indexing.rs => fuzz-indexing.rs} (100%) diff --git a/fuzzers/src/bin/indexing.rs b/fuzzers/src/bin/fuzz-indexing.rs similarity index 100% rename from fuzzers/src/bin/indexing.rs rename to fuzzers/src/bin/fuzz-indexing.rs From 67a583bedfb312454bad6684d6ae4fc5dc69965d Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 13:39:26 +0200 Subject: [PATCH 10/14] handle the panic happening in milli --- fuzzers/src/bin/fuzz-indexing.rs | 92 +++++++++++++++++--------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/fuzzers/src/bin/fuzz-indexing.rs b/fuzzers/src/bin/fuzz-indexing.rs index 880c7c452..6d8ad61ed 100644 --- a/fuzzers/src/bin/fuzz-indexing.rs +++ b/fuzzers/src/bin/fuzz-indexing.rs @@ -34,7 +34,6 @@ struct Opt { /// diskutil erasevolume HFS+ 'RAM Disk' `hdiutil attach -nobrowse -nomount ram://4194304 # create it /// /// hdiutil detach /dev/:the_disk - /// #[clap(long)] path: Option, } @@ -60,56 +59,61 @@ fn main() { let indexer_config = IndexerConfig::default(); let index_documents_config = IndexDocumentsConfig::default(); - loop { - let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); + std::thread::scope(|s| { + loop { + let v: Vec = + std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); - let mut data = Unstructured::new(&v); - let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap(); - // will be used to display the error once a thread crashes - let dbg_input = format!("{:#?}", batches); + let mut data = Unstructured::new(&v); + let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap(); + // will be used to display the error once a thread crashes + let dbg_input = format!("{:#?}", batches); - let mut wtxn = index.write_txn().unwrap(); + let handle = s.spawn(|| { + let mut wtxn = index.write_txn().unwrap(); - for batch in batches { - let mut builder = IndexDocuments::new( - &mut wtxn, - &index, - &indexer_config, - index_documents_config.clone(), - |_| (), - || false, - ) - .unwrap(); + for batch in batches { + let mut builder = IndexDocuments::new( + &mut wtxn, + &index, + &indexer_config, + index_documents_config.clone(), + |_| (), + || false, + ) + .unwrap(); - for op in batch.0 { - match op { - Operation::AddDoc(doc) => { - let documents = - milli::documents::objects_from_json_value(doc.to_d()); - let documents = - milli::documents::documents_batch_reader_from_objects( - documents, - ); - let (b, _added) = - builder.add_documents(documents).expect(&dbg_input); - builder = b; - } - Operation::DeleteDoc(id) => { - let (b, _removed) = - builder.remove_documents(vec![id.to_s()]).unwrap(); - builder = b; + for op in batch.0 { + match op { + Operation::AddDoc(doc) => { + let documents = + milli::documents::objects_from_json_value(doc.to_d()); + let documents = + milli::documents::documents_batch_reader_from_objects( + documents, + ); + let (b, _added) = builder.add_documents(documents).unwrap(); + builder = b; + } + Operation::DeleteDoc(id) => { + let (b, _removed) = + builder.remove_documents(vec![id.to_s()]).unwrap(); + builder = b; + } + } } + builder.execute().unwrap(); + + // after executing a batch we check if the database is corrupted + let res = index.search(&wtxn).execute().unwrap(); + index.documents(&wtxn, res.documents_ids).unwrap(); + progression.fetch_add(1, Ordering::Relaxed); } - } - builder.execute().expect(&dbg_input); - - // after executing a batch we check if the database is corrupted - let res = index.search(&wtxn).execute().expect(&dbg_input); - index.documents(&wtxn, res.documents_ids).expect(&dbg_input); - progression.fetch_add(1, Ordering::Relaxed); + wtxn.abort().unwrap(); + }); + handle.join().expect(&dbg_input); } - wtxn.abort().unwrap(); - } + }); }); handles.push(handle); } From 46fa99f486fc328357cc5317723fb5787711f2ee Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 13:44:32 +0200 Subject: [PATCH 11/14] make the fuzzer stops if an error occurs --- fuzzers/src/bin/fuzz-indexing.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fuzzers/src/bin/fuzz-indexing.rs b/fuzzers/src/bin/fuzz-indexing.rs index 6d8ad61ed..4c6a55575 100644 --- a/fuzzers/src/bin/fuzz-indexing.rs +++ b/fuzzers/src/bin/fuzz-indexing.rs @@ -1,6 +1,6 @@ use std::num::NonZeroUsize; use std::path::PathBuf; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use arbitrary::{Arbitrary, Unstructured}; @@ -41,6 +41,7 @@ struct Opt { fn main() { let opt = Opt::parse(); let progression: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0))); + let stop: &'static AtomicBool = Box::leak(Box::new(AtomicBool::new(false))); let par = opt.par.unwrap_or_else(|| std::thread::available_parallelism().unwrap()).get(); let mut handles = Vec::with_capacity(par); @@ -61,6 +62,9 @@ fn main() { std::thread::scope(|s| { loop { + if stop.load(Ordering::Relaxed) { + return; + } let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); @@ -111,7 +115,10 @@ fn main() { } wtxn.abort().unwrap(); }); - handle.join().expect(&dbg_input); + if let err @ Err(_) = handle.join() { + stop.store(true, Ordering::Relaxed); + err.expect(&dbg_input); + } } }); }); From 23a5b45ebf9e0902fe6f6598598ab0915abfd21b Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 14:02:37 +0200 Subject: [PATCH 12/14] drop the old fuzz file --- milli/tests/fuzz.rs | 111 -------------------------------------------- 1 file changed, 111 deletions(-) delete mode 100644 milli/tests/fuzz.rs diff --git a/milli/tests/fuzz.rs b/milli/tests/fuzz.rs deleted file mode 100644 index 3ecc151ad..000000000 --- a/milli/tests/fuzz.rs +++ /dev/null @@ -1,111 +0,0 @@ -use arbitrary::{Arbitrary, Unstructured}; -use milli::heed::EnvOpenOptions; -use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; -use milli::Index; -use serde_json::{json, Value}; -use tempfile::TempDir; - -#[derive(Debug, Arbitrary)] -enum Document { - One, - Two, - Three, - Four, - Five, - Six, -} - -impl Document { - pub fn to_d(&self) -> Value { - match self { - Document::One => json!({ "id": 0, "doggo": "bernese" }), - Document::Two => json!({ "id": 0, "doggo": "golden" }), - Document::Three => json!({ "id": 0, "catto": "jorts" }), - Document::Four => json!({ "id": 1, "doggo": "bernese" }), - Document::Five => json!({ "id": 1, "doggo": "golden" }), - Document::Six => json!({ "id": 1, "catto": "jorts" }), - } - } -} - -#[derive(Debug, Arbitrary)] -enum DocId { - Zero, - One, -} - -impl DocId { - pub fn to_s(&self) -> String { - match self { - DocId::Zero => "0".to_string(), - DocId::One => "1".to_string(), - } - } -} - -#[derive(Debug, Arbitrary)] -enum Operation { - AddDoc(Document), - DeleteDoc(DocId), -} - -#[derive(Debug, Arbitrary)] -struct Batch([Operation; 5]); - -#[test] -#[ignore] -fn fuzz() { - let mut options = EnvOpenOptions::new(); - options.map_size(1024 * 1024 * 1024 * 1024); - let _tempdir = TempDir::new().unwrap(); - let index = Index::new(options, _tempdir.path()).unwrap(); - let indexer_config = IndexerConfig::default(); - let index_documents_config = IndexDocumentsConfig::default(); - - loop { - let v: Vec = std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect(); - - let mut data = Unstructured::new(&v); - let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap(); - - dbg!(&batches); - - let mut wtxn = index.write_txn().unwrap(); - - for batch in batches { - let mut builder = IndexDocuments::new( - &mut wtxn, - &index, - &indexer_config, - index_documents_config.clone(), - |_| (), - || false, - ) - .unwrap(); - - for op in batch.0 { - match op { - Operation::AddDoc(doc) => { - let documents = milli::documents::objects_from_json_value(doc.to_d()); - let documents = - milli::documents::documents_batch_reader_from_objects(documents); - let (b, _added) = builder.add_documents(documents).unwrap(); - builder = b; - } - Operation::DeleteDoc(id) => { - let (b, _removed) = builder.remove_documents(vec![id.to_s()]).unwrap(); - builder = b; - } - } - } - builder.execute().unwrap(); - // wtxn.commit().unwrap(); - - // after executing a batch we check if the database is corrupted - // let rtxn = index.read_txn().unwrap(); - let res = index.search(&wtxn).execute().unwrap(); - index.documents(&wtxn, res.documents_ids).unwrap(); - } - wtxn.abort().unwrap(); - } -} From f03d99690da2c8f07bddaff90b0782e05771d3e0 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 29 May 2023 14:56:15 +0200 Subject: [PATCH 13/14] run the indexing fuzzer on every merge for as long as possible --- .github/workflows/fuzzer-indexing.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/fuzzer-indexing.yml diff --git a/.github/workflows/fuzzer-indexing.yml b/.github/workflows/fuzzer-indexing.yml new file mode 100644 index 000000000..1d01a6ea5 --- /dev/null +++ b/.github/workflows/fuzzer-indexing.yml @@ -0,0 +1,24 @@ +name: Run the indexing fuzzer + +on: + push: + branches: + - main + +jobs: + fuzz: + name: Setup the action + runs-on: ubuntu-latest + timeout-minutes: 4320 # 72h + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Run benchmarks + - name: Run the fuzzer + run: | + cargo run --release --bin fuzz-indexing From 4e81445d42c4bde0f157d7595030161f18c81f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Mon, 12 Jun 2023 15:30:51 +0200 Subject: [PATCH 14/14] Stop the fuzzer after an hour --- fuzzers/src/bin/fuzz-indexing.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fuzzers/src/bin/fuzz-indexing.rs b/fuzzers/src/bin/fuzz-indexing.rs index 4c6a55575..1d53e069c 100644 --- a/fuzzers/src/bin/fuzz-indexing.rs +++ b/fuzzers/src/bin/fuzz-indexing.rs @@ -130,9 +130,14 @@ fn main() { let start = std::time::Instant::now(); loop { let total = progression.load(Ordering::Relaxed); + let elapsed = start.elapsed().as_secs(); + if elapsed > 3600 { + // after 1 hour, stop the fuzzer, success + std::process::exit(0); + } println!( - "Has been running for {:?}. Tested {} new values for a total of {}.", - start.elapsed(), + "Has been running for {:?} seconds. Tested {} new values for a total of {}.", + elapsed, total - last_value, total );