searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 12:54:26 +01:00
Code Issues Releases Wiki Activity

Continue implementation of task deletion

Browse Source 
1. Matched tasks are a roaring bitmap
2. Start implementation in meilisearch-http
3. Snapshots use meili-snap
4. Rename to TaskDeletion
This commit is contained in:
Loïc Lecrenier 2022-10-13 11:09:00 +02:00 committed by Clément Renault
parent e4d461ecba
commit 9522b75454
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
27 changed files with 290 additions and 456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
Cargo.lock generated
View File

@ -1331,16 +1331,7 @@ dependencies = [
[[package]] [[package]]
name = "filter-parser" name = "filter-parser"
version = "0.33.4" version = "0.33.4"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.33.4#4fc6331cb6526c07f3137584564cfe3493fb25bd" source = "git+https://github.com/meilisearch/milli.git?branch=indexation-abortion#fc03e536153d61da3224698f34fb8c6ee2312c2f"
dependencies = [
"nom",
"nom_locate",
]
[[package]]
name = "filter-parser"
version = "0.34.0"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.34.0#2bf867982ab548a6d749c7534f69b44d3552ef70"
dependencies = [ dependencies = [
"nom", "nom",
"nom_locate", "nom_locate",
@ -1359,15 +1350,7 @@ dependencies = [
[[package]] [[package]]
name = "flatten-serde-json" name = "flatten-serde-json"
version = "0.33.4" version = "0.33.4"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.33.4#4fc6331cb6526c07f3137584564cfe3493fb25bd" source = "git+https://github.com/meilisearch/milli.git?branch=indexation-abortion#fc03e536153d61da3224698f34fb8c6ee2312c2f"
dependencies = [
"serde_json",
]
[[package]]
name = "flatten-serde-json"
version = "0.34.0"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.34.0#2bf867982ab548a6d749c7534f69b44d3552ef70"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -1501,12 +1484,6 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "geoutils"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e006f616a407d396ace1d2ebb3f43ed73189db8b098079bd129928d7645dd1e"
[[package]] [[package]]
name = "geoutils" name = "geoutils"
version = "0.5.1" version = "0.5.1"
@ -1799,9 +1776,10 @@ dependencies = [
"file-store", "file-store",
"insta", "insta",
"log", "log",
"meili-snap",
"meilisearch-types", "meilisearch-types",
"nelson", "nelson",
"roaring 0.9.0", "roaring",
"serde", "serde",
"serde_json", "serde_json",
"synchronoise", "synchronoise",
@ -1910,15 +1888,7 @@ dependencies = [
[[package]] [[package]]
name = "json-depth-checker" name = "json-depth-checker"
version = "0.33.4" version = "0.33.4"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.33.4#4fc6331cb6526c07f3137584564cfe3493fb25bd" source = "git+https://github.com/meilisearch/milli.git?branch=indexation-abortion#fc03e536153d61da3224698f34fb8c6ee2312c2f"
dependencies = [
"serde_json",
]
[[package]]
name = "json-depth-checker"
version = "0.34.0"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.34.0#2bf867982ab548a6d749c7534f69b44d3552ef70"
dependencies = [ dependencies = [
"serde_json", "serde_json",
] ]
@ -2285,7 +2255,6 @@ dependencies = [
"enum-iterator 1.1.3", "enum-iterator 1.1.3",
"hmac", "hmac",
"meilisearch-types", "meilisearch-types",
"milli 0.34.0",
"rand", "rand",
"serde", "serde",
"serde_json", "serde_json",
@ -2386,9 +2355,10 @@ dependencies = [
"enum-iterator 0.7.0", "enum-iterator 0.7.0",
"insta", "insta",
"meili-snap", "meili-snap",
"milli 0.33.4", "milli",
"proptest", "proptest",
"proptest-derive", "proptest-derive",
"roaring",
"serde", "serde",
"serde_json", "serde_json",
"thiserror", "thiserror",
@ -2424,52 +2394,7 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "0.33.4" version = "0.33.4"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.33.4#4fc6331cb6526c07f3137584564cfe3493fb25bd" source = "git+https://github.com/meilisearch/milli.git?branch=indexation-abortion#fc03e536153d61da3224698f34fb8c6ee2312c2f"
dependencies = [
"bimap",
"bincode",
"bstr 0.2.17",
"byteorder",
"charabia",
"concat-arrays",
"crossbeam-channel",
"csv",
"either",
"filter-parser 0.33.4",
"flatten-serde-json 0.33.4",
"fst",
"fxhash",
"geoutils 0.4.1",
"grenad",
"heed",
"itertools",
"json-depth-checker 0.33.4",
"levenshtein_automata",
"log",
"logging_timer",
"memmap2",
"obkv",
"once_cell",
"ordered-float 2.10.0",
"rayon",
"roaring 0.9.0",
"rstar",
"serde",
"serde_json",
"slice-group-by",
"smallstr",
"smallvec",
"smartstring",
"tempfile",
"thiserror",
"time",
"uuid 1.1.2",
]
[[package]]
name = "milli"
version = "0.34.0"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.34.0#2bf867982ab548a6d749c7534f69b44d3552ef70"
dependencies = [ dependencies = [
"bimap", "bimap",
"bincode", "bincode",
@ -2480,24 +2405,24 @@ dependencies = [
"crossbeam-channel", "crossbeam-channel",
"csv", "csv",
"either", "either",
"filter-parser 0.34.0", "filter-parser",
"flatten-serde-json 0.34.0", "flatten-serde-json",
"fst", "fst",
"fxhash", "fxhash",
"geoutils 0.5.1", "geoutils",
"grenad", "grenad",
"heed", "heed",
"itertools", "itertools",
"json-depth-checker 0.34.0", "json-depth-checker",
"levenshtein_automata", "levenshtein_automata",
"log", "log",
"logging_timer", "logging_timer",
"memmap2", "memmap2",
"obkv", "obkv",
"once_cell", "once_cell",
"ordered-float 3.3.0", "ordered-float",
"rayon", "rayon",
"roaring 0.10.1", "roaring",
"rstar", "rstar",
"serde", "serde",
"serde_json", "serde_json",
@ -2690,15 +2615,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "ordered-float"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "3.3.0" version = "3.3.0"
@ -3264,17 +3180,6 @@ dependencies = [
"regex", "regex",
] ]
[[package]]
name = "roaring"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd539cab4e32019956fe7e0cf160bb6d4802f4be2b52c4253d76d3bb0f85a5f7"
dependencies = [
"bytemuck",
"byteorder",
"retain_mut",
]
[[package]] [[package]]
name = "roaring" name = "roaring"
version = "0.10.1" version = "0.10.1"
@ -3284,6 +3189,7 @@ dependencies = [
"bytemuck", "bytemuck",
"byteorder", "byteorder",
"retain_mut", "retain_mut",
"serde",
] ]
[[package]] [[package]]

3
index-scheduler/Cargo.toml
View File

@ -12,7 +12,7 @@ csv = "1.1.6"
file-store = { path = "../file-store" } file-store = { path = "../file-store" }
log = "0.4.14" log = "0.4.14"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
roaring = "0.9.0" roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] } serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] } serde_json = { version = "1.0.85", features = ["preserve_order"] }
tempfile = "3.3.0" tempfile = "3.3.0"
@ -27,3 +27,4 @@ crossbeam = "0.8.2"
nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}
insta = { version = "1.19.1", features = ["json", "redactions"] } insta = { version = "1.19.1", features = ["json", "redactions"] }
big_s = "1.0.2" big_s = "1.0.2"
meili-snap = { path = "../meili-snap" }

150
index-scheduler/src/autobatcher.rs
View File

@ -23,7 +23,7 @@ enum AutobatchKind {
IndexUpdate, IndexUpdate,
IndexSwap, IndexSwap,
CancelTask, CancelTask,
DeleteTasks, TaskDeletion,
DumpExport, DumpExport,
Snapshot, Snapshot,
} }
@ -63,7 +63,7 @@ impl From<KindWithContent> for AutobatchKind {
KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate, KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate,
KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap, KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap,
KindWithContent::CancelTask { .. } => AutobatchKind::CancelTask, KindWithContent::CancelTask { .. } => AutobatchKind::CancelTask,
KindWithContent::DeleteTasks { .. } => AutobatchKind::DeleteTasks, KindWithContent::TaskDeletion { .. } => AutobatchKind::TaskDeletion,
KindWithContent::DumpExport { .. } => AutobatchKind::DumpExport, KindWithContent::DumpExport { .. } => AutobatchKind::DumpExport,
KindWithContent::Snapshot => AutobatchKind::Snapshot, KindWithContent::Snapshot => AutobatchKind::Snapshot,
} }
@ -153,7 +153,7 @@ impl BatchKind {
allow_index_creation, allow_index_creation,
settings_ids: vec![task_id], settings_ids: vec![task_id],
}), }),
K::DumpExport | K::Snapshot | K::CancelTask | K::DeleteTasks => { K::DumpExport | K::Snapshot | K::CancelTask | K::TaskDeletion => {
unreachable!() unreachable!()
} }
} }
@ -378,7 +378,7 @@ impl BatchKind {
import_ids, import_ids,
}) })
} }
(_, K::CancelTask | K::DeleteTasks | K::DumpExport | K::Snapshot) => { (_, K::CancelTask | K::TaskDeletion | K::DumpExport | K::Snapshot) => {
unreachable!() unreachable!()
} }
( (
@ -414,7 +414,7 @@ pub fn autobatch(enqueued: Vec<(TaskId, KindWithContent)>) -> Option<BatchKind>
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::assert_smol_debug_snapshot; use crate::debug_snapshot;
use super::*; use super::*;
use uuid::Uuid; use uuid::Uuid;
@ -492,129 +492,129 @@ mod tests {
#[test] #[test]
fn autobatch_simple_operation_together() { fn autobatch_simple_operation_together() {
// we can autobatch one or multiple DocumentAddition together // we can autobatch one or multiple DocumentAddition together
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp( ReplaceDocuments, true ), doc_imp(ReplaceDocuments, true )]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp( ReplaceDocuments, true ), doc_imp(ReplaceDocuments, true )]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
// we can autobatch one or multiple DocumentUpdate together // we can autobatch one or multiple DocumentUpdate together
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
// we can autobatch one or multiple DocumentDeletion together // we can autobatch one or multiple DocumentDeletion together
assert_smol_debug_snapshot!(autobatch_from([doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_del(), doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_del(), doc_del(), doc_del()]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })");
// we can autobatch one or multiple Settings together // we can autobatch one or multiple Settings together
assert_smol_debug_snapshot!(autobatch_from([settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })"); debug_snapshot!(autobatch_from([settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([settings(true), settings(true), settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([settings(true), settings(true), settings(true)]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })");
} }
#[test] #[test]
fn simple_document_operation_dont_autobatch_with_other() { fn simple_document_operation_dont_autobatch_with_other() {
// addition, updates and deletion can't batch together // addition, updates and deletion can't batch together
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(UpdateDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_del()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_del()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_del()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_del()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_create()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_create()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_create()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_create()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_create()]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del(), idx_create()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_update()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_update()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_update()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_update()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_update()]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del(), idx_update()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_swap()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_swap()]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_swap()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_swap()]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_swap()]), @"Some(DocumentDeletion { deletion_ids: [0] })"); debug_snapshot!(autobatch_from([doc_del(), idx_swap()]), @"Some(DocumentDeletion { deletion_ids: [0] })");
} }
#[test] #[test]
fn document_addition_batch_with_settings() { fn document_addition_batch_with_settings() {
// simple case // simple case
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
// multiple settings and doc addition // multiple settings and doc addition
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), settings(true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
// addition and setting unordered // addition and setting unordered
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0, 2] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(UpdateDocuments, true), settings(true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0, 2] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(UpdateDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 2] })");
// We ensure this kind of batch doesn't batch with forbidden operations // We ensure this kind of batch doesn't batch with forbidden operations
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(UpdateDocuments, true)]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_imp(UpdateDocuments, true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(ReplaceDocuments, true)]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_imp(ReplaceDocuments, true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_del()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_del()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_del()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_del()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_create()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_create()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_create()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_create()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_update()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_update()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_update()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_update()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_swap()]), @"Some(settingsAnddoc_im(Repl)eDocuments)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_swap()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_swap()]), @"Some(settingsAnddoc_im(Upda)Documents)allow_index_creation: true, import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_swap()]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
} }
#[test] #[test]
fn clear_and_additions() { fn clear_and_additions() {
// these two doesn't need to batch // these two doesn't need to batch
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(doc_clr() { ids: [0] })"); debug_snapshot!(autobatch_from([doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentClear { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(doc_clr() { ids: [0] })"); debug_snapshot!(autobatch_from([doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentClear { ids: [0] })");
// Basic use case // Basic use case
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2] })");
// This batch kind doesn't mix with other document addition // This batch kind doesn't mix with other document addition
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(doc_clr() { ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentClear { ids: [0, 1, 2] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(doc_clr() { ids: [0, 1, 2] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_imp(UpdateDocuments, true)]), @"Some(DocumentClear { ids: [0, 1, 2] })");
// But you can batch multiple clear together // But you can batch multiple clear together
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2, 3, 4] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(doc_clr() { ids: [0, 1, 2, 3, 4] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), doc_imp(UpdateDocuments, true), doc_clr(), doc_clr(), doc_clr()]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
} }
#[test] #[test]
fn clear_and_additions_and_settings() { fn clear_and_additions_and_settings() {
// A clear don't need to autobatch the settings that happens AFTER there is no documents // A clear don't need to autobatch the settings that happens AFTER there is no documents
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), settings(true)]), @"Some(doc_clr() { ids: [0] })"); debug_snapshot!(autobatch_from([doc_clr(), settings(true)]), @"Some(DocumentClear { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([settings(true), doc_clr(), settings(true)]), @"Some(clearAndSettings([1) allow_index_creation: true, settings_ids: [0, 2] })"); debug_snapshot!(autobatch_from([settings(true), doc_clr(), settings(true)]), @"Some(ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr()]), @"Some(clearAndSettings([0)2], allow_index_creation: true, settings_ids: [1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr()]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr()]), @"Some(clearAndSettings([0)2], allow_index_creation: true, settings_ids: [1] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr()]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
} }
#[test] #[test]
fn anything_and_index_deletion() { fn anything_and_index_deletion() {
// The indexdeletion doesn't batch with anything that happens AFTER // The indexdeletion doesn't batch with anything that happens AFTER
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_imp(ReplaceDocuments, true)]), @"Some(idx_del() { ids: [0] })"); debug_snapshot!(autobatch_from([idx_del(), doc_imp(ReplaceDocuments, true)]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_imp(UpdateDocuments, true)]), @"Some(idx_del() { ids: [0] })"); debug_snapshot!(autobatch_from([idx_del(), doc_imp(UpdateDocuments, true)]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_del()]), @"Some(idx_del() { ids: [0] })"); debug_snapshot!(autobatch_from([idx_del(), doc_del()]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([idx_del(), doc_clr()]), @"Some(idx_del() { ids: [0] })"); debug_snapshot!(autobatch_from([idx_del(), doc_clr()]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([idx_del(), settings(true)]), @"Some(idx_del() { ids: [0] })"); debug_snapshot!(autobatch_from([idx_del(), settings(true)]), @"Some(IndexDeletion { ids: [0] })");
// The index deletion can accept almost any type of BatchKind and transform it to an idx_del() // The index deletion can accept almost any type of BatchKind and transform it to an idx_del()
// First, the basic cases // First, the basic cases
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_del(), idx_del()]), @"Some(idx_del() { ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_del(), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_clr(), idx_del()]), @"Some(idx_del() { ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 1] })"); debug_snapshot!(autobatch_from([settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 1] })");
// Then the mixed cases // Then the mixed cases
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 2, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_del()]), @"Some(idx_del() { ids: [0, 2, 1] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), idx_del()]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(idx_del() { ids: [1, 3, 0, 2] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(idx_del() { ids: [1, 3, 0, 2] })"); debug_snapshot!(autobatch_from([doc_imp(UpdateDocuments, true), settings(true), doc_clr(), idx_del()]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
} }
#[test] #[test]
fn allowed_and_disallowed_index_creation() { fn allowed_and_disallowed_index_creation() {
// doc_imp(indexes canbe)ixed with those disallowed to do so // doc_imp(indexes canbe)ixed with those disallowed to do so
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, true)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(doc_imp(ReplaceDocuments, true)import_ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), doc_imp(ReplaceDocuments, true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, false)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0, 1] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), doc_imp(ReplaceDocuments, false)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(settingsAnddoc_imp(: ReplaceDocuments: true)import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, true), settings(true)]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), settings(true)]), @"Some(doc_imp(ReplaceDocuments, false)import_ids: [0] })"); debug_snapshot!(autobatch_from([doc_imp(ReplaceDocuments, false), settings(true)]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
} }
} }

91
index-scheduler/src/batch.rs
View File

@ -1,3 +1,5 @@
use std::collections::HashSet;
use crate::{autobatcher::BatchKind, Error, IndexScheduler, Result, TaskId}; use crate::{autobatcher::BatchKind, Error, IndexScheduler, Result, TaskId};
use meilisearch_types::tasks::{Details, Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Details, Kind, KindWithContent, Status, Task};
@ -20,7 +22,7 @@ use uuid::Uuid;
pub(crate) enum Batch { pub(crate) enum Batch {
Cancel(Task), Cancel(Task),
DeleteTasks(Task), TaskDeletion(Task),
Snapshot(Vec<Task>), Snapshot(Vec<Task>),
Dump(Vec<Task>), Dump(Vec<Task>),
IndexOperation(IndexOperation), IndexOperation(IndexOperation),
@ -95,7 +97,7 @@ impl Batch {
pub fn ids(&self) -> Vec<TaskId> { pub fn ids(&self) -> Vec<TaskId> {
match self { match self {
Batch::Cancel(task) Batch::Cancel(task)
| Batch::DeleteTasks(task) | Batch::TaskDeletion(task)
| Batch::IndexCreation { task, .. } | Batch::IndexCreation { task, .. }
| Batch::IndexUpdate { task, .. } => vec![task.uid], | Batch::IndexUpdate { task, .. } => vec![task.uid],
Batch::Snapshot(tasks) | Batch::Dump(tasks) | Batch::IndexDeletion { tasks, .. } => { Batch::Snapshot(tasks) | Batch::Dump(tasks) | Batch::IndexDeletion { tasks, .. } => {
@ -379,13 +381,13 @@ impl IndexScheduler {
} }
// 2. we get the next task to delete // 2. we get the next task to delete
let to_delete = self.get_kind(rtxn, Kind::DeleteTasks)?; let to_delete = self.get_kind(rtxn, Kind::TaskDeletion)?;
if let Some(task_id) = to_delete.min() { if let Some(task_id) = to_delete.min() {
let task = self let task = self
.get_task(rtxn, task_id)? .get_task(rtxn, task_id)?
.ok_or(Error::CorruptedTaskQueue)?; .ok_or(Error::CorruptedTaskQueue)?;
return Ok(Some(Batch::DeleteTasks(task))); return Ok(Some(Batch::TaskDeletion(task)));
} }
// 3. we batch the snapshot. // 3. we batch the snapshot.
@ -445,10 +447,10 @@ impl IndexScheduler {
pub(crate) fn process_batch(&self, batch: Batch) -> Result<Vec<Task>> { pub(crate) fn process_batch(&self, batch: Batch) -> Result<Vec<Task>> {
match batch { match batch {
Batch::Cancel(_) => todo!(), Batch::Cancel(_) => todo!(),
Batch::DeleteTasks(mut task) => { Batch::TaskDeletion(mut task) => {
// 1. Retrieve the tasks that matched the query at enqueue-time. // 1. Retrieve the tasks that matched the query at enqueue-time.
let matched_tasks = let matched_tasks =
if let KindWithContent::DeleteTasks { tasks, query: _ } = &task.kind { if let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind {
tasks tasks
} else { } else {
unreachable!() unreachable!()
@ -459,7 +461,7 @@ impl IndexScheduler {
task.status = Status::Succeeded; task.status = Status::Succeeded;
match &mut task.details { match &mut task.details {
Some(Details::DeleteTasks { Some(Details::TaskDeletion {
matched_tasks: _, matched_tasks: _,
deleted_tasks, deleted_tasks,
original_query: _, original_query: _,
@ -816,49 +818,52 @@ impl IndexScheduler {
/// Delete each given task from all the databases (if it is deleteable). /// Delete each given task from all the databases (if it is deleteable).
/// ///
/// Return the number of tasks that were actually deleted /// Return the number of tasks that were actually deleted
fn delete_matched_tasks(&self, wtxn: &mut RwTxn, matched_tasks: &[u32]) -> Result<usize> { fn delete_matched_tasks(
&self,
wtxn: &mut RwTxn,
matched_tasks: &RoaringBitmap,
) -> Result<usize> {
// 1. Remove from this list the tasks that we are not allowed to delete // 1. Remove from this list the tasks that we are not allowed to delete
let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?; let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?;
let processing_tasks = &self.processing_tasks.read().unwrap().1; let processing_tasks = &self.processing_tasks.read().unwrap().1;
let to_delete_tasks = matched_tasks
.iter() let mut to_delete_tasks = matched_tasks - processing_tasks;
.filter(|&&task_id| { to_delete_tasks -= enqueued_tasks;
!processing_tasks.contains(task_id) && !enqueued_tasks.contains(task_id)
})
.copied();
let to_delete_tasks = RoaringBitmap::from_iter(to_delete_tasks);
// 2. We now have a list of tasks to delete, delete them // 2. We now have a list of tasks to delete, delete them
for task_id in to_delete_tasks.iter() { let mut affected_indexes = HashSet::new();
let task = self.all_tasks.get(wtxn, &BEU32::new(task_id))?.unwrap(); let mut affected_statuses = HashSet::new();
self.delete_task(wtxn, &task)?; let mut affected_kinds = HashSet::new();
}
for task_id in to_delete_tasks.iter() {
if let Some(task) = self.all_tasks.get(wtxn, &BEU32::new(task_id))? {
if let Some(task_indexes) = task.indexes() {
affected_indexes.extend(task_indexes.into_iter().map(|x| x.to_owned()));
}
affected_statuses.insert(task.status);
affected_kinds.insert(task.kind.as_kind());
}
}
for index in affected_indexes {
self.update_index(wtxn, &index, |bitmap| {
*bitmap -= &to_delete_tasks;
})?;
}
for status in affected_statuses {
self.update_status(wtxn, status, |bitmap| {
*bitmap -= &to_delete_tasks;
})?;
}
for kind in affected_kinds {
self.update_kind(wtxn, kind, |bitmap| {
*bitmap -= &to_delete_tasks;
})?;
}
for task in to_delete_tasks.iter() {
self.all_tasks.delete(wtxn, &BEU32::new(task))?;
}
Ok(to_delete_tasks.len() as usize) Ok(to_delete_tasks.len() as usize)
} }
/// Delete the given task from all the databases
fn delete_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> {
let task_id = BEU32::new(task.uid);
if let Some(indexes) = task.indexes() {
for index in indexes {
self.update_index(wtxn, index, |bitmap| {
bitmap.remove(task.uid);
})?;
}
}
self.update_status(wtxn, task.status, |bitmap| {
bitmap.remove(task.uid);
})?;
self.update_kind(wtxn, task.kind.as_kind(), |bitmap| {
(bitmap.remove(task.uid));
})?;
self.all_tasks.delete(wtxn, &task_id)?;
Ok(())
}
} }

72
index-scheduler/src/lib.rs
View File

@ -242,44 +242,52 @@ impl IndexScheduler {
self.index_mapper.indexes(&rtxn) self.index_mapper.indexes(&rtxn)
} }
/// Returns the tasks corresponding to the query. /// Return the task ids corresponding to the query
pub fn get_tasks(&self, query: Query) -> Result<Vec<Task>> { pub fn get_task_ids(&self, query: &Query) -> Result<RoaringBitmap> {
let rtxn = self.env.read_txn()?; let rtxn = self.env.read_txn()?;
let last_task_id = match self.last_task_id(&rtxn)? { let last_task_id = match self.last_task_id(&rtxn)? {
Some(tid) => query.from.map(|from| from.min(tid)).unwrap_or(tid), Some(tid) => query.from.map(|from| from.min(tid)).unwrap_or(tid),
None => return Ok(Vec::new()), None => return Ok(RoaringBitmap::new()),
}; };
// This is the list of all the tasks. // This is the list of all the tasks.
let mut tasks = RoaringBitmap::from_sorted_iter(0..last_task_id).unwrap(); let mut tasks = RoaringBitmap::from_sorted_iter(0..last_task_id).unwrap();
if let Some(uids) = query.uid { if let Some(uids) = &query.uid {
tasks &= RoaringBitmap::from_iter(uids); tasks &= RoaringBitmap::from_iter(uids);
} }
if let Some(status) = query.status { if let Some(status) = &query.status {
let mut status_tasks = RoaringBitmap::new(); let mut status_tasks = RoaringBitmap::new();
for status in status { for status in status {
status_tasks |= self.get_status(&rtxn, status)?; status_tasks |= self.get_status(&rtxn, *status)?;
} }
tasks &= status_tasks; tasks &= status_tasks;
} }
if let Some(kind) = query.kind { if let Some(kind) = &query.kind {
let mut kind_tasks = RoaringBitmap::new(); let mut kind_tasks = RoaringBitmap::new();
for kind in kind { for kind in kind {
kind_tasks |= self.get_kind(&rtxn, kind)?; kind_tasks |= self.get_kind(&rtxn, *kind)?;
} }
tasks &= kind_tasks; tasks &= kind_tasks;
} }
if let Some(index) = query.index_uid { if let Some(index) = &query.index_uid {
let mut index_tasks = RoaringBitmap::new(); let mut index_tasks = RoaringBitmap::new();
for index in index { for index in index {
index_tasks |= self.index_tasks(&rtxn, &index)?; index_tasks |= self.index_tasks(&rtxn, &index)?;
} }
tasks &= index_tasks; tasks &= index_tasks;
} }
rtxn.commit().unwrap();
Ok(tasks)
}
/// Returns the tasks corresponding to the query.
pub fn get_tasks(&self, query: Query) -> Result<Vec<Task>> {
let rtxn = self.env.read_txn()?;
let tasks = self.get_task_ids(&query)?;
let tasks = let tasks =
self.get_existing_tasks(&rtxn, tasks.into_iter().rev().take(query.limit as usize))?; self.get_existing_tasks(&rtxn, tasks.into_iter().rev().take(query.limit as usize))?;
@ -442,7 +450,7 @@ impl IndexScheduler {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use big_s::S; use big_s::S;
use insta::*; use meili_snap::snapshot;
use meilisearch_types::milli::update::IndexDocumentsMethod::ReplaceDocuments; use meilisearch_types::milli::update::IndexDocumentsMethod::ReplaceDocuments;
use tempfile::TempDir; use tempfile::TempDir;
use uuid::Uuid; use uuid::Uuid;
@ -580,7 +588,7 @@ mod tests {
assert_eq!(task.kind.as_kind(), k); assert_eq!(task.kind.as_kind(), k);
} }
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler));
} }
#[test] #[test]
@ -597,7 +605,7 @@ mod tests {
}) })
.unwrap(); .unwrap();
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler));
} }
/// We send a lot of tasks but notify the tasks scheduler only once as /// We send a lot of tasks but notify the tasks scheduler only once as
@ -692,23 +700,23 @@ mod tests {
// here we have registered all the tasks, but the index scheduler // here we have registered all the tasks, but the index scheduler
// has not progressed at all // has not progressed at all
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued");
index_scheduler index_scheduler
.register(KindWithContent::DeleteTasks { .register(KindWithContent::TaskDeletion {
query: "test_query".to_owned(), query: "test_query".to_owned(),
tasks: vec![0, 1], tasks: RoaringBitmap::from_iter(&[0, 1]),
}) })
.unwrap(); .unwrap();
// again, no progress made at all, but one more task is registered // again, no progress made at all, but one more task is registered
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_enqueued");
// now we create the first batch // now we create the first batch
handle.wait_till(Breakpoint::BatchCreated); handle.wait_till(Breakpoint::BatchCreated);
// the task deletion should now be "processing" // the task deletion should now be "processing"
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processing");
handle.wait_till(Breakpoint::AfterProcessing); handle.wait_till(Breakpoint::AfterProcessing);
@ -716,7 +724,7 @@ mod tests {
// because the tasks with ids 0 and 1 were still "enqueued", and thus undeleteable // because the tasks with ids 0 and 1 were still "enqueued", and thus undeleteable
// the "task deletion" task should be marked as "succeeded" and, in its details, the // the "task deletion" task should be marked as "succeeded" and, in its details, the
// number of deleted tasks should be 0 // number of deleted tasks should be 0
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_done");
} }
#[test] #[test]
@ -737,25 +745,25 @@ mod tests {
file0.persist().unwrap(); file0.persist().unwrap();
file1.persist().unwrap(); file1.persist().unwrap();
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_enqueued");
handle.wait_till(Breakpoint::AfterProcessing); handle.wait_till(Breakpoint::AfterProcessing);
// first addition of documents should be successful // first addition of documents should be successful
// TODO: currently the result of this operation is incorrect! // TODO: currently the result of this operation is incorrect!
// only the first task should be successful, because it should not be batched with // only the first task should be successful, because it should not be batched with
// the second task, that operates on a different index! // the second task, that operates on a different index!
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_tasks_processed");
// Now we delete the first task // Now we delete the first task
index_scheduler index_scheduler
.register(KindWithContent::DeleteTasks { .register(KindWithContent::TaskDeletion {
query: "test_query".to_owned(), query: "test_query".to_owned(),
tasks: vec![0], tasks: RoaringBitmap::from_iter(&[0]),
}) })
.unwrap(); .unwrap();
handle.wait_till(Breakpoint::AfterProcessing); handle.wait_till(Breakpoint::AfterProcessing);
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "task_deletion_processed");
} }
#[test] #[test]
@ -784,15 +792,15 @@ mod tests {
.unwrap(); .unwrap();
file.persist().unwrap(); file.persist().unwrap();
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler));
handle.wait_till(Breakpoint::BatchCreated); handle.wait_till(Breakpoint::BatchCreated);
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler));
handle.wait_till(Breakpoint::AfterProcessing); handle.wait_till(Breakpoint::AfterProcessing);
assert_snapshot!(snapshot_index_scheduler(&index_scheduler)); snapshot!(snapshot_index_scheduler(&index_scheduler));
} }
#[test] #[test]
@ -833,18 +841,10 @@ mod tests {
} }
#[macro_export] #[macro_export]
macro_rules! assert_smol_debug_snapshot { macro_rules! debug_snapshot {
($value:expr, @$snapshot:literal) => {{ ($value:expr, @$snapshot:literal) => {{
let value = format!("{:?}", $value); let value = format!("{:?}", $value);
insta::assert_snapshot!(value, stringify!($value), @$snapshot); meili_snap::snapshot!(value, @$snapshot);
}};
($name:expr, $value:expr) => {{
let value = format!("{:?}", $value);
insta::assert_snapshot!(Some($name), value, stringify!($value));
}};
($value:expr) => {{
let value = format!("{:?}", $value);
insta::assert_snapshot!($crate::_macro_support::AutoName, value, stringify!($value));
}}; }};
} }

11
index-scheduler/src/snapshot.rs
View File

@ -1,9 +1,12 @@
use meilisearch_types::heed::{ use meilisearch_types::milli::{RoaringBitmapCodec, BEU32};
use meilisearch_types::tasks::Details;
use meilisearch_types::{
heed::{
types::{OwnedType, SerdeBincode, SerdeJson, Str}, types::{OwnedType, SerdeBincode, SerdeJson, Str},
Database, RoTxn, Database, RoTxn,
},
tasks::Task,
}; };
use meilisearch_types::milli::{RoaringBitmapCodec, BEU32};
use meilisearch_types::tasks::{Details, Task};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use crate::{index_mapper::IndexMapper, IndexScheduler, Kind, Status}; use crate::{index_mapper::IndexMapper, IndexScheduler, Kind, Status};
@ -127,7 +130,7 @@ fn snaphsot_details(d: &Details) -> String {
Details::ClearAll { deleted_documents } => { Details::ClearAll { deleted_documents } => {
format!("{{ deleted_documents: {deleted_documents:?} }}") format!("{{ deleted_documents: {deleted_documents:?} }}")
}, },
Details::DeleteTasks { Details::TaskDeletion {
matched_tasks, matched_tasks,
deleted_tasks, deleted_tasks,
original_query, original_query,

3
index-scheduler/src/snapshots/index_scheduler__tests__document_addition.snap → index-scheduler/src/snapshots/lib.rs/document_addition/1.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -13,7 +12,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
enqueued [0,] enqueued [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,] "documentImport" [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
doggos [0,] doggos [0,]

3
index-scheduler/src/snapshots/index_scheduler__tests__document_addition-2.snap → index-scheduler/src/snapshots/lib.rs/document_addition/2.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -13,7 +12,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
enqueued [0,] enqueued [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,] "documentImport" [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
doggos [0,] doggos [0,]

3
index-scheduler/src/snapshots/index_scheduler__tests__document_addition-3.snap → index-scheduler/src/snapshots/lib.rs/document_addition/3.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -14,7 +13,7 @@ enqueued []
succeeded [0,] succeeded [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,] "documentImport" [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
doggos [0,] doggos [0,]

1
index-scheduler/src/snapshots/index_scheduler__tests__insert_task_while_another_task_is_processing.snap → index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/1.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:

3
index-scheduler/src/snapshots/index_scheduler__tests__register.snap → index-scheduler/src/snapshots/lib.rs/register/1.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -17,7 +16,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
enqueued [0,1,2,3,4,] enqueued [0,1,2,3,4,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,3,4,] "documentImport" [1,3,4,]
"indexCreation" [0,] "indexCreation" [0,]
"cancelTask" [2,] "cancelTask" [2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------

3
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_deleteable.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -14,7 +13,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
enqueued [0,1,] enqueued [0,1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,1,] "documentImport" [0,1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [0,] catto [0,]

3
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_deleteable-2.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -15,7 +14,7 @@ enqueued []
succeeded [0,1,] succeeded [0,1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [0,1,] "documentImport" [0,1,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [0,] catto [0,]

7
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_deleteable-3.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -8,15 +7,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: 1 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: 1 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0] }} 2 {uid: 2, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0]> }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
enqueued [] enqueued []
succeeded [1,2,] succeeded [1,2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,] "documentImport" [1,]
"deleteTasks" [2,] "taskDeletion" [2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [] catto []

3
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_undeleteable.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -15,7 +14,7 @@ expression: snapshot_index_scheduler(&index_scheduler)
enqueued [0,1,2,] enqueued [0,1,2,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,] "documentImport" [1,2,]
"indexCreation" [0,] "indexCreation" [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:

7
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_undeleteable-4.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -10,16 +9,16 @@ expression: snapshot_index_scheduler(&index_scheduler)
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }} 3 {uid: 3, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
enqueued [0,1,2,] enqueued [0,1,2,]
succeeded [3,] succeeded [3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,] "documentImport" [1,2,]
"indexCreation" [0,] "indexCreation" [0,]
"deleteTasks" [3,] "taskDeletion" [3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [0,1,] catto [0,1,]

7
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_undeleteable-2.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -10,15 +9,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
enqueued [0,1,2,3,] enqueued [0,1,2,3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,] "documentImport" [1,2,]
"indexCreation" [0,] "indexCreation" [0,]
"deleteTasks" [3,] "taskDeletion" [3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [0,1,] catto [0,1,]

7
index-scheduler/src/snapshots/index_scheduler__tests__task_deletion_undeleteable-3.snap → index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap
View File

@ -1,6 +1,5 @@
--- ---
source: index-scheduler/src/lib.rs source: index-scheduler/src/lib.rs
expression: snapshot_index_scheduler(&index_scheduler)
--- ---
### Autobatching Enabled = true ### Autobatching Enabled = true
### Processing Tasks: ### Processing Tasks:
@ -10,15 +9,15 @@ expression: snapshot_index_scheduler(&index_scheduler)
0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: 0 }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }}
2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 5000, indexed_documents: 0 }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 5000, allow_index_creation: true }}
3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: DeleteTasks { query: "test_query", tasks: [0, 1] }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_query: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
enqueued [0,1,2,3,] enqueued [0,1,2,3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Kind: ### Kind:
{"documentImport":{"method":"ReplaceDocuments","allow_index_creation":true}} [1,2,] "documentImport" [1,2,]
"indexCreation" [0,] "indexCreation" [0,]
"deleteTasks" [3,] "taskDeletion" [3,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Index Tasks: ### Index Tasks:
catto [0,1,] catto [0,1,]

1
meilisearch-auth/Cargo.toml
View File

@ -7,7 +7,6 @@ edition = "2021"
enum-iterator = "1.1.2" enum-iterator = "1.1.2"
hmac = "0.12.1" hmac = "0.12.1"
meilisearch-types = { path = "../meilisearch-types" } meilisearch-types = { path = "../meilisearch-types" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.34.0", default-features = false }
rand = "0.8.5" rand = "0.8.5"
serde = { version = "1.0.145", features = ["derive"] } serde = { version = "1.0.145", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] } serde_json = { version = "1.0.85", features = ["preserve_order"] }

135
meilisearch-auth/src/action.rs
View File

@ -1,135 +0,0 @@
use serde::{Deserialize, Serialize};
use std::hash::Hash;
#[derive(
enum_iterator::Sequence, Copy, Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Hash,
)]
#[repr(u8)]
pub enum Action {
#[serde(rename = "*")]
All = 0,
#[serde(rename = "search")]
Search,
#[serde(rename = "documents.*")]
DocumentsAll,
#[serde(rename = "documents.add")]
DocumentsAdd,
#[serde(rename = "documents.get")]
DocumentsGet,
#[serde(rename = "documents.delete")]
DocumentsDelete,
#[serde(rename = "indexes.*")]
IndexesAll,
#[serde(rename = "indexes.create")]
IndexesAdd,
#[serde(rename = "indexes.get")]
IndexesGet,
#[serde(rename = "indexes.update")]
IndexesUpdate,
#[serde(rename = "indexes.delete")]
IndexesDelete,
#[serde(rename = "tasks.*")]
TasksAll,
#[serde(rename = "tasks.get")]
TasksGet,
#[serde(rename = "settings.*")]
SettingsAll,
#[serde(rename = "settings.get")]
SettingsGet,
#[serde(rename = "settings.update")]
SettingsUpdate,
#[serde(rename = "stats.*")]
StatsAll,
#[serde(rename = "stats.get")]
StatsGet,
#[serde(rename = "metrics.*")]
MetricsAll,
#[serde(rename = "metrics.get")]
MetricsGet,
#[serde(rename = "dumps.*")]
DumpsAll,
#[serde(rename = "dumps.create")]
DumpsCreate,
#[serde(rename = "version")]
Version,
#[serde(rename = "keys.create")]
KeysAdd,
#[serde(rename = "keys.get")]
KeysGet,
#[serde(rename = "keys.update")]
KeysUpdate,
#[serde(rename = "keys.delete")]
KeysDelete,
}
impl Action {
pub const fn from_repr(repr: u8) -> Option<Self> {
use actions::*;
match repr {
ALL => Some(Self::All),
SEARCH => Some(Self::Search),
DOCUMENTS_ALL => Some(Self::DocumentsAll),
DOCUMENTS_ADD => Some(Self::DocumentsAdd),
DOCUMENTS_GET => Some(Self::DocumentsGet),
DOCUMENTS_DELETE => Some(Self::DocumentsDelete),
INDEXES_ALL => Some(Self::IndexesAll),
INDEXES_CREATE => Some(Self::IndexesAdd),
INDEXES_GET => Some(Self::IndexesGet),
INDEXES_UPDATE => Some(Self::IndexesUpdate),
INDEXES_DELETE => Some(Self::IndexesDelete),
TASKS_ALL => Some(Self::TasksAll),
TASKS_GET => Some(Self::TasksGet),
SETTINGS_ALL => Some(Self::SettingsAll),
SETTINGS_GET => Some(Self::SettingsGet),
SETTINGS_UPDATE => Some(Self::SettingsUpdate),
STATS_ALL => Some(Self::StatsAll),
STATS_GET => Some(Self::StatsGet),
METRICS_ALL => Some(Self::MetricsAll),
METRICS_GET => Some(Self::MetricsGet),
DUMPS_ALL => Some(Self::DumpsAll),
DUMPS_CREATE => Some(Self::DumpsCreate),
VERSION => Some(Self::Version),
KEYS_CREATE => Some(Self::KeysAdd),
KEYS_GET => Some(Self::KeysGet),
KEYS_UPDATE => Some(Self::KeysUpdate),
KEYS_DELETE => Some(Self::KeysDelete),
_otherwise => None,
}
}
pub const fn repr(&self) -> u8 {
*self as u8
}
}
pub mod actions {
use super::Action::*;
pub(crate) const ALL: u8 = All.repr();
pub const SEARCH: u8 = Search.repr();
pub const DOCUMENTS_ALL: u8 = DocumentsAll.repr();
pub const DOCUMENTS_ADD: u8 = DocumentsAdd.repr();
pub const DOCUMENTS_GET: u8 = DocumentsGet.repr();
pub const DOCUMENTS_DELETE: u8 = DocumentsDelete.repr();
pub const INDEXES_ALL: u8 = IndexesAll.repr();
pub const INDEXES_CREATE: u8 = IndexesAdd.repr();
pub const INDEXES_GET: u8 = IndexesGet.repr();
pub const INDEXES_UPDATE: u8 = IndexesUpdate.repr();
pub const INDEXES_DELETE: u8 = IndexesDelete.repr();
pub const TASKS_ALL: u8 = TasksAll.repr();
pub const TASKS_GET: u8 = TasksGet.repr();
pub const SETTINGS_ALL: u8 = SettingsAll.repr();
pub const SETTINGS_GET: u8 = SettingsGet.repr();
pub const SETTINGS_UPDATE: u8 = SettingsUpdate.repr();
pub const STATS_ALL: u8 = StatsAll.repr();
pub const STATS_GET: u8 = StatsGet.repr();
pub const METRICS_ALL: u8 = MetricsAll.repr();
pub const METRICS_GET: u8 = MetricsGet.repr();
pub const DUMPS_ALL: u8 = DumpsAll.repr();
pub const DUMPS_CREATE: u8 = DumpsCreate.repr();
pub const VERSION: u8 = Version.repr();
pub const KEYS_CREATE: u8 = KeysAdd.repr();
pub const KEYS_GET: u8 = KeysGet.repr();
pub const KEYS_UPDATE: u8 = KeysUpdate.repr();
pub const KEYS_DELETE: u8 = KeysDelete.repr();
}

2
meilisearch-auth/src/error.rs
View File

@ -1,7 +1,7 @@
use std::error::Error; use std::error::Error;
use meilisearch_types::error::{Code, ErrorCode}; use meilisearch_types::error::{Code, ErrorCode};
use meilisearch_types::{internal_error, keys}; use meilisearch_types::{internal_error, keys, milli};
pub type Result<T> = std::result::Result<T, AuthControllerError>; pub type Result<T> = std::result::Result<T, AuthControllerError>;

1
meilisearch-auth/src/store.rs
View File

@ -12,6 +12,7 @@ use std::sync::Arc;
use hmac::{Hmac, Mac}; use hmac::{Hmac, Mac};
use meilisearch_types::keys::KeyId; use meilisearch_types::keys::KeyId;
use meilisearch_types::star_or::StarOr; use meilisearch_types::star_or::StarOr;
use meilisearch_types::milli;
use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
use milli::heed::{Database, Env, EnvOpenOptions, RwTxn}; use milli::heed::{Database, Env, EnvOpenOptions, RwTxn};
use sha2::Sha256; use sha2::Sha256;

1
meilisearch-http/Cargo.toml
View File

@ -82,6 +82,7 @@ tokio-stream = "0.1.10"
toml = "0.5.9" toml = "0.5.9"
uuid = { version = "1.1.2", features = ["serde", "v4"] } uuid = { version = "1.1.2", features = ["serde", "v4"] }
walkdir = "2.3.2" walkdir = "2.3.2"
yaup = "0.2.0"
prometheus = { version = "0.13.2", features = ["process"], optional = true } prometheus = { version = "0.13.2", features = ["process"], optional = true }
lazy_static = "1.4.0" lazy_static = "1.4.0"

68
meilisearch-http/src/routes/tasks.rs
View File

@ -1,11 +1,12 @@
use actix_web::web::Data; use actix_web::web::Data;
use actix_web::{web, HttpRequest, HttpResponse}; use actix_web::{web, HttpRequest, HttpResponse};
use index_scheduler::{IndexScheduler, TaskId}; use env_logger::filter;
use index_scheduler::{IndexScheduler, Query, TaskId};
use meilisearch_types::error::ResponseError; use meilisearch_types::error::ResponseError;
use meilisearch_types::index_uid::IndexUid; use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::settings::{Settings, Unchecked}; use meilisearch_types::settings::{Settings, Unchecked};
use meilisearch_types::star_or::StarOr; use meilisearch_types::star_or::StarOr;
use meilisearch_types::tasks::{serialize_duration, Details, Kind, Status, Task}; use meilisearch_types::tasks::{serialize_duration, Details, Kind, KindWithContent, Status, Task};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_cs::vec::CS; use serde_cs::vec::CS;
use serde_json::json; use serde_json::json;
@ -21,7 +22,8 @@ const DEFAULT_LIMIT: fn() -> u32 = || 20;
pub fn configure(cfg: &mut web::ServiceConfig) { pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks)))) cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks))))
.service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task)))); .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task))))
.service(web::resource("").route(web::delete().to(SeqHandler(delete_tasks))));
} }
#[derive(Debug, Clone, PartialEq, Serialize)] #[derive(Debug, Clone, PartialEq, Serialize)]
@ -140,7 +142,7 @@ impl From<Details> for DetailsView {
deleted_documents: Some(deleted_documents), deleted_documents: Some(deleted_documents),
..DetailsView::default() ..DetailsView::default()
}, },
Details::DeleteTasks { Details::TaskDeletion {
matched_tasks, matched_tasks,
deleted_tasks, deleted_tasks,
original_query, original_query,
@ -195,6 +197,29 @@ fn task_status_matches_events(status: &TaskStatus, events: &[TaskEvent]) -> bool
}) })
} }
async fn delete_tasks(
index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_DELETE }>, Data<IndexScheduler>>,
params: web::Query<Query>,
_req: HttpRequest,
_analytics: web::Data<dyn Analytics>,
) -> Result<HttpResponse, ResponseError> {
let query = params.into_inner();
let filtered_query = filter_out_inaccessible_indexes_from_query(&index_scheduler, &query);
let tasks = index_scheduler.get_task_ids(&filtered_query)?;
let filtered_query_string = yaup::to_string(&filtered_query).unwrap();
let task_deletion = KindWithContent::TaskDeletion {
query: filtered_query_string,
tasks,
};
// TODO: Lo: analytics
let task = index_scheduler.register(task_deletion)?;
let task_view: TaskView = task.into();
Ok(HttpResponse::Ok().json(task_view))
}
async fn get_tasks( async fn get_tasks(
index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>, index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>,
params: web::Query<TasksFilterQuery>, params: web::Query<TasksFilterQuery>,
@ -318,3 +343,38 @@ async fn get_task(
Err(index_scheduler::Error::TaskNotFound(task_id).into()) Err(index_scheduler::Error::TaskNotFound(task_id).into())
} }
} }
fn filter_out_inaccessible_indexes_from_query<const ACTION: u8>(
index_scheduler: &GuardedData<ActionPolicy<ACTION>, Data<IndexScheduler>>,
query: &Query,
) -> Query {
let mut query = query.clone();
// First remove all indexes from the query, we will add them back later
let indexes = query.index_uid.take();
let search_rules = &index_scheduler.filters().search_rules;
let mut query = index_scheduler::Query::default();
// We filter on potential indexes and make sure that the search filter
// restrictions are also applied.
match indexes {
Some(indexes) => {
for name in indexes.iter() {
if search_rules.is_index_authorized(&name) {
query = query.with_index(name.to_string());
}
}
}
None => {
if !search_rules.is_index_authorized("*") {
for (index, _policy) in search_rules.clone() {
query = query.with_index(index.to_string());
}
}
}
};
query
}

3
meilisearch-types/Cargo.toml
View File

@ -8,10 +8,11 @@ edition = "2021"
actix-web = { version = "4.2.1", default-features = false } actix-web = { version = "4.2.1", default-features = false }
csv = "1.1.6" csv = "1.1.6"
either = { version = "1.6.1", features = ["serde"] } either = { version = "1.6.1", features = ["serde"] }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.33.4", default-features = false } milli = { git = "https://github.com/meilisearch/milli.git", branch = "indexation-abortion", default-features = false }
enum-iterator = "0.7.0" enum-iterator = "0.7.0"
proptest = { version = "1.0.0", optional = true } proptest = { version = "1.0.0", optional = true }
proptest-derive = { version = "0.3.0", optional = true } proptest-derive = { version = "0.3.0", optional = true }
roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.145", features = ["derive"] } serde = { version = "1.0.145", features = ["derive"] }
serde_json = "1.0.85" serde_json = "1.0.85"
time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }

3
meilisearch-types/src/keys.rs
View File

@ -224,6 +224,8 @@ pub enum Action {
IndexesDelete, IndexesDelete,
#[serde(rename = "tasks.*")] #[serde(rename = "tasks.*")]
TasksAll, TasksAll,
#[serde(rename = "tasks.*")]
TasksDelete,
#[serde(rename = "tasks.get")] #[serde(rename = "tasks.get")]
TasksGet, TasksGet,
#[serde(rename = "settings.*")] #[serde(rename = "settings.*")]
@ -311,6 +313,7 @@ pub mod actions {
pub const INDEXES_UPDATE: u8 = IndexesUpdate.repr(); pub const INDEXES_UPDATE: u8 = IndexesUpdate.repr();
pub const INDEXES_DELETE: u8 = IndexesDelete.repr(); pub const INDEXES_DELETE: u8 = IndexesDelete.repr();
pub const TASKS_ALL: u8 = TasksAll.repr(); pub const TASKS_ALL: u8 = TasksAll.repr();
pub const TASKS_DELETE: u8 = TasksDelete.repr();
pub const TASKS_GET: u8 = TasksGet.repr(); pub const TASKS_GET: u8 = TasksGet.repr();
pub const SETTINGS_ALL: u8 = SettingsAll.repr(); pub const SETTINGS_ALL: u8 = SettingsAll.repr();
pub const SETTINGS_GET: u8 = SettingsGet.repr(); pub const SETTINGS_GET: u8 = SettingsGet.repr();

29
meilisearch-types/src/tasks.rs
View File

@ -1,4 +1,5 @@
use milli::update::IndexDocumentsMethod; use milli::update::IndexDocumentsMethod;
use roaring::RoaringBitmap;
use serde::{Deserialize, Serialize, Serializer}; use serde::{Deserialize, Serialize, Serializer};
use std::{ use std::{
fmt::{Display, Write}, fmt::{Display, Write},
@ -42,7 +43,7 @@ impl Task {
DumpExport { .. } DumpExport { .. }
| Snapshot | Snapshot
| CancelTask { .. } | CancelTask { .. }
| DeleteTasks { .. } | TaskDeletion { .. }
| IndexSwap { .. } => None, | IndexSwap { .. } => None,
DocumentImport { index_uid, .. } DocumentImport { index_uid, .. }
| DocumentDeletion { index_uid, .. } | DocumentDeletion { index_uid, .. }
@ -59,7 +60,7 @@ impl Task {
use KindWithContent::*; use KindWithContent::*;
match &self.kind { match &self.kind {
DumpExport { .. } | Snapshot | CancelTask { .. } | DeleteTasks { .. } => None, DumpExport { .. } | Snapshot | CancelTask { .. } | TaskDeletion { .. } => None,
DocumentImport { index_uid, .. } DocumentImport { index_uid, .. }
| DocumentDeletion { index_uid, .. } | DocumentDeletion { index_uid, .. }
| DocumentClear { index_uid } | DocumentClear { index_uid }
@ -114,9 +115,9 @@ pub enum KindWithContent {
CancelTask { CancelTask {
tasks: Vec<TaskId>, tasks: Vec<TaskId>,
}, },
DeleteTasks { TaskDeletion {
query: String, query: String,
tasks: Vec<TaskId>, tasks: RoaringBitmap,
}, },
DumpExport { DumpExport {
output: PathBuf, output: PathBuf,
@ -136,7 +137,7 @@ impl KindWithContent {
KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate,
KindWithContent::IndexSwap { .. } => Kind::IndexSwap, KindWithContent::IndexSwap { .. } => Kind::IndexSwap,
KindWithContent::CancelTask { .. } => Kind::CancelTask, KindWithContent::CancelTask { .. } => Kind::CancelTask,
KindWithContent::DeleteTasks { .. } => Kind::DeleteTasks, KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion,
KindWithContent::DumpExport { .. } => Kind::DumpExport, KindWithContent::DumpExport { .. } => Kind::DumpExport,
KindWithContent::Snapshot => Kind::Snapshot, KindWithContent::Snapshot => Kind::Snapshot,
} }
@ -146,7 +147,7 @@ impl KindWithContent {
use KindWithContent::*; use KindWithContent::*;
match self { match self {
DumpExport { .. } | Snapshot | CancelTask { .. } | DeleteTasks { .. } => None, DumpExport { .. } | Snapshot | CancelTask { .. } | TaskDeletion { .. } => None,
DocumentImport { index_uid, .. } DocumentImport { index_uid, .. }
| DocumentDeletion { index_uid, .. } | DocumentDeletion { index_uid, .. }
| DocumentClear { index_uid } | DocumentClear { index_uid }
@ -192,8 +193,8 @@ impl KindWithContent {
KindWithContent::CancelTask { .. } => { KindWithContent::CancelTask { .. } => {
None // TODO: check correctness of this return value None // TODO: check correctness of this return value
} }
KindWithContent::DeleteTasks { query, tasks } => Some(Details::DeleteTasks { KindWithContent::TaskDeletion { query, tasks } => Some(Details::TaskDeletion {
matched_tasks: tasks.len(), matched_tasks: tasks.len() as usize,
deleted_tasks: None, deleted_tasks: None,
original_query: query.clone(), original_query: query.clone(),
}), }),
@ -203,7 +204,7 @@ impl KindWithContent {
} }
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub enum Status { pub enum Status {
Enqueued, Enqueued,
@ -240,7 +241,7 @@ impl FromStr for Status {
} }
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub enum Kind { pub enum Kind {
DocumentImport, DocumentImport,
@ -252,7 +253,7 @@ pub enum Kind {
IndexUpdate, IndexUpdate,
IndexSwap, IndexSwap,
CancelTask, CancelTask,
DeleteTasks, TaskDeletion,
DumpExport, DumpExport,
Snapshot, Snapshot,
} }
@ -272,7 +273,7 @@ impl FromStr for Kind {
"index_update" => Ok(Kind::IndexUpdate), "index_update" => Ok(Kind::IndexUpdate),
"index_swap" => Ok(Kind::IndexSwap), "index_swap" => Ok(Kind::IndexSwap),
"cancel_task" => Ok(Kind::CancelTask), "cancel_task" => Ok(Kind::CancelTask),
"delete_tasks" => Ok(Kind::DeleteTasks), "task_deletion" => Ok(Kind::TaskDeletion),
"dump_export" => Ok(Kind::DumpExport), "dump_export" => Ok(Kind::DumpExport),
"snapshot" => Ok(Kind::Snapshot), "snapshot" => Ok(Kind::Snapshot),
s => Err(ResponseError::from_msg( s => Err(ResponseError::from_msg(
@ -304,7 +305,7 @@ pub enum Details {
ClearAll { ClearAll {
deleted_documents: Option<u64>, deleted_documents: Option<u64>,
}, },
DeleteTasks { TaskDeletion {
matched_tasks: usize, matched_tasks: usize,
deleted_tasks: Option<usize>, deleted_tasks: Option<usize>,
original_query: String, original_query: String,
@ -373,7 +374,7 @@ mod tests {
#[test] #[test]
fn bad_deser() { fn bad_deser() {
let details = Details::DeleteTasks { let details = Details::TaskDeletion {
matched_tasks: 1, matched_tasks: 1,
deleted_tasks: None, deleted_tasks: None,
original_query: "hello".to_owned(), original_query: "hello".to_owned(),