searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 12:54:26 +01:00
Code Issues Releases Wiki Activity

add a bunch of tests

Browse Source
This commit is contained in:
Tamo 2022-09-14 01:48:58 +02:00 committed by Clément Renault
parent 803f2157af
commit c97d51a624
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 880 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Cargo.lock generated
View File

@ -790,6 +790,19 @@ dependencies = [
"syn 1.0.101", "syn 1.0.101",
] ]
[[package]]
name = "console"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"terminal_size",
"winapi",
]
[[package]] [[package]]
name = "constant_time_eq" name = "constant_time_eq"
version = "0.1.5" version = "0.1.5"
@ -1089,6 +1102,12 @@ dependencies = [
"void", "void",
] ]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]] [[package]]
name = "encoding" name = "encoding"
version = "0.2.33" version = "0.2.33"
@ -1766,6 +1785,7 @@ dependencies = [
"csv", "csv",
"file-store", "file-store",
"index", "index",
"insta",
"log", "log",
"milli 0.33.0", "milli 0.33.0",
"nelson", "nelson",
@ -1788,6 +1808,19 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "insta"
version = "1.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581d4e3314cae4536e5d22ffd23189d4a374696c5ef733eadafae0ed273fd303"
dependencies = [
"console",
"lazy_static",
"linked-hash-map",
"similar",
"yaml-rust",
]
[[package]] [[package]]
name = "instant" name = "instant"
version = "0.1.12" version = "0.1.12"
@ -2110,6 +2143,12 @@ dependencies = [
"yada", "yada",
] ]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]] [[package]]
name = "lmdb-rkv-sys" name = "lmdb-rkv-sys"
version = "0.15.0" version = "0.15.0"
@ -3521,6 +3560,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "similar"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62ac7f900db32bf3fd12e0117dd3dc4da74bc52ebaac97f39668446d89694803"
[[package]] [[package]]
name = "simple_asn1" name = "simple_asn1"
version = "0.6.2" version = "0.6.2"
@ -3742,6 +3787,16 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "termtree" name = "termtree"
version = "0.2.4" version = "0.2.4"
@ -4317,6 +4372,15 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1" checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1"
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]] [[package]]
name = "yaup" name = "yaup"
version = "0.2.1" version = "0.2.1"

1
index-scheduler/Cargo.toml
View File

@ -22,3 +22,4 @@ uuid = { version = "1.1.2", features = ["serde", "v4"] }
[dev-dependencies] [dev-dependencies]
nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}
insta = "1.19.1"

816
index-scheduler/src/autobatcher.rs
View File

@ -2,6 +2,7 @@ use std::ops::ControlFlow;
use crate::{task::Kind, TaskId}; use crate::{task::Kind, TaskId};
#[derive(Debug)]
pub enum BatchKind { pub enum BatchKind {
DocumentClear { DocumentClear {
ids: Vec<TaskId>, ids: Vec<TaskId>,
@ -367,5 +368,818 @@ pub fn autobatch(enqueued: Vec<(TaskId, Kind)>) -> Option<BatchKind> {
}; };
} }
None Some(acc)
}
#[cfg(test)]
mod tests {
use super::*;
use insta::*;
use Kind::*;
fn input_from(input: impl IntoIterator<Item = Kind>) -> Vec<(TaskId, Kind)> {
input
.into_iter()
.enumerate()
.map(|(id, kind)| (id as TaskId, kind))
.collect()
}
#[test]
fn autobatch_simple_operation_together() {
// we can autobatch one or multiple DocumentAddition together
assert_debug_snapshot!(autobatch(input_from([DocumentAddition])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentAddition])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
1,
2,
],
},
)
"###);
// we can autobatch one or multiple DocumentUpdate together
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentUpdate])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
1,
2,
],
},
)
"###);
// we can autobatch one or multiple DocumentDeletion together
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentDeletion, DocumentDeletion])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
1,
2,
],
},
)
"###);
// we can autobatch one or multiple Settings together
assert_debug_snapshot!(autobatch(input_from([Settings])), @r###"
Some(
Settings {
settings_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([Settings, Settings, Settings])), @r###"
Some(
Settings {
settings_ids: [
0,
1,
2,
],
},
)
"###);
}
#[test]
fn simple_document_operation_dont_autobatch_with_other() {
// addition, updates and deletion can't batch together
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentUpdate])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentDeletion])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentAddition])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentDeletion])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentAddition])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentUpdate])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexCreation])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexCreation])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexCreation])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexUpdate])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexUpdate])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexUpdate])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexRename])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexRename])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexRename])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexSwap])), @r###"
Some(
DocumentAddition {
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexSwap])), @r###"
Some(
DocumentUpdate {
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexSwap])), @r###"
Some(
DocumentDeletion {
deletion_ids: [
0,
],
},
)
"###);
}
#[test]
fn document_addition_batch_with_settings() {
// simple case
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
// multiple settings and doc addition
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, Settings, Settings])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
2,
3,
],
addition_ids: [
0,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, Settings, Settings])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
2,
3,
],
addition_ids: [
0,
1,
],
},
)
"###);
// addition and setting unordered
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentAddition, Settings])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
3,
],
addition_ids: [
0,
2,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentUpdate, Settings])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
3,
],
update_ids: [
0,
2,
],
},
)
"###);
// We ensure this kind of batch doesn't batch with forbidden operations
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentUpdate])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentAddition])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentDeletion])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentDeletion])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexCreation])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexCreation])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexUpdate])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexUpdate])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexRename])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexRename])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexSwap])), @r###"
Some(
SettingsAndDocumentAddition {
settings_ids: [
1,
],
addition_ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexSwap])), @r###"
Some(
SettingsAndDocumentUpdate {
settings_ids: [
1,
],
update_ids: [
0,
],
},
)
"###);
}
#[test]
fn clear_and_additions() {
// these two doesn't need to batch
assert_debug_snapshot!(autobatch(input_from([DocumentClear, DocumentAddition])), @r###"
Some(
DocumentClear {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentClear, DocumentUpdate])), @r###"
Some(
DocumentClear {
ids: [
0,
],
},
)
"###);
// Basic use case
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
],
},
)
"###);
// This batch kind doesn't mix with other document addition
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentAddition])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentUpdate])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
],
},
)
"###);
// But you can batch multiple clear together
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentClear, DocumentClear])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
3,
4,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentClear, DocumentClear])), @r###"
Some(
DocumentClear {
ids: [
0,
1,
2,
3,
4,
],
},
)
"###);
}
#[test]
fn clear_and_additions_and_settings() {
// A clear don't need to autobatch the settings that happens AFTER there is no documents
assert_debug_snapshot!(autobatch(input_from([DocumentClear, Settings])), @r###"
Some(
DocumentClear {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([Settings, DocumentClear, Settings])), @r###"
Some(
ClearAndSettings {
other: [
1,
],
settings_ids: [
0,
2,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentClear])), @r###"
Some(
ClearAndSettings {
other: [
0,
2,
],
settings_ids: [
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentClear])), @r###"
Some(
ClearAndSettings {
other: [
0,
2,
],
settings_ids: [
1,
],
},
)
"###);
}
#[test]
fn anything_and_index_deletion() {
// The indexdeletion doesn't batch with anything that happens AFTER
assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentAddition])), @r###"
Some(
IndexDeletion {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentUpdate])), @r###"
Some(
IndexDeletion {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentClear])), @r###"
Some(
IndexDeletion {
ids: [
0,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([IndexDeletion, Settings])), @r###"
Some(
IndexDeletion {
ids: [
0,
],
},
)
"###);
// The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion
// First, the basic cases
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentClear, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([Settings, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
1,
],
},
)
"###);
// Then the mixed cases
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
2,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
0,
2,
1,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentClear, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
1,
3,
0,
2,
],
},
)
"###);
assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentClear, IndexDeletion])), @r###"
Some(
IndexDeletion {
ids: [
1,
3,
0,
2,
],
},
)
"###);
}
} }