searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-24 13:40:31 +01:00
Code Issues Releases Wiki Activity

Fix tests after adding DeletionStrategy

Browse Source
This commit is contained in:
Louis Dureuil 2022-12-19 09:47:29 +01:00
parent 171c942282
commit ad9937c755
No known key found for this signature in database
5 changed files with 109 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
milli/src/index.rs
View File

@ -1192,8 +1192,8 @@ pub(crate) mod tests {
use crate::error::{Error, InternalError}; use crate::error::{Error, InternalError};
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{ use crate::update::{
self, DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, self, DeleteDocuments, DeletionStrategy, IndexDocuments, IndexDocumentsConfig,
IndexerConfig, Settings, IndexDocumentsMethod, IndexerConfig, Settings,
}; };
use crate::{db_snap, obkv_to_json, Index}; use crate::{db_snap, obkv_to_json, Index};
@ -1282,6 +1282,17 @@ pub(crate) mod tests {
builder.execute(drop, || false)?; builder.execute(drop, || false)?;
Ok(()) Ok(())
} }
pub fn delete_document(&self, external_document_id: &str) {
let mut wtxn = self.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &self).unwrap();
delete.strategy(self.index_documents_config.deletion_strategy);
delete.delete_external_id(external_document_id);
delete.execute().unwrap();
wtxn.commit().unwrap();
}
} }
#[test] #[test]
@ -1487,7 +1498,9 @@ pub(crate) mod tests {
use big_s::S; use big_s::S;
use maplit::hashset; use maplit::hashset;
let index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let index = index;
index index
.update_settings(|settings| { .update_settings(|settings| {
@ -1657,7 +1670,8 @@ pub(crate) mod tests {
} }
// Second Batch: replace the documents with soft-deletion // Second Batch: replace the documents with soft-deletion
{ {
index.index_documents_config.disable_soft_deletion = false; index.index_documents_config.deletion_strategy =
crate::update::DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -1726,7 +1740,7 @@ pub(crate) mod tests {
drop(rtxn); drop(rtxn);
// Third Batch: replace the documents with soft-deletion again // Third Batch: replace the documents with soft-deletion again
{ {
index.index_documents_config.disable_soft_deletion = false; index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -1795,7 +1809,7 @@ pub(crate) mod tests {
// Fourth Batch: replace the documents without soft-deletion // Fourth Batch: replace the documents without soft-deletion
{ {
index.index_documents_config.disable_soft_deletion = true; index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -1867,6 +1881,7 @@ pub(crate) mod tests {
fn bug_3021_first() { fn bug_3021_first() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index index
@ -1891,11 +1906,7 @@ pub(crate) mod tests {
"###); "###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]"); db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap(); index.delete_document("34");
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("34");
delete.execute().unwrap();
wtxn.commit().unwrap();
db_snap!(index, documents_ids, @"[0, ]"); db_snap!(index, documents_ids, @"[0, ]");
db_snap!(index, external_documents_ids, 2, @r###" db_snap!(index, external_documents_ids, 2, @r###"
@ -1936,11 +1947,7 @@ pub(crate) mod tests {
db_snap!(index, soft_deleted_documents_ids, 4, @"[]"); db_snap!(index, soft_deleted_documents_ids, 4, @"[]");
// We do the test again, but deleting the document with id 0 instead of id 1 now // We do the test again, but deleting the document with id 0 instead of id 1 now
let mut wtxn = index.write_txn().unwrap(); index.delete_document("38");
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("38");
delete.execute().unwrap();
wtxn.commit().unwrap();
db_snap!(index, documents_ids, @"[1, ]"); db_snap!(index, documents_ids, @"[1, ]");
db_snap!(index, external_documents_ids, 5, @r###" db_snap!(index, external_documents_ids, 5, @r###"
@ -1987,6 +1994,7 @@ pub(crate) mod tests {
fn bug_3021_second() { fn bug_3021_second() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index index
@ -2011,11 +2019,7 @@ pub(crate) mod tests {
"###); "###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]"); db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap(); index.delete_document("34");
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("34");
delete.execute().unwrap();
wtxn.commit().unwrap();
db_snap!(index, documents_ids, @"[0, ]"); db_snap!(index, documents_ids, @"[0, ]");
db_snap!(index, external_documents_ids, 2, @r###" db_snap!(index, external_documents_ids, 2, @r###"
@ -2116,6 +2120,7 @@ pub(crate) mod tests {
fn bug_3021_third() { fn bug_3021_third() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index index
@ -2142,11 +2147,7 @@ pub(crate) mod tests {
"###); "###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]"); db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap(); index.delete_document("3");
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("3");
delete.execute().unwrap();
wtxn.commit().unwrap();
db_snap!(index, documents_ids, @"[1, 2, ]"); db_snap!(index, documents_ids, @"[1, 2, ]");
db_snap!(index, external_documents_ids, 2, @r###" db_snap!(index, external_documents_ids, 2, @r###"
@ -2158,7 +2159,7 @@ pub(crate) mod tests {
"###); "###);
db_snap!(index, soft_deleted_documents_ids, 2, @"[0, ]"); db_snap!(index, soft_deleted_documents_ids, 2, @"[0, ]");
index.index_documents_config.disable_soft_deletion = true; index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap(); index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap();

121
milli/src/update/delete_documents.rs
View File

@ -685,7 +685,7 @@ mod tests {
wtxn: &mut RwTxn<'t, '_>, wtxn: &mut RwTxn<'t, '_>,
index: &'t Index, index: &'t Index,
external_ids: &[&str], external_ids: &[&str],
disable_soft_deletion: bool, strategy: DeletionStrategy,
) -> Vec<u32> { ) -> Vec<u32> {
let external_document_ids = index.external_documents_ids(wtxn).unwrap(); let external_document_ids = index.external_documents_ids(wtxn).unwrap();
let ids_to_delete: Vec<u32> = external_ids let ids_to_delete: Vec<u32> = external_ids
@ -695,14 +695,14 @@ mod tests {
// Delete some documents. // Delete some documents.
let mut builder = DeleteDocuments::new(wtxn, index).unwrap(); let mut builder = DeleteDocuments::new(wtxn, index).unwrap();
builder.disable_soft_deletion(disable_soft_deletion); builder.strategy(strategy);
external_ids.iter().for_each(|id| drop(builder.delete_external_id(id))); external_ids.iter().for_each(|id| drop(builder.delete_external_id(id)));
builder.execute().unwrap(); builder.execute().unwrap();
ids_to_delete ids_to_delete
} }
fn delete_documents_with_numbers_as_primary_key_(disable_soft_deletion: bool) { fn delete_documents_with_numbers_as_primary_key_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new(); let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
@ -722,17 +722,17 @@ mod tests {
builder.delete_document(0); builder.delete_document(0);
builder.delete_document(1); builder.delete_document(1);
builder.delete_document(2); builder.delete_document(2);
builder.disable_soft_deletion(disable_soft_deletion); builder.strategy(deletion_strategy);
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
// All these snapshots should be empty since the database was cleared // All these snapshots should be empty since the database was cleared
db_snap!(index, documents_ids, disable_soft_deletion); db_snap!(index, documents_ids, deletion_strategy);
db_snap!(index, word_docids, disable_soft_deletion); db_snap!(index, word_docids, deletion_strategy);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion); db_snap!(index, word_pair_proximity_docids, deletion_strategy);
db_snap!(index, facet_id_exists_docids, disable_soft_deletion); db_snap!(index, facet_id_exists_docids, deletion_strategy);
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -741,11 +741,11 @@ mod tests {
#[test] #[test]
fn delete_documents_with_numbers_as_primary_key() { fn delete_documents_with_numbers_as_primary_key() {
delete_documents_with_numbers_as_primary_key_(true); delete_documents_with_numbers_as_primary_key_(DeletionStrategy::AlwaysHard);
delete_documents_with_numbers_as_primary_key_(false); delete_documents_with_numbers_as_primary_key_(DeletionStrategy::AlwaysSoft);
} }
fn delete_documents_with_strange_primary_key_(disable_soft_deletion: bool) { fn delete_documents_with_strange_primary_key_(strategy: DeletionStrategy) {
let index = TempIndex::new(); let index = TempIndex::new();
index index
@ -771,24 +771,24 @@ mod tests {
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_external_id("0"); builder.delete_external_id("0");
builder.delete_external_id("1"); builder.delete_external_id("1");
builder.disable_soft_deletion(disable_soft_deletion); builder.strategy(strategy);
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, documents_ids, disable_soft_deletion); db_snap!(index, documents_ids, strategy);
db_snap!(index, word_docids, disable_soft_deletion); db_snap!(index, word_docids, strategy);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion); db_snap!(index, word_pair_proximity_docids, strategy);
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, strategy);
} }
#[test] #[test]
fn delete_documents_with_strange_primary_key() { fn delete_documents_with_strange_primary_key() {
delete_documents_with_strange_primary_key_(true); delete_documents_with_strange_primary_key_(DeletionStrategy::AlwaysHard);
delete_documents_with_strange_primary_key_(false); delete_documents_with_strange_primary_key_(DeletionStrategy::AlwaysSoft);
} }
fn filtered_placeholder_search_should_not_return_deleted_documents_( fn filtered_placeholder_search_should_not_return_deleted_documents_(
disable_soft_deletion: bool, deletion_strategy: DeletionStrategy,
) { ) {
let index = TempIndex::new(); let index = TempIndex::new();
@ -832,7 +832,7 @@ mod tests {
) )
.unwrap(); .unwrap();
delete_documents(&mut wtxn, &index, &["1_4", "1_70", "1_72"], disable_soft_deletion); delete_documents(&mut wtxn, &index, &["1_4", "1_70", "1_72"], deletion_strategy);
// Placeholder search with filter // Placeholder search with filter
let filter = Filter::from_str("label = sign").unwrap().unwrap(); let filter = Filter::from_str("label = sign").unwrap().unwrap();
@ -841,21 +841,27 @@ mod tests {
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
db_snap!(index, word_docids, disable_soft_deletion); db_snap!(index, word_docids, deletion_strategy);
db_snap!(index, facet_id_f64_docids, disable_soft_deletion); db_snap!(index, facet_id_f64_docids, deletion_strategy);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion); db_snap!(index, word_pair_proximity_docids, deletion_strategy);
db_snap!(index, facet_id_exists_docids, disable_soft_deletion); db_snap!(index, facet_id_exists_docids, deletion_strategy);
db_snap!(index, facet_id_string_docids, disable_soft_deletion); db_snap!(index, facet_id_string_docids, deletion_strategy);
} }
#[test] #[test]
fn filtered_placeholder_search_should_not_return_deleted_documents() { fn filtered_placeholder_search_should_not_return_deleted_documents() {
filtered_placeholder_search_should_not_return_deleted_documents_(true); filtered_placeholder_search_should_not_return_deleted_documents_(
filtered_placeholder_search_should_not_return_deleted_documents_(false); DeletionStrategy::AlwaysHard,
);
filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysSoft,
);
} }
fn placeholder_search_should_not_return_deleted_documents_(disable_soft_deletion: bool) { fn placeholder_search_should_not_return_deleted_documents_(
deletion_strategy: DeletionStrategy,
) {
let index = TempIndex::new(); let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
@ -896,8 +902,7 @@ mod tests {
) )
.unwrap(); .unwrap();
let deleted_internal_ids = let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"], deletion_strategy);
delete_documents(&mut wtxn, &index, &["1_4"], disable_soft_deletion);
// Placeholder search // Placeholder search
let results = index.search(&wtxn).execute().unwrap(); let results = index.search(&wtxn).execute().unwrap();
@ -915,11 +920,11 @@ mod tests {
#[test] #[test]
fn placeholder_search_should_not_return_deleted_documents() { fn placeholder_search_should_not_return_deleted_documents() {
placeholder_search_should_not_return_deleted_documents_(true); placeholder_search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
placeholder_search_should_not_return_deleted_documents_(false); placeholder_search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
} }
fn search_should_not_return_deleted_documents_(disable_soft_deletion: bool) { fn search_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new(); let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
@ -961,7 +966,7 @@ mod tests {
.unwrap(); .unwrap();
let deleted_internal_ids = let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], disable_soft_deletion); delete_documents(&mut wtxn, &index, &["1_7", "1_52"], deletion_strategy);
// search for abstract // search for abstract
let results = index.search(&wtxn).query("abstract").execute().unwrap(); let results = index.search(&wtxn).query("abstract").execute().unwrap();
@ -976,17 +981,17 @@ mod tests {
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
} }
#[test] #[test]
fn search_should_not_return_deleted_documents() { fn search_should_not_return_deleted_documents() {
search_should_not_return_deleted_documents_(true); search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
search_should_not_return_deleted_documents_(false); search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
} }
fn geo_filtered_placeholder_search_should_not_return_deleted_documents_( fn geo_filtered_placeholder_search_should_not_return_deleted_documents_(
disable_soft_deletion: bool, deletion_strategy: DeletionStrategy,
) { ) {
let index = TempIndex::new(); let index = TempIndex::new();
@ -1024,7 +1029,7 @@ mod tests {
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"]; let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
let deleted_internal_ids = let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &external_ids_to_delete, disable_soft_deletion); delete_documents(&mut wtxn, &index, &external_ids_to_delete, deletion_strategy);
// Placeholder search with geo filter // Placeholder search with geo filter
let filter = Filter::from_str("_geoRadius(50.6924, 3.1763, 20000)").unwrap().unwrap(); let filter = Filter::from_str("_geoRadius(50.6924, 3.1763, 20000)").unwrap().unwrap();
@ -1040,18 +1045,22 @@ mod tests {
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
db_snap!(index, facet_id_f64_docids, disable_soft_deletion); db_snap!(index, facet_id_f64_docids, deletion_strategy);
db_snap!(index, facet_id_string_docids, disable_soft_deletion); db_snap!(index, facet_id_string_docids, deletion_strategy);
} }
#[test] #[test]
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() { fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
geo_filtered_placeholder_search_should_not_return_deleted_documents_(true); geo_filtered_placeholder_search_should_not_return_deleted_documents_(
geo_filtered_placeholder_search_should_not_return_deleted_documents_(false); DeletionStrategy::AlwaysHard,
);
geo_filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysSoft,
);
} }
fn get_documents_should_not_return_deleted_documents_(disable_soft_deletion: bool) { fn get_documents_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new(); let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
@ -1094,7 +1103,7 @@ mod tests {
let deleted_external_ids = ["1_7", "1_52"]; let deleted_external_ids = ["1_7", "1_52"];
let deleted_internal_ids = let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &deleted_external_ids, disable_soft_deletion); delete_documents(&mut wtxn, &index, &deleted_external_ids, deletion_strategy);
// list all documents // list all documents
let results = index.all_documents(&wtxn).unwrap(); let results = index.all_documents(&wtxn).unwrap();
@ -1125,16 +1134,16 @@ mod tests {
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
} }
#[test] #[test]
fn get_documents_should_not_return_deleted_documents() { fn get_documents_should_not_return_deleted_documents() {
get_documents_should_not_return_deleted_documents_(true); get_documents_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
get_documents_should_not_return_deleted_documents_(false); get_documents_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
} }
fn stats_should_not_return_deleted_documents_(disable_soft_deletion: bool) { fn stats_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new(); let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
@ -1168,7 +1177,7 @@ mod tests {
{ "docid": "1_69", "label": ["geometry"]} { "docid": "1_69", "label": ["geometry"]}
])).unwrap(); ])).unwrap();
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], disable_soft_deletion); delete_documents(&mut wtxn, &index, &["1_7", "1_52"], deletion_strategy);
// count internal documents // count internal documents
let results = index.number_of_documents(&wtxn).unwrap(); let results = index.number_of_documents(&wtxn).unwrap();
@ -1182,12 +1191,12 @@ mod tests {
wtxn.commit().unwrap(); wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion); db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
} }
#[test] #[test]
fn stats_should_not_return_deleted_documents() { fn stats_should_not_return_deleted_documents() {
stats_should_not_return_deleted_documents_(true); stats_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
stats_should_not_return_deleted_documents_(false); stats_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
} }
} }

8
milli/src/update/facet/delete.rs
View File

@ -122,7 +122,7 @@ mod tests {
use crate::documents::documents_batch_reader_from_objects; use crate::documents::documents_batch_reader_from_objects;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::update::facet::test_helpers::ordered_string; use crate::update::facet::test_helpers::ordered_string;
use crate::update::DeleteDocuments; use crate::update::{DeleteDocuments, DeletionStrategy};
#[test] #[test]
fn delete_mixed_incremental_and_bulk() { fn delete_mixed_incremental_and_bulk() {
@ -165,7 +165,7 @@ mod tests {
let mut wtxn = index.env.write_txn().unwrap(); let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true); builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter(0..100)); builder.delete_documents(&RoaringBitmap::from_iter(0..100));
// by deleting the first 100 documents, we expect that: // by deleting the first 100 documents, we expect that:
// - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13) // - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13)
@ -224,7 +224,7 @@ mod tests {
let mut wtxn = index.env.write_txn().unwrap(); let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true); builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter(0..100)); builder.delete_documents(&RoaringBitmap::from_iter(0..100));
// by deleting the first 100 documents, we expect that: // by deleting the first 100 documents, we expect that:
// - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13) // - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13)
@ -283,7 +283,7 @@ mod tests {
for docid in docids_to_delete.into_iter().take(990) { for docid in docids_to_delete.into_iter().take(990) {
let mut wtxn = index.env.write_txn().unwrap(); let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true); builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter([docid])); builder.delete_documents(&RoaringBitmap::from_iter([docid]));
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

5
milli/src/update/facet/mod.rs
View File

@ -463,11 +463,14 @@ mod tests {
use crate::db_snap; use crate::db_snap;
use crate::documents::documents_batch_reader_from_objects; use crate::documents::documents_batch_reader_from_objects;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::update::DeletionStrategy;
#[test] #[test]
fn replace_all_identical_soft_deletion_then_hard_deletion() { fn replace_all_identical_soft_deletion_then_hard_deletion() {
let mut index = TempIndex::new_with_map_size(4096 * 1000 * 100); let mut index = TempIndex::new_with_map_size(4096 * 1000 * 100);
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_primary_key("id".to_owned()); settings.set_primary_key("id".to_owned());
@ -521,7 +524,7 @@ mod tests {
db_snap!(index, soft_deleted_documents_ids, "replaced_1_soft", @"6c975deb900f286d2f6456d2d5c3a123"); db_snap!(index, soft_deleted_documents_ids, "replaced_1_soft", @"6c975deb900f286d2f6456d2d5c3a123");
// Then replace the last document while disabling soft_deletion // Then replace the last document while disabling soft_deletion
index.index_documents_config.disable_soft_deletion = true; index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut documents = vec![]; let mut documents = vec![];
for i in 999..1000 { for i in 999..1000 {
documents.push( documents.push(

12
milli/src/update/prefix_word_pairs/mod.rs
View File

@ -163,7 +163,7 @@ mod tests {
use crate::db_snap; use crate::db_snap;
use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::update::{DeleteDocuments, IndexDocumentsMethod}; use crate::update::{DeleteDocuments, DeletionStrategy, IndexDocumentsMethod};
fn documents_with_enough_different_words_for_prefixes( fn documents_with_enough_different_words_for_prefixes(
prefixes: &[&str], prefixes: &[&str],
@ -351,7 +351,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.disable_soft_deletion(true); delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter([50])); delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -363,7 +363,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.disable_soft_deletion(true); delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter(0..50)); delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -435,6 +435,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter([50])); delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -446,6 +447,8 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter(0..50)); delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -471,6 +474,7 @@ mod tests {
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50); index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index index
.update_settings(|settings| { .update_settings(|settings| {
@ -530,7 +534,7 @@ mod tests {
fn replace_hard_deletion() { fn replace_hard_deletion() {
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50); index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.disable_soft_deletion = true; index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index index