From 36296bbb20e9c545d131117d85b1d3718d985378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Thu, 1 Sep 2022 11:33:50 +0200 Subject: [PATCH] Add facet incremental indexing snapshot tests + fix bug --- milli/src/search/facet/mod.rs | 13 +- .../filter_distribution_all/0.snap | 32 + .../filter_distribution_all/1.snap | 5 + .../filter_distribution_all_stop_early/1.snap | 4 + .../random_looking_index_snap.hash.snap | 2 +- .../filter_range_decreasing/0.hash.snap | 2 +- .../filter_range_decreasing/1.hash.snap | 2 +- .../filter_range_increasing/0.hash.snap | 2 +- .../filter_range_increasing/1.hash.snap | 2 +- .../filter_range_pinch/0.hash.snap | 2 +- .../filter_range_pinch/1.hash.snap | 2 +- .../random_looking_index_snap.hash.snap | 2 +- .../filter_sort/0.snap | 32 + .../filter_sort/1.snap | 1 + .../random_looking_index_snap.hash.snap | 2 +- .../filter_sort_descending/0.snap | 44 ++ .../filter_sort_descending/1.snap | 9 +- .../random_looking_index_snap.hash.snap | 2 +- milli/src/update/facet/incremental.rs | 679 +++++++++++++++++- .../default/facet_id_f64_docids.hash.snap | 2 +- .../facet_id_f64_docids.hash.snap | 2 +- .../facet_id_f64_docids.hash.snap | 2 +- .../facet_id_f64_docids.hash.snap | 2 +- .../facet_id_f64_docids.hash.snap | 2 +- .../facet_id_f64_docids.hash.snap | 2 +- .../default/facet_id_string_docids.hash.snap | 2 +- .../facet_id_string_docids.hash.snap | 2 +- .../incremental.rs/append/append.hash.snap | 4 + .../incremental.rs/delete_from_end/0.snap | 4 + .../delete_from_end/100.hash.snap | 4 + .../incremental.rs/delete_from_end/15.snap | 23 + .../delete_from_end/150.hash.snap | 4 + .../incremental.rs/delete_from_end/17.snap | 26 + .../delete_from_end/200.hash.snap | 4 + .../delete_from_start/127.hash.snap | 4 + .../incremental.rs/delete_from_start/215.snap | 54 ++ .../incremental.rs/delete_from_start/255.snap | 4 + .../delete_shuffled/127.hash.snap | 4 + .../delete_shuffled/215.hash.snap | 4 + .../incremental.rs/delete_shuffled/255.snap | 4 + .../in_place_level0_delete.hash.snap | 4 + .../in_place_level0_insert.snap | 20 + .../many_field_ids_append.hash.snap | 4 + .../many_field_ids_prepend.hash.snap | 4 + .../merge_values/merge_values.hash.snap | 4 + .../incremental.rs/prepend/prepend.hash.snap | 4 + .../shuffle_merge_string/1.hash.snap | 4 + .../shuffle_merge_string/2.hash.snap | 4 + .../shuffled/shuffled.hash.snap | 4 + 49 files changed, 1028 insertions(+), 22 deletions(-) create mode 100644 milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/in_place_level0_delete.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/1.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/2.hash.snap create mode 100644 milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap diff --git a/milli/src/search/facet/mod.rs b/milli/src/search/facet/mod.rs index 0ed80dd92..42c0f065a 100644 --- a/milli/src/search/facet/mod.rs +++ b/milli/src/search/facet/mod.rs @@ -77,7 +77,7 @@ pub(crate) fn get_highest_level<'t>( } #[cfg(test)] -mod test { +pub mod test { use crate::update::FacetsUpdateIncremental; use heed::{BytesDecode, BytesEncode, Env, RwTxn}; use roaring::RoaringBitmap; @@ -160,6 +160,17 @@ mod test { let key_bytes = BoundCodec::bytes_encode(&key).unwrap(); update.insert(rwtxn, field_id, &key_bytes, docids).unwrap(); } + pub fn delete<'a>( + &self, + rwtxn: &'a mut RwTxn, + field_id: u16, + key: &'a >::EItem, + value: u32, + ) { + let update = FacetsUpdateIncremental::new(self.db.content); + let key_bytes = BoundCodec::bytes_encode(&key).unwrap(); + update.delete(rwtxn, field_id, &key_bytes, value).unwrap(); + } } impl Display for FacetIndex diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap index fe5f69d7d..2b6123289 100644 --- a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap +++ b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap @@ -225,4 +225,36 @@ source: milli/src/search/facet/facet_distribution_iter.rs 221: 1 222: 1 223: 1 +224: 1 +225: 1 +226: 1 +227: 1 +228: 1 +229: 1 +230: 1 +231: 1 +232: 1 +233: 1 +234: 1 +235: 1 +236: 1 +237: 1 +238: 1 +239: 1 +240: 1 +241: 1 +242: 1 +243: 1 +244: 1 +245: 1 +246: 1 +247: 1 +248: 1 +249: 1 +250: 1 +251: 1 +252: 1 +253: 1 +254: 1 +255: 1 diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap index dd5e761ea..d0c0dd98d 100644 --- a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap +++ b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap @@ -96,5 +96,10 @@ source: milli/src/search/facet/facet_distribution_iter.rs 216: 1 219: 1 220: 1 +223: 1 226: 1 +235: 1 +236: 1 +238: 1 +243: 1 diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap index dd5e761ea..95c719bb0 100644 --- a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap +++ b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap @@ -96,5 +96,9 @@ source: milli/src/search/facet/facet_distribution_iter.rs 216: 1 219: 1 220: 1 +223: 1 226: 1 +235: 1 +236: 1 +238: 1 diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/random_looking_index_snap/random_looking_index_snap.hash.snap b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/random_looking_index_snap/random_looking_index_snap.hash.snap index da2b49adc..661e1a35b 100644 --- a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/random_looking_index_snap/random_looking_index_snap.hash.snap +++ b/milli/src/search/facet/snapshots/facet_distribution_iter.rs/random_looking_index_snap/random_looking_index_snap.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_distribution_iter.rs --- -ea4022977d09c7854c833146276348de +3256c76a7c1b768a013e78d5fa6e9ff9 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/0.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/0.hash.snap index e835d8934..7bf13e05c 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/0.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/0.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -52d0b31f312572c10959418434e36581 +fcedc563a82c1c61f50174a5f3f982b6 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/1.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/1.hash.snap index 150f00f7b..100b928d7 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/1.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/1.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -2cb9e819529823d488e141edb4307f97 +6cc26e77fc6bd9145deedf14cf422b03 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/0.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/0.hash.snap index 4f05823f4..db11ce952 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/0.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/0.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -38a4352c48905f5b121d1217734862da +c1c7a0bb91d53d33724583b6d4a99f16 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/1.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/1.hash.snap index d2c8a3559..f5a81c121 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/1.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/1.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -aefc1ec120fa884cc8396a68bd7de42f +12213d3f1047a0c3d08e4670a7d688e7 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/0.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/0.hash.snap index 3fb0c94b0..07664807e 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/0.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/0.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -9e360d7bcd29ac2c23bc241df941fd23 +3456db9a1bb94c33c1e9f656184ee711 diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/1.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/1.hash.snap index 44fa88004..ef530faa1 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/1.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/1.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -f0606b9af67de9ede9d469514ea1741f +2127cd818b457e0611e0c8e1a871602a diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/random_looking_index_snap/random_looking_index_snap.hash.snap b/milli/src/search/facet/snapshots/facet_range_search.rs/random_looking_index_snap/random_looking_index_snap.hash.snap index cf4b29ba3..67a2f6bd9 100644 --- a/milli/src/search/facet/snapshots/facet_range_search.rs/random_looking_index_snap/random_looking_index_snap.hash.snap +++ b/milli/src/search/facet/snapshots/facet_range_search.rs/random_looking_index_snap/random_looking_index_snap.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_range_search.rs --- -ea4022977d09c7854c833146276348de +3256c76a7c1b768a013e78d5fa6e9ff9 diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/0.snap b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/0.snap index 9dcd92ed7..2d0f6e213 100644 --- a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/0.snap +++ b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/0.snap @@ -25,4 +25,36 @@ source: milli/src/search/facet/facet_sort_ascending.rs [221, ] [222, ] [223, ] +[224, ] +[225, ] +[226, ] +[227, ] +[228, ] +[229, ] +[230, ] +[231, ] +[232, ] +[233, ] +[234, ] +[235, ] +[236, ] +[237, ] +[238, ] +[239, ] +[240, ] +[241, ] +[242, ] +[243, ] +[244, ] +[245, ] +[246, ] +[247, ] +[248, ] +[249, ] +[250, ] +[251, ] +[252, ] +[253, ] +[254, ] +[255, ] diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/1.snap b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/1.snap index a81e7377b..20d666494 100644 --- a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/1.snap +++ b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/1.snap @@ -50,4 +50,5 @@ source: milli/src/search/facet/facet_sort_ascending.rs [216, ] [220, ] [226, ] +[238, ] diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap index 785ff325c..64ff762db 100644 --- a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap +++ b/milli/src/search/facet/snapshots/facet_sort_ascending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_sort_ascending.rs --- -ea4022977d09c7854c833146276348de +3256c76a7c1b768a013e78d5fa6e9ff9 diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap b/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap index 05a18f000..032763c74 100644 --- a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap +++ b/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap @@ -1,10 +1,54 @@ --- source: milli/src/search/facet/facet_sort_descending.rs --- +[255, ] +[254, ] +[253, ] +[252, ] +[251, ] +[250, ] +[249, ] +[248, ] [247, ] [246, ] [245, ] [244, ] +[243, ] +[242, ] +[241, ] +[240, ] +[239, ] +[238, ] +[237, ] +[236, ] +[235, ] +[234, ] +[233, ] +[232, ] +[231, ] +[230, ] +[229, ] +[228, ] +[227, ] +[226, ] +[225, ] +[224, ] +[223, ] +[222, ] +[221, ] +[220, ] +[219, ] +[218, ] +[217, ] +[216, ] +[215, ] +[214, ] +[213, ] +[212, ] +[211, ] +[210, ] +[209, ] +[208, ] [207, ] [206, ] [205, ] diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap b/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap index 9890c1aab..4c62cfee4 100644 --- a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap +++ b/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap @@ -2,8 +2,15 @@ source: milli/src/search/facet/facet_sort_descending.rs --- [243, ] +[238, ] +[236, ] [235, ] [226, ] +[223, ] +[220, ] +[219, ] +[216, ] +[210, ] [209, ] [208, ] [207, ] @@ -35,12 +42,10 @@ source: milli/src/search/facet/facet_sort_descending.rs [241, ] [239, ] [237, ] -[236, ] [233, ] [231, ] [230, ] [224, ] -[223, ] [215, ] [211, ] [203, ] diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap b/milli/src/search/facet/snapshots/facet_sort_descending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap index b68843376..0649e3c5d 100644 --- a/milli/src/search/facet/snapshots/facet_sort_descending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap +++ b/milli/src/search/facet/snapshots/facet_sort_descending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/search/facet/facet_sort_descending.rs --- -ea4022977d09c7854c833146276348de +3256c76a7c1b768a013e78d5fa6e9ff9 diff --git a/milli/src/update/facet/incremental.rs b/milli/src/update/facet/incremental.rs index 6dd1f7ac5..712d7271c 100644 --- a/milli/src/update/facet/incremental.rs +++ b/milli/src/update/facet/incremental.rs @@ -297,7 +297,7 @@ impl FacetsUpdateIncremental { .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &highest_level_prefix)?; let mut to_add = vec![]; - for _ in 0..group_size { + for _ in 0..self.min_level_size { let mut first_key = None; let mut values = RoaringBitmap::new(); for _ in 0..group_size { @@ -459,3 +459,680 @@ impl FacetsUpdateIncremental { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::milli_snap; + use crate::{ + heed_codec::facet::new::{ + ordered_f64_codec::OrderedF64Codec, str_ref::StrRefCodec, FacetGroupValueCodec, + FacetKeyCodec, MyByteSlice, + }, + search::facet::{get_highest_level, test::FacetIndex}, + }; + use heed::{types::ByteSlice, BytesDecode, BytesEncode}; + use rand::Rng; + use rand::{seq::SliceRandom, SeedableRng}; + use roaring::RoaringBitmap; + + pub fn verify_structure_validity(index: &FacetIndex, field_id: u16) + where + for<'a> C: BytesDecode<'a> + BytesEncode<'a, EItem = >::DItem>, + { + let FacetIndex { env, db, .. } = index; + + let txn = env.write_txn().unwrap(); + let mut field_id_prefix = vec![]; + field_id_prefix.extend_from_slice(&field_id.to_be_bytes()); + + let highest_level = get_highest_level(&txn, index.db.content, field_id).unwrap(); + txn.commit().unwrap(); + + let txn = env.read_txn().unwrap(); + for level_no in (1..=highest_level).rev() { + let mut level_no_prefix = vec![]; + level_no_prefix.extend_from_slice(&field_id.to_be_bytes()); + level_no_prefix.push(level_no); + + let mut iter = db + .content + .as_polymorph() + .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &level_no_prefix) + .unwrap(); + while let Some(el) = iter.next() { + let (key, value) = el.unwrap(); + let key = FacetKeyCodec::::bytes_decode(&key).unwrap(); + + let mut prefix_start_below = vec![]; + prefix_start_below.extend_from_slice(&field_id.to_be_bytes()); + prefix_start_below.push(level_no - 1); + prefix_start_below.extend_from_slice(&key.left_bound); + + let start_below = { + let mut start_below_iter = db + .content + .as_polymorph() + .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>( + &txn, + &prefix_start_below, + ) + .unwrap(); + let (key_bytes, _) = start_below_iter.next().unwrap().unwrap(); + FacetKeyCodec::::bytes_decode(&key_bytes).unwrap() + }; + + assert!(value.size > 0 && (value.size as usize) < db.max_group_size); + + let mut actual_size = 0; + let mut values_below = RoaringBitmap::new(); + let mut iter_below = + db.content.range(&txn, &(start_below..)).unwrap().take(value.size as usize); + while let Some(el) = iter_below.next() { + let (_, value) = el.unwrap(); + actual_size += 1; + values_below |= value.bitmap; + } + assert_eq!(actual_size, value.size, "{key:?} start_below: {start_below:?}"); + + assert_eq!(value.bitmap, values_below); + } + } + } + #[test] + fn append() { + let index = FacetIndex::::new(4, 8); + for i in 0..256u16 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + #[test] + fn many_field_ids_append() { + let index = FacetIndex::::new(4, 8); + for i in 0..256u16 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + for i in 0..256u16 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 2, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + for i in 0..256u16 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 1, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + verify_structure_validity(&index, 1); + verify_structure_validity(&index, 2); + milli_snap!(format!("{index}")); + } + #[test] + fn many_field_ids_prepend() { + let index = FacetIndex::::new(4, 8); + for i in (0..256).into_iter().rev() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + for i in (0..256).into_iter().rev() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 2, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + for i in (0..256).into_iter().rev() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i as u32); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 1, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + verify_structure_validity(&index, 1); + verify_structure_validity(&index, 2); + milli_snap!(format!("{index}")); + } + + #[test] + fn prepend() { + let index = FacetIndex::::new(4, 8); + let mut txn = index.env.write_txn().unwrap(); + + for i in (0..256).into_iter().rev() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + } + txn.commit().unwrap(); + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + + #[test] + fn shuffled() { + let index = FacetIndex::::new(4, 8); + let mut txn = index.env.write_txn().unwrap(); + + let mut keys = (0..256).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + + for (_i, key) in keys.into_iter().enumerate() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(key); + index.insert(&mut txn, 0, &(key as f64), &bitmap); + } + txn.commit().unwrap(); + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + + #[test] + fn merge_values() { + let index = FacetIndex::::new(4, 8); + + let mut keys = (0..256).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + for (_i, key) in keys.into_iter().enumerate() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(key); + bitmap.insert(rng.gen_range(256..512)); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(key as f64), &bitmap); + txn.commit().unwrap(); + } + + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + + #[test] + fn delete_from_end() { + let index = FacetIndex::::new(4, 8); + for i in 0..256 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(&(i as f64)), &bitmap); + txn.commit().unwrap(); + } + + for i in (200..256).into_iter().rev() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 200); + + for i in (150..200).into_iter().rev() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 150); + + for i in (100..150).into_iter().rev() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 100); + + for i in (17..100).into_iter().rev() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 17); + + let mut txn = index.env.write_txn().unwrap(); + for i in (15..17).into_iter().rev() { + index.delete(&mut txn, 0, &(i as f64), i as u32); + } + txn.commit().unwrap(); + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 15); + for i in (0..15).into_iter().rev() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 0); + } + + #[test] + fn delete_from_start() { + let index = FacetIndex::::new(4, 8); + + for i in 0..256 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + + for i in 0..128 { + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 127); + for i in 128..216 { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 215); + for i in 216..256 { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(i as f64), i as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 255); + } + + #[test] + fn delete_shuffled() { + let index = FacetIndex::::new(4, 8); + + for i in 0..256 { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(i); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(i as f64), &bitmap); + txn.commit().unwrap(); + } + + let mut keys = (0..256).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + + for i in 0..128 { + let key = keys[i]; + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(key as f64), key as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 127); + for i in 128..216 { + let key = keys[i]; + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(key as f64), key as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 215); + for i in 216..256 { + let key = keys[i]; + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(key as f64), key as u32); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 255); + } + + #[test] + fn in_place_level0_insert() { + let index = FacetIndex::::new(4, 8); + let mut keys = (0..16).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + for i in 0..4 { + for &key in keys.iter() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(rng.gen_range(i * 256..(i + 1) * 256)); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(key as f64), &bitmap); + txn.commit().unwrap(); + } + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + + #[test] + fn in_place_level0_delete() { + let index = FacetIndex::::new(4, 8); + + let mut keys = (0..64).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + + for &key in keys.iter() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(key); + bitmap.insert(key + 100); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &(key as f64), &bitmap); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + + for &key in keys.iter() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &(key as f64), key + 100); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}")); + } + + #[test] + fn shuffle_merge_string() { + let index = FacetIndex::::new(4, 8); + + let mut keys = (1000..1064).into_iter().collect::>(); + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + keys.shuffle(&mut rng); + + for &key in keys.iter() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(key); + bitmap.insert(key + 100); + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.insert(&mut txn, 0, &format!("{key:x}").as_str(), &bitmap); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 1); + + for &key in keys.iter() { + verify_structure_validity(&index, 0); + let mut txn = index.env.write_txn().unwrap(); + index.delete(&mut txn, 0, &format!("{key:x}").as_str(), key + 100); + txn.commit().unwrap(); + } + verify_structure_validity(&index, 0); + milli_snap!(format!("{index}"), 2); + } + + // fuzz tests +} +// #[cfg(all(test, fuzzing))] +// mod fuzz { +// use crate::codec::U16Codec; + +// use super::tests::verify_structure_validity; +// use super::*; +// use fuzzcheck::mutators::integer_within_range::U16WithinRangeMutator; +// use fuzzcheck::DefaultMutator; +// use roaring::RoaringBitmap; +// use std::collections::BTreeMap; +// use std::collections::HashMap; + +// #[derive(Default)] +// pub struct TrivialDatabase { +// pub elements: BTreeMap>, +// } +// impl TrivialDatabase +// where +// T: Ord + Clone + Copy + Eq + std::fmt::Debug, +// { +// pub fn insert(&mut self, field_id: u16, new_key: T, new_values: &RoaringBitmap) { +// if new_values.is_empty() { +// return; +// } +// let values_field_id = self.elements.entry(field_id).or_default(); +// let values = values_field_id.entry(new_key).or_default(); +// *values |= new_values; +// } +// pub fn delete(&mut self, field_id: u16, key: T, value: u32) { +// if let Some(values_field_id) = self.elements.get_mut(&field_id) { +// if let Some(values) = values_field_id.get_mut(&key) { +// values.remove(value); +// if values.is_empty() { +// values_field_id.remove(&key); +// } +// } +// if values_field_id.is_empty() { +// self.elements.remove(&field_id); +// } +// } +// } +// } +// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)] +// struct Operation { +// key: Key, +// #[field_mutator(U16WithinRangeMutator = { U16WithinRangeMutator::new(..=3) })] +// field_id: u16, +// kind: OperationKind, +// } +// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)] +// enum OperationKind { +// Insert(Vec), +// Delete(u8), +// } + +// fn compare_with_trivial_database( +// tempdir: Rc, +// group_size: u8, +// max_group_size: u8, +// operations: &[Operation], +// ) { +// let index = FacetIndex::::open_from_tempdir(tempdir, group_size, max_group_size); +// let mut trivial_db = TrivialDatabase::::default(); +// let mut value_to_keys = HashMap::>::new(); +// let mut txn = index.env.write_txn().unwrap(); +// for Operation { key, field_id, kind } in operations { +// match kind { +// OperationKind::Insert(values) => { +// let mut bitmap = RoaringBitmap::new(); +// for value in values { +// bitmap.insert(*value as u32); +// value_to_keys.entry(*value).or_default().push(*key); +// } +// index.insert(&mut txn, *field_id, key, &bitmap); +// trivial_db.insert(*field_id, *key, &bitmap); +// } +// OperationKind::Delete(value) => { +// if let Some(keys) = value_to_keys.get(value) { +// for key in keys { +// index.delete(&mut txn, *field_id, key, *value as u32); +// trivial_db.delete(*field_id, *key, *value as u32); +// } +// } +// } +// } +// } +// for (field_id, values_field_id) in trivial_db.elements.iter() { +// let level0iter = index +// .db +// .content +// .as_polymorph() +// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>( +// &mut txn, +// &field_id.to_be_bytes(), +// ) +// .unwrap(); + +// for ((key, values), group) in values_field_id.iter().zip(level0iter) { +// let (group_key, group_values) = group.unwrap(); +// let group_key = FacetKeyCodec::::bytes_decode(group_key).unwrap(); +// assert_eq!(key, &group_key.left_bound); +// assert_eq!(values, &group_values.bitmap); +// } +// } + +// txn.commit().unwrap(); +// let mut txn = index.env.write_txn().unwrap(); +// for (field_id, values_field_id) in trivial_db.elements.iter() { +// let level0iter = index +// .db +// .content +// .as_polymorph() +// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &field_id.to_be_bytes()) +// .unwrap(); + +// for ((key, values), group) in values_field_id.iter().zip(level0iter) { +// let (group_key, group_values) = group.unwrap(); +// let group_key = FacetKeyCodec::::bytes_decode(group_key).unwrap(); +// assert_eq!(key, &group_key.left_bound); +// assert_eq!(values, &group_values.bitmap); +// } +// verify_structure_validity(&index, *field_id); +// } + +// index.db.content.clear(&mut txn).unwrap(); +// txn.commit().unwrap(); +// } + +// #[test] +// fn fuzz() { +// let tempdir = Rc::new(TempDir::new().unwrap()); +// let tempdir_cloned = tempdir.clone(); +// let result = fuzzcheck::fuzz_test(move |x: &(u8, u8, Vec>)| { +// compare_with_trivial_database(tempdir_cloned.clone(), x.0, x.1, &x.2) +// }) +// .default_mutator() +// .serde_serializer() +// .default_sensor_and_pool_with_custom_filter(|file, function| { +// if file.is_relative() +// && !function.contains("serde") +// && !function.contains("tests::") +// && !function.contains("fuzz::") +// && !function.contains("display_bitmap") +// { +// true +// } else { +// false +// } +// }) +// .arguments_from_cargo_fuzzcheck() +// .launch(); +// assert!(!result.found_test_failure); +// } + +// #[test] +// fn reproduce_bug() { +// let operations = r#" +// [ +// {"key":0, "field_id": 0, "kind":{"Insert":[109]}}, +// {"key":143, "field_id": 0, "kind":{"Insert":[243]}}, +// {"key":90, "field_id": 0, "kind":{"Insert":[217]}}, +// {"key":172, "field_id": 0, "kind":{"Insert":[94]}}, +// {"key":27, "field_id": 0, "kind":{"Insert":[4]}}, +// {"key":124, "field_id": 0, "kind":{"Insert":[0]}}, +// {"key":123, "field_id": 0, "kind":{"Insert":[0]}}, +// {"key":67, "field_id": 0, "kind":{"Insert":[109]}}, +// {"key":13, "field_id": 0, "kind":{"Insert":[0]}}, +// {"key":162, "field_id": 0, "kind":{"Insert":[213]}}, +// {"key":235, "field_id": 0, "kind":{"Insert":[67]}}, +// {"key":251, "field_id": 0, "kind":{"Insert":[50]}}, +// {"key":218, "field_id": 0, "kind":{"Insert":[164]}}, +// {"key":166, "field_id": 0, "kind":{"Insert":[67]}}, +// {"key":64, "field_id": 0, "kind":{"Insert":[61]}}, +// {"key":183, "field_id": 0, "kind":{"Insert":[210]}}, +// {"key":250, "field_id": 0, "kind":{"Delete":50}} +// ] +// "#; +// let operations: Vec> = serde_json::from_str(operations).unwrap(); +// let tempdir = TempDir::new().unwrap(); +// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations); +// } + +// #[test] +// fn reproduce_bug2() { +// let operations = r#" +// [ +// {"key":102, "field_id": 0, "kind":{"Insert":[122]}}, +// {"key":73, "field_id": 0, "kind":{"Insert":[132]}}, +// {"key":20, "field_id": 0, "kind":{"Insert":[215]}}, +// {"key":39, "field_id": 0, "kind":{"Insert":[152]}}, +// {"key":151, "field_id": 0, "kind":{"Insert":[226]}}, +// {"key":17, "field_id": 0, "kind":{"Insert":[101]}}, +// {"key":74, "field_id": 0, "kind":{"Insert":[210]}}, +// {"key":2, "field_id": 0, "kind":{"Insert":[130]}}, +// {"key":64, "field_id": 0, "kind":{"Insert":[180]}}, +// {"key":83, "field_id": 0, "kind":{"Insert":[250]}}, +// {"key":80, "field_id": 0, "kind":{"Insert":[210]}}, +// {"key":113, "field_id": 0, "kind":{"Insert":[63]}}, +// {"key":201, "field_id": 0, "kind":{"Insert":[210]}}, +// {"key":200, "field_id": 0, "kind":{"Insert":[5]}}, +// {"key":93, "field_id": 0, "kind":{"Insert":[98]}}, +// {"key":162, "field_id": 0, "kind":{"Insert":[5]}}, +// {"key":80, "field_id": 0, "kind":{"Delete":210}} +// ] +// "#; +// let operations: Vec> = serde_json::from_str(operations).unwrap(); +// let tempdir = TempDir::new().unwrap(); +// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations); +// } +// #[test] +// fn reproduce_bug3() { +// let operations = r#" +// [ +// {"key":27488, "field_id": 0, "kind":{"Insert":[206]}}, +// {"key":64716, "field_id": 0, "kind":{"Insert":[216]}}, +// {"key":60886, "field_id": 0, "kind":{"Insert":[206]}}, +// {"key":59509, "field_id": 0, "kind":{"Insert":[187,231]}}, +// {"key":55057, "field_id": 0, "kind":{"Insert":[37]}}, +// {"key":45200, "field_id": 0, "kind":{"Insert":[206]}}, +// {"key":55056, "field_id": 0, "kind":{"Insert":[37]}}, +// {"key":63679, "field_id": 0, "kind":{"Insert":[206]}}, +// {"key":52155, "field_id": 0, "kind":{"Insert":[74]}}, +// {"key":20648, "field_id": 0, "kind":{"Insert":[47,138,157]}} +// ] +// "#; +// let operations: Vec> = serde_json::from_str(operations).unwrap(); +// let tempdir = TempDir::new().unwrap(); +// compare_with_trivial_database(Rc::new(tempdir), 0, 7, &operations); +// } + +// #[test] +// fn reproduce_bug4() { +// let operations = r#" +// [{"key":63499, "field_id": 0, "kind":{"Insert":[87]}},{"key":25374, "field_id": 0, "kind":{"Insert":[14]}},{"key":64481, "field_id": 0, "kind":{"Delete":87}},{"key":23038, "field_id": 0, "kind":{"Insert":[173]}},{"key":14862, "field_id": 0, "kind":{"Insert":[8]}},{"key":13145, "field_id": 0, "kind":{"Insert":[5,64]}},{"key":23446, "field_id": 0, "kind":{"Insert":[86,59]}},{"key":17972, "field_id": 0, "kind":{"Insert":[58,137]}},{"key":21273, "field_id": 0, "kind":{"Insert":[121,132,81,147]}},{"key":28264, "field_id": 0, "kind":{"Insert":[36]}},{"key":46659, "field_id": 0, "kind":{"Insert":[]}}] +// "#; +// let operations: Vec> = serde_json::from_str(operations).unwrap(); +// let tempdir = TempDir::new().unwrap(); +// compare_with_trivial_database(Rc::new(tempdir), 2, 1, &operations); +// } +// } diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/default/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/default/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/default/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/default/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_large_levels/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_large_levels/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_large_levels/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_large_levels/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_small_levels/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_small_levels/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_small_levels/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_small_levels/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_large_levels/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_large_levels/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_large_levels/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_large_levels/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_small_levels/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_small_levels/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_small_levels/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_small_levels/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/tiny_groups_tiny_levels/facet_id_f64_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/tiny_groups_tiny_levels/facet_id_f64_docids.hash.snap index b165e1619..b990c31c7 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/tiny_groups_tiny_levels/facet_id_f64_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_number/tiny_groups_tiny_levels/facet_id_f64_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -8bc439472ccda008dc5c28aa789f433d +947949d1a5c9c4e895c89fba46cbba68 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/default/facet_id_string_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/default/facet_id_string_docids.hash.snap index bc0668408..7ed43424a 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/default/facet_id_string_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/default/facet_id_string_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -834f27a924de1acbd3cd94c0d7f10315 +5ce8009d3eb023e4b9c0a6e7fa4e6262 diff --git a/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/tiny_groups_tiny_levels/facet_id_string_docids.hash.snap b/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/tiny_groups_tiny_levels/facet_id_string_docids.hash.snap index bc0668408..7ed43424a 100644 --- a/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/tiny_groups_tiny_levels/facet_id_string_docids.hash.snap +++ b/milli/src/update/facet/snapshots/bulk.rs/test_facets_string/tiny_groups_tiny_levels/facet_id_string_docids.hash.snap @@ -1,4 +1,4 @@ --- source: milli/src/update/facet/bulk.rs --- -834f27a924de1acbd3cd94c0d7f10315 +5ce8009d3eb023e4b9c0a6e7fa4e6262 diff --git a/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap new file mode 100644 index 000000000..919f3fe7c --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +5dbfa134cc44abeb3ab6242fc182e48e diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap new file mode 100644 index 000000000..b006c11ab --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- + diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap new file mode 100644 index 000000000..bdeeefc13 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +6ed7bf5d440599b3b10b37549a271fdf diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap new file mode 100644 index 000000000..08534cbd4 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap @@ -0,0 +1,23 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +0 0 k0 1 "[0, ]" +0 0 k1 1 "[1, ]" +0 0 k2 1 "[2, ]" +0 0 k3 1 "[3, ]" +0 0 k4 1 "[4, ]" +0 0 k5 1 "[5, ]" +0 0 k6 1 "[6, ]" +0 0 k7 1 "[7, ]" +0 0 k8 1 "[8, ]" +0 0 k9 1 "[9, ]" +0 0 k10 1 "[10, ]" +0 0 k11 1 "[11, ]" +0 0 k12 1 "[12, ]" +0 0 k13 1 "[13, ]" +0 0 k14 1 "[14, ]" +0 1 k0 4 "[0, 1, 2, 3, ]" +0 1 k4 4 "[4, 5, 6, 7, ]" +0 1 k8 4 "[8, 9, 10, 11, ]" +0 1 k12 3 "[12, 13, 14, ]" + diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap new file mode 100644 index 000000000..e9ccc990f --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +b5203f0df0036ebaa133dd77d63a00eb diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap new file mode 100644 index 000000000..a98803604 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap @@ -0,0 +1,26 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +0 0 k0 1 "[0, ]" +0 0 k1 1 "[1, ]" +0 0 k2 1 "[2, ]" +0 0 k3 1 "[3, ]" +0 0 k4 1 "[4, ]" +0 0 k5 1 "[5, ]" +0 0 k6 1 "[6, ]" +0 0 k7 1 "[7, ]" +0 0 k8 1 "[8, ]" +0 0 k9 1 "[9, ]" +0 0 k10 1 "[10, ]" +0 0 k11 1 "[11, ]" +0 0 k12 1 "[12, ]" +0 0 k13 1 "[13, ]" +0 0 k14 1 "[14, ]" +0 0 k15 1 "[15, ]" +0 0 k16 1 "[16, ]" +0 1 k0 4 "[0, 1, 2, 3, ]" +0 1 k4 4 "[4, 5, 6, 7, ]" +0 1 k8 4 "[8, 9, 10, 11, ]" +0 1 k12 4 "[12, 13, 14, 15, ]" +0 1 k16 1 "[16, ]" + diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap new file mode 100644 index 000000000..bb07123a9 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +95497d8579740868ee0bfc655b0bf782 diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap new file mode 100644 index 000000000..8714af061 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +d565c2f7bbd9e13e12de40cfbbfba6bb diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap new file mode 100644 index 000000000..1bba99454 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap @@ -0,0 +1,54 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +0 0 k216 1 "[216, ]" +0 0 k217 1 "[217, ]" +0 0 k218 1 "[218, ]" +0 0 k219 1 "[219, ]" +0 0 k220 1 "[220, ]" +0 0 k221 1 "[221, ]" +0 0 k222 1 "[222, ]" +0 0 k223 1 "[223, ]" +0 0 k224 1 "[224, ]" +0 0 k225 1 "[225, ]" +0 0 k226 1 "[226, ]" +0 0 k227 1 "[227, ]" +0 0 k228 1 "[228, ]" +0 0 k229 1 "[229, ]" +0 0 k230 1 "[230, ]" +0 0 k231 1 "[231, ]" +0 0 k232 1 "[232, ]" +0 0 k233 1 "[233, ]" +0 0 k234 1 "[234, ]" +0 0 k235 1 "[235, ]" +0 0 k236 1 "[236, ]" +0 0 k237 1 "[237, ]" +0 0 k238 1 "[238, ]" +0 0 k239 1 "[239, ]" +0 0 k240 1 "[240, ]" +0 0 k241 1 "[241, ]" +0 0 k242 1 "[242, ]" +0 0 k243 1 "[243, ]" +0 0 k244 1 "[244, ]" +0 0 k245 1 "[245, ]" +0 0 k246 1 "[246, ]" +0 0 k247 1 "[247, ]" +0 0 k248 1 "[248, ]" +0 0 k249 1 "[249, ]" +0 0 k250 1 "[250, ]" +0 0 k251 1 "[251, ]" +0 0 k252 1 "[252, ]" +0 0 k253 1 "[253, ]" +0 0 k254 1 "[254, ]" +0 0 k255 1 "[255, ]" +0 1 k216 4 "[216, 217, 218, 219, ]" +0 1 k220 4 "[220, 221, 222, 223, ]" +0 1 k224 4 "[224, 225, 226, 227, ]" +0 1 k228 4 "[228, 229, 230, 231, ]" +0 1 k232 4 "[232, 233, 234, 235, ]" +0 1 k236 4 "[236, 237, 238, 239, ]" +0 1 k240 4 "[240, 241, 242, 243, ]" +0 1 k244 4 "[244, 245, 246, 247, ]" +0 1 k248 4 "[248, 249, 250, 251, ]" +0 1 k252 4 "[252, 253, 254, 255, ]" + diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap new file mode 100644 index 000000000..b006c11ab --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- + diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap new file mode 100644 index 000000000..6815ee609 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +7cb503827ba17e9670296cc9531a1380 diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap new file mode 100644 index 000000000..6860385ee --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +b061f43e379e16f0617c05d3313d0078 diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap new file mode 100644 index 000000000..b006c11ab --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- + diff --git a/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/in_place_level0_delete.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/in_place_level0_delete.hash.snap new file mode 100644 index 000000000..f96b42b27 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/in_place_level0_delete.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +81fc9489d6b163935b97433477dea63b diff --git a/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap b/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap new file mode 100644 index 000000000..82a7ce716 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap @@ -0,0 +1,20 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +0 0 k0 1 "[3, 435, 583, 849, ]" +0 0 k1 1 "[35, 494, 693, 796, ]" +0 0 k2 1 "[76, 420, 526, 909, ]" +0 0 k3 1 "[133, 451, 653, 806, ]" +0 0 k4 1 "[131, 464, 656, 853, ]" +0 0 k5 1 "[61, 308, 701, 903, ]" +0 0 k6 1 "[144, 449, 674, 794, ]" +0 0 k7 1 "[182, 451, 735, 941, ]" +0 0 k8 1 "[6, 359, 679, 1003, ]" +0 0 k9 1 "[197, 418, 659, 904, ]" +0 0 k10 1 "[88, 297, 567, 800, ]" +0 0 k11 1 "[150, 309, 530, 946, ]" +0 0 k12 1 "[156, 466, 567, 892, ]" +0 0 k13 1 "[46, 425, 610, 807, ]" +0 0 k14 1 "[236, 433, 549, 891, ]" +0 0 k15 1 "[207, 472, 603, 974, ]" + diff --git a/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap new file mode 100644 index 000000000..fd4beeca8 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +7f8aa18d2b3a6422d55c03bede0563db diff --git a/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap new file mode 100644 index 000000000..fd4beeca8 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +7f8aa18d2b3a6422d55c03bede0563db diff --git a/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap new file mode 100644 index 000000000..d055892f5 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +b3e2de9020d9e0f3941bc3a179c795ba diff --git a/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap new file mode 100644 index 000000000..919f3fe7c --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +5dbfa134cc44abeb3ab6242fc182e48e diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/1.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/1.hash.snap new file mode 100644 index 000000000..2b6805676 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/1.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +4fc800f49201a336295af0542fdf01ab diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/2.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/2.hash.snap new file mode 100644 index 000000000..1802eb952 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/2.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +9343355bf535ed4a0c956df2b229d5e6 diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap b/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap new file mode 100644 index 000000000..5ef88bfb4 --- /dev/null +++ b/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap @@ -0,0 +1,4 @@ +--- +source: milli/src/update/facet/incremental.rs +--- +fd65ce7d96a07aafb0ef6cfb5bf016b8