searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-11 15:38:55 +01:00
Code Issues Releases Wiki Activity

Add facet incremental indexing snapshot tests + fix bug

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-09-01 11:33:50 +02:00 committed by Loïc Lecrenier
parent 07ff92c663
commit 36296bbb20
49 changed files with 1028 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
milli/src/search/facet/mod.rs
View File

@ -77,7 +77,7 @@ pub(crate) fn get_highest_level<'t>(
}
#[cfg(test)]
mod test {
pub mod test {
use crate::update::FacetsUpdateIncremental;
use heed::{BytesDecode, BytesEncode, Env, RwTxn};
use roaring::RoaringBitmap;
@ -160,6 +160,17 @@ mod test {
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
update.insert(rwtxn, field_id, &key_bytes, docids).unwrap();
}
pub fn delete<'a>(
&self,
rwtxn: &'a mut RwTxn,
field_id: u16,
key: &'a <BoundCodec as BytesEncode<'a>>::EItem,
value: u32,
) {
let update = FacetsUpdateIncremental::new(self.db.content);
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
update.delete(rwtxn, field_id, &key_bytes, value).unwrap();
}
}
impl<BoundCodec> Display for FacetIndex<BoundCodec>

32
milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap
View File

@ -225,4 +225,36 @@ source: milli/src/search/facet/facet_distribution_iter.rs
221: 1
222: 1
223: 1
224: 1
225: 1
226: 1
227: 1
228: 1
229: 1
230: 1
231: 1
232: 1
233: 1
234: 1
235: 1
236: 1
237: 1
238: 1
239: 1
240: 1
241: 1
242: 1
243: 1
244: 1
245: 1
246: 1
247: 1
248: 1
249: 1
250: 1
251: 1
252: 1
253: 1
254: 1
255: 1

5
milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap
View File

@ -96,5 +96,10 @@ source: milli/src/search/facet/facet_distribution_iter.rs
216: 1
219: 1
220: 1
223: 1
226: 1
235: 1
236: 1
238: 1
243: 1

4
milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap
View File

@ -96,5 +96,9 @@ source: milli/src/search/facet/facet_distribution_iter.rs
216: 1
219: 1
220: 1
223: 1
226: 1
235: 1
236: 1
238: 1

2
milli/src/search/facet/snapshots/facet_distribution_iter.rs/random_looking_index_snap/random_looking_index_snap.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_distribution_iter.rs
---
ea4022977d09c7854c833146276348de
3256c76a7c1b768a013e78d5fa6e9ff9

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/0.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
52d0b31f312572c10959418434e36581
fcedc563a82c1c61f50174a5f3f982b6

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/1.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
2cb9e819529823d488e141edb4307f97
6cc26e77fc6bd9145deedf14cf422b03

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/0.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
38a4352c48905f5b121d1217734862da
c1c7a0bb91d53d33724583b6d4a99f16

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/1.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
aefc1ec120fa884cc8396a68bd7de42f
12213d3f1047a0c3d08e4670a7d688e7

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/0.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
9e360d7bcd29ac2c23bc241df941fd23
3456db9a1bb94c33c1e9f656184ee711

2
milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/1.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
f0606b9af67de9ede9d469514ea1741f
2127cd818b457e0611e0c8e1a871602a

2
milli/src/search/facet/snapshots/facet_range_search.rs/random_looking_index_snap/random_looking_index_snap.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_range_search.rs
---
ea4022977d09c7854c833146276348de
3256c76a7c1b768a013e78d5fa6e9ff9

32
milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/0.snap
View File

@ -25,4 +25,36 @@ source: milli/src/search/facet/facet_sort_ascending.rs
[221, ]
[222, ]
[223, ]
[224, ]
[225, ]
[226, ]
[227, ]
[228, ]
[229, ]
[230, ]
[231, ]
[232, ]
[233, ]
[234, ]
[235, ]
[236, ]
[237, ]
[238, ]
[239, ]
[240, ]
[241, ]
[242, ]
[243, ]
[244, ]
[245, ]
[246, ]
[247, ]
[248, ]
[249, ]
[250, ]
[251, ]
[252, ]
[253, ]
[254, ]
[255, ]

1
milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort/1.snap
View File

@ -50,4 +50,5 @@ source: milli/src/search/facet/facet_sort_ascending.rs
[216, ]
[220, ]
[226, ]
[238, ]

2
milli/src/search/facet/snapshots/facet_sort_ascending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_sort_ascending.rs
---
ea4022977d09c7854c833146276348de
3256c76a7c1b768a013e78d5fa6e9ff9

44
milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap
View File

@ -1,10 +1,54 @@
---
source: milli/src/search/facet/facet_sort_descending.rs
---
[255, ]
[254, ]
[253, ]
[252, ]
[251, ]
[250, ]
[249, ]
[248, ]
[247, ]
[246, ]
[245, ]
[244, ]
[243, ]
[242, ]
[241, ]
[240, ]
[239, ]
[238, ]
[237, ]
[236, ]
[235, ]
[234, ]
[233, ]
[232, ]
[231, ]
[230, ]
[229, ]
[228, ]
[227, ]
[226, ]
[225, ]
[224, ]
[223, ]
[222, ]
[221, ]
[220, ]
[219, ]
[218, ]
[217, ]
[216, ]
[215, ]
[214, ]
[213, ]
[212, ]
[211, ]
[210, ]
[209, ]
[208, ]
[207, ]
[206, ]
[205, ]

9
milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap
View File

@ -2,8 +2,15 @@
source: milli/src/search/facet/facet_sort_descending.rs
---
[243, ]
[238, ]
[236, ]
[235, ]
[226, ]
[223, ]
[220, ]
[219, ]
[216, ]
[210, ]
[209, ]
[208, ]
[207, ]
@ -35,12 +42,10 @@ source: milli/src/search/facet/facet_sort_descending.rs
[241, ]
[239, ]
[237, ]
[236, ]
[233, ]
[231, ]
[230, ]
[224, ]
[223, ]
[215, ]
[211, ]
[203, ]

2
milli/src/search/facet/snapshots/facet_sort_descending.rs/random_looking_index_snap/random_looking_index_snap.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/search/facet/facet_sort_descending.rs
---
ea4022977d09c7854c833146276348de
3256c76a7c1b768a013e78d5fa6e9ff9

679
milli/src/update/facet/incremental.rs
View File

@ -297,7 +297,7 @@ impl FacetsUpdateIncremental {
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &highest_level_prefix)?;
let mut to_add = vec![];
for _ in 0..group_size {
for _ in 0..self.min_level_size {
let mut first_key = None;
let mut values = RoaringBitmap::new();
for _ in 0..group_size {
@ -459,3 +459,680 @@ impl FacetsUpdateIncremental {
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::milli_snap;
use crate::{
heed_codec::facet::new::{
ordered_f64_codec::OrderedF64Codec, str_ref::StrRefCodec, FacetGroupValueCodec,
FacetKeyCodec, MyByteSlice,
},
search::facet::{get_highest_level, test::FacetIndex},
};
use heed::{types::ByteSlice, BytesDecode, BytesEncode};
use rand::Rng;
use rand::{seq::SliceRandom, SeedableRng};
use roaring::RoaringBitmap;
pub fn verify_structure_validity<C>(index: &FacetIndex<C>, field_id: u16)
where
for<'a> C: BytesDecode<'a> + BytesEncode<'a, EItem = <C as BytesDecode<'a>>::DItem>,
{
let FacetIndex { env, db, .. } = index;
let txn = env.write_txn().unwrap();
let mut field_id_prefix = vec![];
field_id_prefix.extend_from_slice(&field_id.to_be_bytes());
let highest_level = get_highest_level(&txn, index.db.content, field_id).unwrap();
txn.commit().unwrap();
let txn = env.read_txn().unwrap();
for level_no in (1..=highest_level).rev() {
let mut level_no_prefix = vec![];
level_no_prefix.extend_from_slice(&field_id.to_be_bytes());
level_no_prefix.push(level_no);
let mut iter = db
.content
.as_polymorph()
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &level_no_prefix)
.unwrap();
while let Some(el) = iter.next() {
let (key, value) = el.unwrap();
let key = FacetKeyCodec::<MyByteSlice>::bytes_decode(&key).unwrap();
let mut prefix_start_below = vec![];
prefix_start_below.extend_from_slice(&field_id.to_be_bytes());
prefix_start_below.push(level_no - 1);
prefix_start_below.extend_from_slice(&key.left_bound);
let start_below = {
let mut start_below_iter = db
.content
.as_polymorph()
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(
&txn,
&prefix_start_below,
)
.unwrap();
let (key_bytes, _) = start_below_iter.next().unwrap().unwrap();
FacetKeyCodec::<MyByteSlice>::bytes_decode(&key_bytes).unwrap()
};
assert!(value.size > 0 && (value.size as usize) < db.max_group_size);
let mut actual_size = 0;
let mut values_below = RoaringBitmap::new();
let mut iter_below =
db.content.range(&txn, &(start_below..)).unwrap().take(value.size as usize);
while let Some(el) = iter_below.next() {
let (_, value) = el.unwrap();
actual_size += 1;
values_below |= value.bitmap;
}
assert_eq!(actual_size, value.size, "{key:?} start_below: {start_below:?}");
assert_eq!(value.bitmap, values_below);
}
}
}
#[test]
fn append() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn many_field_ids_append() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 2, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 1, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
verify_structure_validity(&index, 1);
verify_structure_validity(&index, 2);
milli_snap!(format!("{index}"));
}
#[test]
fn many_field_ids_prepend() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 2, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 1, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
verify_structure_validity(&index, 1);
verify_structure_validity(&index, 2);
milli_snap!(format!("{index}"));
}
#[test]
fn prepend() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap();
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
index.insert(&mut txn, 0, &(i as f64), &bitmap);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn shuffled() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap();
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
index.insert(&mut txn, 0, &(key as f64), &bitmap);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn merge_values() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(rng.gen_range(256..512));
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn delete_from_end() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(&(i as f64)), &bitmap);
txn.commit().unwrap();
}
for i in (200..256).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 200);
for i in (150..200).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 150);
for i in (100..150).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 100);
for i in (17..100).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 17);
let mut txn = index.env.write_txn().unwrap();
for i in (15..17).into_iter().rev() {
index.delete(&mut txn, 0, &(i as f64), i as u32);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 15);
for i in (0..15).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 0);
}
#[test]
fn delete_from_start() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..128 {
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 127);
for i in 128..216 {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 215);
for i in 216..256 {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 255);
}
#[test]
fn delete_shuffled() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for i in 0..128 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 127);
for i in 128..216 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 215);
for i in 216..256 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 255);
}
#[test]
fn in_place_level0_insert() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..16).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for i in 0..4 {
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(rng.gen_range(i * 256..(i + 1) * 256));
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn in_place_level0_delete() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..64).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(key + 100);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
for &key in keys.iter() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key + 100);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn shuffle_merge_string() {
let index = FacetIndex::<StrRefCodec>::new(4, 8);
let mut keys = (1000..1064).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(key + 100);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &format!("{key:x}").as_str(), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 1);
for &key in keys.iter() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &format!("{key:x}").as_str(), key + 100);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 2);
}
// fuzz tests
}
// #[cfg(all(test, fuzzing))]
// mod fuzz {
// use crate::codec::U16Codec;
// use super::tests::verify_structure_validity;
// use super::*;
// use fuzzcheck::mutators::integer_within_range::U16WithinRangeMutator;
// use fuzzcheck::DefaultMutator;
// use roaring::RoaringBitmap;
// use std::collections::BTreeMap;
// use std::collections::HashMap;
// #[derive(Default)]
// pub struct TrivialDatabase<T> {
// pub elements: BTreeMap<u16, BTreeMap<T, RoaringBitmap>>,
// }
// impl<T> TrivialDatabase<T>
// where
// T: Ord + Clone + Copy + Eq + std::fmt::Debug,
// {
// pub fn insert(&mut self, field_id: u16, new_key: T, new_values: &RoaringBitmap) {
// if new_values.is_empty() {
// return;
// }
// let values_field_id = self.elements.entry(field_id).or_default();
// let values = values_field_id.entry(new_key).or_default();
// *values |= new_values;
// }
// pub fn delete(&mut self, field_id: u16, key: T, value: u32) {
// if let Some(values_field_id) = self.elements.get_mut(&field_id) {
// if let Some(values) = values_field_id.get_mut(&key) {
// values.remove(value);
// if values.is_empty() {
// values_field_id.remove(&key);
// }
// }
// if values_field_id.is_empty() {
// self.elements.remove(&field_id);
// }
// }
// }
// }
// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)]
// struct Operation<Key> {
// key: Key,
// #[field_mutator(U16WithinRangeMutator = { U16WithinRangeMutator::new(..=3) })]
// field_id: u16,
// kind: OperationKind,
// }
// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)]
// enum OperationKind {
// Insert(Vec<u8>),
// Delete(u8),
// }
// fn compare_with_trivial_database(
// tempdir: Rc<TempDir>,
// group_size: u8,
// max_group_size: u8,
// operations: &[Operation<u16>],
// ) {
// let index = FacetIndex::<OrderedF64Codec>::open_from_tempdir(tempdir, group_size, max_group_size);
// let mut trivial_db = TrivialDatabase::<u16>::default();
// let mut value_to_keys = HashMap::<u8, Vec<u16>>::new();
// let mut txn = index.env.write_txn().unwrap();
// for Operation { key, field_id, kind } in operations {
// match kind {
// OperationKind::Insert(values) => {
// let mut bitmap = RoaringBitmap::new();
// for value in values {
// bitmap.insert(*value as u32);
// value_to_keys.entry(*value).or_default().push(*key);
// }
// index.insert(&mut txn, *field_id, key, &bitmap);
// trivial_db.insert(*field_id, *key, &bitmap);
// }
// OperationKind::Delete(value) => {
// if let Some(keys) = value_to_keys.get(value) {
// for key in keys {
// index.delete(&mut txn, *field_id, key, *value as u32);
// trivial_db.delete(*field_id, *key, *value as u32);
// }
// }
// }
// }
// }
// for (field_id, values_field_id) in trivial_db.elements.iter() {
// let level0iter = index
// .db
// .content
// .as_polymorph()
// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(
// &mut txn,
// &field_id.to_be_bytes(),
// )
// .unwrap();
// for ((key, values), group) in values_field_id.iter().zip(level0iter) {
// let (group_key, group_values) = group.unwrap();
// let group_key = FacetKeyCodec::<U16Codec>::bytes_decode(group_key).unwrap();
// assert_eq!(key, &group_key.left_bound);
// assert_eq!(values, &group_values.bitmap);
// }
// }
// txn.commit().unwrap();
// let mut txn = index.env.write_txn().unwrap();
// for (field_id, values_field_id) in trivial_db.elements.iter() {
// let level0iter = index
// .db
// .content
// .as_polymorph()
// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &field_id.to_be_bytes())
// .unwrap();
// for ((key, values), group) in values_field_id.iter().zip(level0iter) {
// let (group_key, group_values) = group.unwrap();
// let group_key = FacetKeyCodec::<U16Codec>::bytes_decode(group_key).unwrap();
// assert_eq!(key, &group_key.left_bound);
// assert_eq!(values, &group_values.bitmap);
// }
// verify_structure_validity(&index, *field_id);
// }
// index.db.content.clear(&mut txn).unwrap();
// txn.commit().unwrap();
// }
// #[test]
// fn fuzz() {
// let tempdir = Rc::new(TempDir::new().unwrap());
// let tempdir_cloned = tempdir.clone();
// let result = fuzzcheck::fuzz_test(move |x: &(u8, u8, Vec<Operation<u16>>)| {
// compare_with_trivial_database(tempdir_cloned.clone(), x.0, x.1, &x.2)
// })
// .default_mutator()
// .serde_serializer()
// .default_sensor_and_pool_with_custom_filter(|file, function| {
// if file.is_relative()
// && !function.contains("serde")
// && !function.contains("tests::")
// && !function.contains("fuzz::")
// && !function.contains("display_bitmap")
// {
// true
// } else {
// false
// }
// })
// .arguments_from_cargo_fuzzcheck()
// .launch();
// assert!(!result.found_test_failure);
// }
// #[test]
// fn reproduce_bug() {
// let operations = r#"
// [
// {"key":0, "field_id": 0, "kind":{"Insert":[109]}},
// {"key":143, "field_id": 0, "kind":{"Insert":[243]}},
// {"key":90, "field_id": 0, "kind":{"Insert":[217]}},
// {"key":172, "field_id": 0, "kind":{"Insert":[94]}},
// {"key":27, "field_id": 0, "kind":{"Insert":[4]}},
// {"key":124, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":123, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":67, "field_id": 0, "kind":{"Insert":[109]}},
// {"key":13, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":162, "field_id": 0, "kind":{"Insert":[213]}},
// {"key":235, "field_id": 0, "kind":{"Insert":[67]}},
// {"key":251, "field_id": 0, "kind":{"Insert":[50]}},
// {"key":218, "field_id": 0, "kind":{"Insert":[164]}},
// {"key":166, "field_id": 0, "kind":{"Insert":[67]}},
// {"key":64, "field_id": 0, "kind":{"Insert":[61]}},
// {"key":183, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":250, "field_id": 0, "kind":{"Delete":50}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations);
// }
// #[test]
// fn reproduce_bug2() {
// let operations = r#"
// [
// {"key":102, "field_id": 0, "kind":{"Insert":[122]}},
// {"key":73, "field_id": 0, "kind":{"Insert":[132]}},
// {"key":20, "field_id": 0, "kind":{"Insert":[215]}},
// {"key":39, "field_id": 0, "kind":{"Insert":[152]}},
// {"key":151, "field_id": 0, "kind":{"Insert":[226]}},
// {"key":17, "field_id": 0, "kind":{"Insert":[101]}},
// {"key":74, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":2, "field_id": 0, "kind":{"Insert":[130]}},
// {"key":64, "field_id": 0, "kind":{"Insert":[180]}},
// {"key":83, "field_id": 0, "kind":{"Insert":[250]}},
// {"key":80, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":113, "field_id": 0, "kind":{"Insert":[63]}},
// {"key":201, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":200, "field_id": 0, "kind":{"Insert":[5]}},
// {"key":93, "field_id": 0, "kind":{"Insert":[98]}},
// {"key":162, "field_id": 0, "kind":{"Insert":[5]}},
// {"key":80, "field_id": 0, "kind":{"Delete":210}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations);
// }
// #[test]
// fn reproduce_bug3() {
// let operations = r#"
// [
// {"key":27488, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":64716, "field_id": 0, "kind":{"Insert":[216]}},
// {"key":60886, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":59509, "field_id": 0, "kind":{"Insert":[187,231]}},
// {"key":55057, "field_id": 0, "kind":{"Insert":[37]}},
// {"key":45200, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":55056, "field_id": 0, "kind":{"Insert":[37]}},
// {"key":63679, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":52155, "field_id": 0, "kind":{"Insert":[74]}},
// {"key":20648, "field_id": 0, "kind":{"Insert":[47,138,157]}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 0, 7, &operations);
// }
// #[test]
// fn reproduce_bug4() {
// let operations = r#"
// [{"key":63499, "field_id": 0, "kind":{"Insert":[87]}},{"key":25374, "field_id": 0, "kind":{"Insert":[14]}},{"key":64481, "field_id": 0, "kind":{"Delete":87}},{"key":23038, "field_id": 0, "kind":{"Insert":[173]}},{"key":14862, "field_id": 0, "kind":{"Insert":[8]}},{"key":13145, "field_id": 0, "kind":{"Insert":[5,64]}},{"key":23446, "field_id": 0, "kind":{"Insert":[86,59]}},{"key":17972, "field_id": 0, "kind":{"Insert":[58,137]}},{"key":21273, "field_id": 0, "kind":{"Insert":[121,132,81,147]}},{"key":28264, "field_id": 0, "kind":{"Insert":[36]}},{"key":46659, "field_id": 0, "kind":{"Insert":[]}}]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 2, 1, &operations);
// }
// }

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/default/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_large_levels/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/large_groups_small_levels/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_large_levels/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/small_groups_small_levels/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_number/tiny_groups_tiny_levels/facet_id_f64_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
8bc439472ccda008dc5c28aa789f433d
947949d1a5c9c4e895c89fba46cbba68

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_string/default/facet_id_string_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
834f27a924de1acbd3cd94c0d7f10315
5ce8009d3eb023e4b9c0a6e7fa4e6262

2
milli/src/update/facet/snapshots/bulk.rs/test_facets_string/tiny_groups_tiny_levels/facet_id_string_docids.hash.snap
View File

@ -1,4 +1,4 @@
---
source: milli/src/update/facet/bulk.rs
---
834f27a924de1acbd3cd94c0d7f10315
5ce8009d3eb023e4b9c0a6e7fa4e6262

4
milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
5dbfa134cc44abeb3ab6242fc182e48e

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
6ed7bf5d440599b3b10b37549a271fdf

23
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap Normal file
View File

@ -0,0 +1,23 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[0, ]"
0 0 k1 1 "[1, ]"
0 0 k2 1 "[2, ]"
0 0 k3 1 "[3, ]"
0 0 k4 1 "[4, ]"
0 0 k5 1 "[5, ]"
0 0 k6 1 "[6, ]"
0 0 k7 1 "[7, ]"
0 0 k8 1 "[8, ]"
0 0 k9 1 "[9, ]"
0 0 k10 1 "[10, ]"
0 0 k11 1 "[11, ]"
0 0 k12 1 "[12, ]"
0 0 k13 1 "[13, ]"
0 0 k14 1 "[14, ]"
0 1 k0 4 "[0, 1, 2, 3, ]"
0 1 k4 4 "[4, 5, 6, 7, ]"
0 1 k8 4 "[8, 9, 10, 11, ]"
0 1 k12 3 "[12, 13, 14, ]"

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b5203f0df0036ebaa133dd77d63a00eb

26
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap Normal file
View File

@ -0,0 +1,26 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[0, ]"
0 0 k1 1 "[1, ]"
0 0 k2 1 "[2, ]"
0 0 k3 1 "[3, ]"
0 0 k4 1 "[4, ]"
0 0 k5 1 "[5, ]"
0 0 k6 1 "[6, ]"
0 0 k7 1 "[7, ]"
0 0 k8 1 "[8, ]"
0 0 k9 1 "[9, ]"
0 0 k10 1 "[10, ]"
0 0 k11 1 "[11, ]"
0 0 k12 1 "[12, ]"
0 0 k13 1 "[13, ]"
0 0 k14 1 "[14, ]"
0 0 k15 1 "[15, ]"
0 0 k16 1 "[16, ]"
0 1 k0 4 "[0, 1, 2, 3, ]"
0 1 k4 4 "[4, 5, 6, 7, ]"
0 1 k8 4 "[8, 9, 10, 11, ]"
0 1 k12 4 "[12, 13, 14, 15, ]"
0 1 k16 1 "[16, ]"

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
95497d8579740868ee0bfc655b0bf782

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
d565c2f7bbd9e13e12de40cfbbfba6bb

54
milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap Normal file
View File

@ -0,0 +1,54 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k216 1 "[216, ]"
0 0 k217 1 "[217, ]"
0 0 k218 1 "[218, ]"
0 0 k219 1 "[219, ]"
0 0 k220 1 "[220, ]"
0 0 k221 1 "[221, ]"
0 0 k222 1 "[222, ]"
0 0 k223 1 "[223, ]"
0 0 k224 1 "[224, ]"
0 0 k225 1 "[225, ]"
0 0 k226 1 "[226, ]"
0 0 k227 1 "[227, ]"
0 0 k228 1 "[228, ]"
0 0 k229 1 "[229, ]"
0 0 k230 1 "[230, ]"
0 0 k231 1 "[231, ]"
0 0 k232 1 "[232, ]"
0 0 k233 1 "[233, ]"
0 0 k234 1 "[234, ]"
0 0 k235 1 "[235, ]"
0 0 k236 1 "[236, ]"
0 0 k237 1 "[237, ]"
0 0 k238 1 "[238, ]"
0 0 k239 1 "[239, ]"
0 0 k240 1 "[240, ]"
0 0 k241 1 "[241, ]"
0 0 k242 1 "[242, ]"
0 0 k243 1 "[243, ]"
0 0 k244 1 "[244, ]"
0 0 k245 1 "[245, ]"
0 0 k246 1 "[246, ]"
0 0 k247 1 "[247, ]"
0 0 k248 1 "[248, ]"
0 0 k249 1 "[249, ]"
0 0 k250 1 "[250, ]"
0 0 k251 1 "[251, ]"
0 0 k252 1 "[252, ]"
0 0 k253 1 "[253, ]"
0 0 k254 1 "[254, ]"
0 0 k255 1 "[255, ]"
0 1 k216 4 "[216, 217, 218, 219, ]"
0 1 k220 4 "[220, 221, 222, 223, ]"
0 1 k224 4 "[224, 225, 226, 227, ]"
0 1 k228 4 "[228, 229, 230, 231, ]"
0 1 k232 4 "[232, 233, 234, 235, ]"
0 1 k236 4 "[236, 237, 238, 239, ]"
0 1 k240 4 "[240, 241, 242, 243, ]"
0 1 k244 4 "[244, 245, 246, 247, ]"
0 1 k248 4 "[248, 249, 250, 251, ]"
0 1 k252 4 "[252, 253, 254, 255, ]"

4
milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

4
milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7cb503827ba17e9670296cc9531a1380

4
milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b061f43e379e16f0617c05d3313d0078

4
milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

4
milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/in_place_level0_delete.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
81fc9489d6b163935b97433477dea63b

20
milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap Normal file
View File

@ -0,0 +1,20 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[3, 435, 583, 849, ]"
0 0 k1 1 "[35, 494, 693, 796, ]"
0 0 k2 1 "[76, 420, 526, 909, ]"
0 0 k3 1 "[133, 451, 653, 806, ]"
0 0 k4 1 "[131, 464, 656, 853, ]"
0 0 k5 1 "[61, 308, 701, 903, ]"
0 0 k6 1 "[144, 449, 674, 794, ]"
0 0 k7 1 "[182, 451, 735, 941, ]"
0 0 k8 1 "[6, 359, 679, 1003, ]"
0 0 k9 1 "[197, 418, 659, 904, ]"
0 0 k10 1 "[88, 297, 567, 800, ]"
0 0 k11 1 "[150, 309, 530, 946, ]"
0 0 k12 1 "[156, 466, 567, 892, ]"
0 0 k13 1 "[46, 425, 610, 807, ]"
0 0 k14 1 "[236, 433, 549, 891, ]"
0 0 k15 1 "[207, 472, 603, 974, ]"

4
milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7f8aa18d2b3a6422d55c03bede0563db

4
milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7f8aa18d2b3a6422d55c03bede0563db

4
milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b3e2de9020d9e0f3941bc3a179c795ba

4
milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
5dbfa134cc44abeb3ab6242fc182e48e

4
milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/1.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
4fc800f49201a336295af0542fdf01ab

4
milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string/2.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
9343355bf535ed4a0c956df2b229d5e6

4
milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap Normal file
View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
fd65ce7d96a07aafb0ef6cfb5bf016b8