Add facet incremental indexing snapshot tests + fix bug

This commit is contained in:
Loïc Lecrenier 2022-09-01 11:33:50 +02:00 committed by Loïc Lecrenier
parent 07ff92c663
commit 36296bbb20
49 changed files with 1028 additions and 22 deletions

View File

@ -77,7 +77,7 @@ pub(crate) fn get_highest_level<'t>(
} }
#[cfg(test)] #[cfg(test)]
mod test { pub mod test {
use crate::update::FacetsUpdateIncremental; use crate::update::FacetsUpdateIncremental;
use heed::{BytesDecode, BytesEncode, Env, RwTxn}; use heed::{BytesDecode, BytesEncode, Env, RwTxn};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -160,6 +160,17 @@ mod test {
let key_bytes = BoundCodec::bytes_encode(&key).unwrap(); let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
update.insert(rwtxn, field_id, &key_bytes, docids).unwrap(); update.insert(rwtxn, field_id, &key_bytes, docids).unwrap();
} }
pub fn delete<'a>(
&self,
rwtxn: &'a mut RwTxn,
field_id: u16,
key: &'a <BoundCodec as BytesEncode<'a>>::EItem,
value: u32,
) {
let update = FacetsUpdateIncremental::new(self.db.content);
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
update.delete(rwtxn, field_id, &key_bytes, value).unwrap();
}
} }
impl<BoundCodec> Display for FacetIndex<BoundCodec> impl<BoundCodec> Display for FacetIndex<BoundCodec>

View File

@ -225,4 +225,36 @@ source: milli/src/search/facet/facet_distribution_iter.rs
221: 1 221: 1
222: 1 222: 1
223: 1 223: 1
224: 1
225: 1
226: 1
227: 1
228: 1
229: 1
230: 1
231: 1
232: 1
233: 1
234: 1
235: 1
236: 1
237: 1
238: 1
239: 1
240: 1
241: 1
242: 1
243: 1
244: 1
245: 1
246: 1
247: 1
248: 1
249: 1
250: 1
251: 1
252: 1
253: 1
254: 1
255: 1

View File

@ -96,5 +96,10 @@ source: milli/src/search/facet/facet_distribution_iter.rs
216: 1 216: 1
219: 1 219: 1
220: 1 220: 1
223: 1
226: 1 226: 1
235: 1
236: 1
238: 1
243: 1

View File

@ -96,5 +96,9 @@ source: milli/src/search/facet/facet_distribution_iter.rs
216: 1 216: 1
219: 1 219: 1
220: 1 220: 1
223: 1
226: 1 226: 1
235: 1
236: 1
238: 1

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_distribution_iter.rs source: milli/src/search/facet/facet_distribution_iter.rs
--- ---
ea4022977d09c7854c833146276348de 3256c76a7c1b768a013e78d5fa6e9ff9

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
52d0b31f312572c10959418434e36581 fcedc563a82c1c61f50174a5f3f982b6

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
2cb9e819529823d488e141edb4307f97 6cc26e77fc6bd9145deedf14cf422b03

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
38a4352c48905f5b121d1217734862da c1c7a0bb91d53d33724583b6d4a99f16

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
aefc1ec120fa884cc8396a68bd7de42f 12213d3f1047a0c3d08e4670a7d688e7

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
9e360d7bcd29ac2c23bc241df941fd23 3456db9a1bb94c33c1e9f656184ee711

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
f0606b9af67de9ede9d469514ea1741f 2127cd818b457e0611e0c8e1a871602a

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_range_search.rs source: milli/src/search/facet/facet_range_search.rs
--- ---
ea4022977d09c7854c833146276348de 3256c76a7c1b768a013e78d5fa6e9ff9

View File

@ -25,4 +25,36 @@ source: milli/src/search/facet/facet_sort_ascending.rs
[221, ] [221, ]
[222, ] [222, ]
[223, ] [223, ]
[224, ]
[225, ]
[226, ]
[227, ]
[228, ]
[229, ]
[230, ]
[231, ]
[232, ]
[233, ]
[234, ]
[235, ]
[236, ]
[237, ]
[238, ]
[239, ]
[240, ]
[241, ]
[242, ]
[243, ]
[244, ]
[245, ]
[246, ]
[247, ]
[248, ]
[249, ]
[250, ]
[251, ]
[252, ]
[253, ]
[254, ]
[255, ]

View File

@ -50,4 +50,5 @@ source: milli/src/search/facet/facet_sort_ascending.rs
[216, ] [216, ]
[220, ] [220, ]
[226, ] [226, ]
[238, ]

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_sort_ascending.rs source: milli/src/search/facet/facet_sort_ascending.rs
--- ---
ea4022977d09c7854c833146276348de 3256c76a7c1b768a013e78d5fa6e9ff9

View File

@ -1,10 +1,54 @@
--- ---
source: milli/src/search/facet/facet_sort_descending.rs source: milli/src/search/facet/facet_sort_descending.rs
--- ---
[255, ]
[254, ]
[253, ]
[252, ]
[251, ]
[250, ]
[249, ]
[248, ]
[247, ] [247, ]
[246, ] [246, ]
[245, ] [245, ]
[244, ] [244, ]
[243, ]
[242, ]
[241, ]
[240, ]
[239, ]
[238, ]
[237, ]
[236, ]
[235, ]
[234, ]
[233, ]
[232, ]
[231, ]
[230, ]
[229, ]
[228, ]
[227, ]
[226, ]
[225, ]
[224, ]
[223, ]
[222, ]
[221, ]
[220, ]
[219, ]
[218, ]
[217, ]
[216, ]
[215, ]
[214, ]
[213, ]
[212, ]
[211, ]
[210, ]
[209, ]
[208, ]
[207, ] [207, ]
[206, ] [206, ]
[205, ] [205, ]

View File

@ -2,8 +2,15 @@
source: milli/src/search/facet/facet_sort_descending.rs source: milli/src/search/facet/facet_sort_descending.rs
--- ---
[243, ] [243, ]
[238, ]
[236, ]
[235, ] [235, ]
[226, ] [226, ]
[223, ]
[220, ]
[219, ]
[216, ]
[210, ]
[209, ] [209, ]
[208, ] [208, ]
[207, ] [207, ]
@ -35,12 +42,10 @@ source: milli/src/search/facet/facet_sort_descending.rs
[241, ] [241, ]
[239, ] [239, ]
[237, ] [237, ]
[236, ]
[233, ] [233, ]
[231, ] [231, ]
[230, ] [230, ]
[224, ] [224, ]
[223, ]
[215, ] [215, ]
[211, ] [211, ]
[203, ] [203, ]

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/search/facet/facet_sort_descending.rs source: milli/src/search/facet/facet_sort_descending.rs
--- ---
ea4022977d09c7854c833146276348de 3256c76a7c1b768a013e78d5fa6e9ff9

View File

@ -297,7 +297,7 @@ impl FacetsUpdateIncremental {
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &highest_level_prefix)?; .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &highest_level_prefix)?;
let mut to_add = vec![]; let mut to_add = vec![];
for _ in 0..group_size { for _ in 0..self.min_level_size {
let mut first_key = None; let mut first_key = None;
let mut values = RoaringBitmap::new(); let mut values = RoaringBitmap::new();
for _ in 0..group_size { for _ in 0..group_size {
@ -459,3 +459,680 @@ impl FacetsUpdateIncremental {
Ok(()) Ok(())
} }
} }
#[cfg(test)]
mod tests {
use crate::milli_snap;
use crate::{
heed_codec::facet::new::{
ordered_f64_codec::OrderedF64Codec, str_ref::StrRefCodec, FacetGroupValueCodec,
FacetKeyCodec, MyByteSlice,
},
search::facet::{get_highest_level, test::FacetIndex},
};
use heed::{types::ByteSlice, BytesDecode, BytesEncode};
use rand::Rng;
use rand::{seq::SliceRandom, SeedableRng};
use roaring::RoaringBitmap;
pub fn verify_structure_validity<C>(index: &FacetIndex<C>, field_id: u16)
where
for<'a> C: BytesDecode<'a> + BytesEncode<'a, EItem = <C as BytesDecode<'a>>::DItem>,
{
let FacetIndex { env, db, .. } = index;
let txn = env.write_txn().unwrap();
let mut field_id_prefix = vec![];
field_id_prefix.extend_from_slice(&field_id.to_be_bytes());
let highest_level = get_highest_level(&txn, index.db.content, field_id).unwrap();
txn.commit().unwrap();
let txn = env.read_txn().unwrap();
for level_no in (1..=highest_level).rev() {
let mut level_no_prefix = vec![];
level_no_prefix.extend_from_slice(&field_id.to_be_bytes());
level_no_prefix.push(level_no);
let mut iter = db
.content
.as_polymorph()
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &level_no_prefix)
.unwrap();
while let Some(el) = iter.next() {
let (key, value) = el.unwrap();
let key = FacetKeyCodec::<MyByteSlice>::bytes_decode(&key).unwrap();
let mut prefix_start_below = vec![];
prefix_start_below.extend_from_slice(&field_id.to_be_bytes());
prefix_start_below.push(level_no - 1);
prefix_start_below.extend_from_slice(&key.left_bound);
let start_below = {
let mut start_below_iter = db
.content
.as_polymorph()
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(
&txn,
&prefix_start_below,
)
.unwrap();
let (key_bytes, _) = start_below_iter.next().unwrap().unwrap();
FacetKeyCodec::<MyByteSlice>::bytes_decode(&key_bytes).unwrap()
};
assert!(value.size > 0 && (value.size as usize) < db.max_group_size);
let mut actual_size = 0;
let mut values_below = RoaringBitmap::new();
let mut iter_below =
db.content.range(&txn, &(start_below..)).unwrap().take(value.size as usize);
while let Some(el) = iter_below.next() {
let (_, value) = el.unwrap();
actual_size += 1;
values_below |= value.bitmap;
}
assert_eq!(actual_size, value.size, "{key:?} start_below: {start_below:?}");
assert_eq!(value.bitmap, values_below);
}
}
}
#[test]
fn append() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn many_field_ids_append() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 2, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 1, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
verify_structure_validity(&index, 1);
verify_structure_validity(&index, 2);
milli_snap!(format!("{index}"));
}
#[test]
fn many_field_ids_prepend() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 2, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 1, &(i as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
verify_structure_validity(&index, 1);
verify_structure_validity(&index, 2);
milli_snap!(format!("{index}"));
}
#[test]
fn prepend() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap();
for i in (0..256).into_iter().rev() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
index.insert(&mut txn, 0, &(i as f64), &bitmap);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn shuffled() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap();
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
index.insert(&mut txn, 0, &(key as f64), &bitmap);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn merge_values() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(rng.gen_range(256..512));
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn delete_from_end() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(&(i as f64)), &bitmap);
txn.commit().unwrap();
}
for i in (200..256).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 200);
for i in (150..200).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 150);
for i in (100..150).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 100);
for i in (17..100).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 17);
let mut txn = index.env.write_txn().unwrap();
for i in (15..17).into_iter().rev() {
index.delete(&mut txn, 0, &(i as f64), i as u32);
}
txn.commit().unwrap();
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 15);
for i in (0..15).into_iter().rev() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 0);
}
#[test]
fn delete_from_start() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
for i in 0..128 {
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 127);
for i in 128..216 {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 215);
for i in 216..256 {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(i as f64), i as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 255);
}
#[test]
fn delete_shuffled() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
for i in 0..256 {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(i);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(i as f64), &bitmap);
txn.commit().unwrap();
}
let mut keys = (0..256).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for i in 0..128 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 127);
for i in 128..216 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 215);
for i in 216..256 {
let key = keys[i];
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key as u32);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 255);
}
#[test]
fn in_place_level0_insert() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..16).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for i in 0..4 {
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(rng.gen_range(i * 256..(i + 1) * 256));
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn in_place_level0_delete() {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut keys = (0..64).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(key + 100);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &(key as f64), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
for &key in keys.iter() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &(key as f64), key + 100);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"));
}
#[test]
fn shuffle_merge_string() {
let index = FacetIndex::<StrRefCodec>::new(4, 8);
let mut keys = (1000..1064).into_iter().collect::<Vec<_>>();
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
keys.shuffle(&mut rng);
for &key in keys.iter() {
let mut bitmap = RoaringBitmap::new();
bitmap.insert(key);
bitmap.insert(key + 100);
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.insert(&mut txn, 0, &format!("{key:x}").as_str(), &bitmap);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 1);
for &key in keys.iter() {
verify_structure_validity(&index, 0);
let mut txn = index.env.write_txn().unwrap();
index.delete(&mut txn, 0, &format!("{key:x}").as_str(), key + 100);
txn.commit().unwrap();
}
verify_structure_validity(&index, 0);
milli_snap!(format!("{index}"), 2);
}
// fuzz tests
}
// #[cfg(all(test, fuzzing))]
// mod fuzz {
// use crate::codec::U16Codec;
// use super::tests::verify_structure_validity;
// use super::*;
// use fuzzcheck::mutators::integer_within_range::U16WithinRangeMutator;
// use fuzzcheck::DefaultMutator;
// use roaring::RoaringBitmap;
// use std::collections::BTreeMap;
// use std::collections::HashMap;
// #[derive(Default)]
// pub struct TrivialDatabase<T> {
// pub elements: BTreeMap<u16, BTreeMap<T, RoaringBitmap>>,
// }
// impl<T> TrivialDatabase<T>
// where
// T: Ord + Clone + Copy + Eq + std::fmt::Debug,
// {
// pub fn insert(&mut self, field_id: u16, new_key: T, new_values: &RoaringBitmap) {
// if new_values.is_empty() {
// return;
// }
// let values_field_id = self.elements.entry(field_id).or_default();
// let values = values_field_id.entry(new_key).or_default();
// *values |= new_values;
// }
// pub fn delete(&mut self, field_id: u16, key: T, value: u32) {
// if let Some(values_field_id) = self.elements.get_mut(&field_id) {
// if let Some(values) = values_field_id.get_mut(&key) {
// values.remove(value);
// if values.is_empty() {
// values_field_id.remove(&key);
// }
// }
// if values_field_id.is_empty() {
// self.elements.remove(&field_id);
// }
// }
// }
// }
// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)]
// struct Operation<Key> {
// key: Key,
// #[field_mutator(U16WithinRangeMutator = { U16WithinRangeMutator::new(..=3) })]
// field_id: u16,
// kind: OperationKind,
// }
// #[derive(Clone, DefaultMutator, serde::Serialize, serde::Deserialize)]
// enum OperationKind {
// Insert(Vec<u8>),
// Delete(u8),
// }
// fn compare_with_trivial_database(
// tempdir: Rc<TempDir>,
// group_size: u8,
// max_group_size: u8,
// operations: &[Operation<u16>],
// ) {
// let index = FacetIndex::<OrderedF64Codec>::open_from_tempdir(tempdir, group_size, max_group_size);
// let mut trivial_db = TrivialDatabase::<u16>::default();
// let mut value_to_keys = HashMap::<u8, Vec<u16>>::new();
// let mut txn = index.env.write_txn().unwrap();
// for Operation { key, field_id, kind } in operations {
// match kind {
// OperationKind::Insert(values) => {
// let mut bitmap = RoaringBitmap::new();
// for value in values {
// bitmap.insert(*value as u32);
// value_to_keys.entry(*value).or_default().push(*key);
// }
// index.insert(&mut txn, *field_id, key, &bitmap);
// trivial_db.insert(*field_id, *key, &bitmap);
// }
// OperationKind::Delete(value) => {
// if let Some(keys) = value_to_keys.get(value) {
// for key in keys {
// index.delete(&mut txn, *field_id, key, *value as u32);
// trivial_db.delete(*field_id, *key, *value as u32);
// }
// }
// }
// }
// }
// for (field_id, values_field_id) in trivial_db.elements.iter() {
// let level0iter = index
// .db
// .content
// .as_polymorph()
// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(
// &mut txn,
// &field_id.to_be_bytes(),
// )
// .unwrap();
// for ((key, values), group) in values_field_id.iter().zip(level0iter) {
// let (group_key, group_values) = group.unwrap();
// let group_key = FacetKeyCodec::<U16Codec>::bytes_decode(group_key).unwrap();
// assert_eq!(key, &group_key.left_bound);
// assert_eq!(values, &group_values.bitmap);
// }
// }
// txn.commit().unwrap();
// let mut txn = index.env.write_txn().unwrap();
// for (field_id, values_field_id) in trivial_db.elements.iter() {
// let level0iter = index
// .db
// .content
// .as_polymorph()
// .prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(&txn, &field_id.to_be_bytes())
// .unwrap();
// for ((key, values), group) in values_field_id.iter().zip(level0iter) {
// let (group_key, group_values) = group.unwrap();
// let group_key = FacetKeyCodec::<U16Codec>::bytes_decode(group_key).unwrap();
// assert_eq!(key, &group_key.left_bound);
// assert_eq!(values, &group_values.bitmap);
// }
// verify_structure_validity(&index, *field_id);
// }
// index.db.content.clear(&mut txn).unwrap();
// txn.commit().unwrap();
// }
// #[test]
// fn fuzz() {
// let tempdir = Rc::new(TempDir::new().unwrap());
// let tempdir_cloned = tempdir.clone();
// let result = fuzzcheck::fuzz_test(move |x: &(u8, u8, Vec<Operation<u16>>)| {
// compare_with_trivial_database(tempdir_cloned.clone(), x.0, x.1, &x.2)
// })
// .default_mutator()
// .serde_serializer()
// .default_sensor_and_pool_with_custom_filter(|file, function| {
// if file.is_relative()
// && !function.contains("serde")
// && !function.contains("tests::")
// && !function.contains("fuzz::")
// && !function.contains("display_bitmap")
// {
// true
// } else {
// false
// }
// })
// .arguments_from_cargo_fuzzcheck()
// .launch();
// assert!(!result.found_test_failure);
// }
// #[test]
// fn reproduce_bug() {
// let operations = r#"
// [
// {"key":0, "field_id": 0, "kind":{"Insert":[109]}},
// {"key":143, "field_id": 0, "kind":{"Insert":[243]}},
// {"key":90, "field_id": 0, "kind":{"Insert":[217]}},
// {"key":172, "field_id": 0, "kind":{"Insert":[94]}},
// {"key":27, "field_id": 0, "kind":{"Insert":[4]}},
// {"key":124, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":123, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":67, "field_id": 0, "kind":{"Insert":[109]}},
// {"key":13, "field_id": 0, "kind":{"Insert":[0]}},
// {"key":162, "field_id": 0, "kind":{"Insert":[213]}},
// {"key":235, "field_id": 0, "kind":{"Insert":[67]}},
// {"key":251, "field_id": 0, "kind":{"Insert":[50]}},
// {"key":218, "field_id": 0, "kind":{"Insert":[164]}},
// {"key":166, "field_id": 0, "kind":{"Insert":[67]}},
// {"key":64, "field_id": 0, "kind":{"Insert":[61]}},
// {"key":183, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":250, "field_id": 0, "kind":{"Delete":50}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations);
// }
// #[test]
// fn reproduce_bug2() {
// let operations = r#"
// [
// {"key":102, "field_id": 0, "kind":{"Insert":[122]}},
// {"key":73, "field_id": 0, "kind":{"Insert":[132]}},
// {"key":20, "field_id": 0, "kind":{"Insert":[215]}},
// {"key":39, "field_id": 0, "kind":{"Insert":[152]}},
// {"key":151, "field_id": 0, "kind":{"Insert":[226]}},
// {"key":17, "field_id": 0, "kind":{"Insert":[101]}},
// {"key":74, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":2, "field_id": 0, "kind":{"Insert":[130]}},
// {"key":64, "field_id": 0, "kind":{"Insert":[180]}},
// {"key":83, "field_id": 0, "kind":{"Insert":[250]}},
// {"key":80, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":113, "field_id": 0, "kind":{"Insert":[63]}},
// {"key":201, "field_id": 0, "kind":{"Insert":[210]}},
// {"key":200, "field_id": 0, "kind":{"Insert":[5]}},
// {"key":93, "field_id": 0, "kind":{"Insert":[98]}},
// {"key":162, "field_id": 0, "kind":{"Insert":[5]}},
// {"key":80, "field_id": 0, "kind":{"Delete":210}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 4, 8, &operations);
// }
// #[test]
// fn reproduce_bug3() {
// let operations = r#"
// [
// {"key":27488, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":64716, "field_id": 0, "kind":{"Insert":[216]}},
// {"key":60886, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":59509, "field_id": 0, "kind":{"Insert":[187,231]}},
// {"key":55057, "field_id": 0, "kind":{"Insert":[37]}},
// {"key":45200, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":55056, "field_id": 0, "kind":{"Insert":[37]}},
// {"key":63679, "field_id": 0, "kind":{"Insert":[206]}},
// {"key":52155, "field_id": 0, "kind":{"Insert":[74]}},
// {"key":20648, "field_id": 0, "kind":{"Insert":[47,138,157]}}
// ]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 0, 7, &operations);
// }
// #[test]
// fn reproduce_bug4() {
// let operations = r#"
// [{"key":63499, "field_id": 0, "kind":{"Insert":[87]}},{"key":25374, "field_id": 0, "kind":{"Insert":[14]}},{"key":64481, "field_id": 0, "kind":{"Delete":87}},{"key":23038, "field_id": 0, "kind":{"Insert":[173]}},{"key":14862, "field_id": 0, "kind":{"Insert":[8]}},{"key":13145, "field_id": 0, "kind":{"Insert":[5,64]}},{"key":23446, "field_id": 0, "kind":{"Insert":[86,59]}},{"key":17972, "field_id": 0, "kind":{"Insert":[58,137]}},{"key":21273, "field_id": 0, "kind":{"Insert":[121,132,81,147]}},{"key":28264, "field_id": 0, "kind":{"Insert":[36]}},{"key":46659, "field_id": 0, "kind":{"Insert":[]}}]
// "#;
// let operations: Vec<Operation<u16>> = serde_json::from_str(operations).unwrap();
// let tempdir = TempDir::new().unwrap();
// compare_with_trivial_database(Rc::new(tempdir), 2, 1, &operations);
// }
// }

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
8bc439472ccda008dc5c28aa789f433d 947949d1a5c9c4e895c89fba46cbba68

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
834f27a924de1acbd3cd94c0d7f10315 5ce8009d3eb023e4b9c0a6e7fa4e6262

View File

@ -1,4 +1,4 @@
--- ---
source: milli/src/update/facet/bulk.rs source: milli/src/update/facet/bulk.rs
--- ---
834f27a924de1acbd3cd94c0d7f10315 5ce8009d3eb023e4b9c0a6e7fa4e6262

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
5dbfa134cc44abeb3ab6242fc182e48e

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
6ed7bf5d440599b3b10b37549a271fdf

View File

@ -0,0 +1,23 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[0, ]"
0 0 k1 1 "[1, ]"
0 0 k2 1 "[2, ]"
0 0 k3 1 "[3, ]"
0 0 k4 1 "[4, ]"
0 0 k5 1 "[5, ]"
0 0 k6 1 "[6, ]"
0 0 k7 1 "[7, ]"
0 0 k8 1 "[8, ]"
0 0 k9 1 "[9, ]"
0 0 k10 1 "[10, ]"
0 0 k11 1 "[11, ]"
0 0 k12 1 "[12, ]"
0 0 k13 1 "[13, ]"
0 0 k14 1 "[14, ]"
0 1 k0 4 "[0, 1, 2, 3, ]"
0 1 k4 4 "[4, 5, 6, 7, ]"
0 1 k8 4 "[8, 9, 10, 11, ]"
0 1 k12 3 "[12, 13, 14, ]"

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b5203f0df0036ebaa133dd77d63a00eb

View File

@ -0,0 +1,26 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[0, ]"
0 0 k1 1 "[1, ]"
0 0 k2 1 "[2, ]"
0 0 k3 1 "[3, ]"
0 0 k4 1 "[4, ]"
0 0 k5 1 "[5, ]"
0 0 k6 1 "[6, ]"
0 0 k7 1 "[7, ]"
0 0 k8 1 "[8, ]"
0 0 k9 1 "[9, ]"
0 0 k10 1 "[10, ]"
0 0 k11 1 "[11, ]"
0 0 k12 1 "[12, ]"
0 0 k13 1 "[13, ]"
0 0 k14 1 "[14, ]"
0 0 k15 1 "[15, ]"
0 0 k16 1 "[16, ]"
0 1 k0 4 "[0, 1, 2, 3, ]"
0 1 k4 4 "[4, 5, 6, 7, ]"
0 1 k8 4 "[8, 9, 10, 11, ]"
0 1 k12 4 "[12, 13, 14, 15, ]"
0 1 k16 1 "[16, ]"

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
95497d8579740868ee0bfc655b0bf782

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
d565c2f7bbd9e13e12de40cfbbfba6bb

View File

@ -0,0 +1,54 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k216 1 "[216, ]"
0 0 k217 1 "[217, ]"
0 0 k218 1 "[218, ]"
0 0 k219 1 "[219, ]"
0 0 k220 1 "[220, ]"
0 0 k221 1 "[221, ]"
0 0 k222 1 "[222, ]"
0 0 k223 1 "[223, ]"
0 0 k224 1 "[224, ]"
0 0 k225 1 "[225, ]"
0 0 k226 1 "[226, ]"
0 0 k227 1 "[227, ]"
0 0 k228 1 "[228, ]"
0 0 k229 1 "[229, ]"
0 0 k230 1 "[230, ]"
0 0 k231 1 "[231, ]"
0 0 k232 1 "[232, ]"
0 0 k233 1 "[233, ]"
0 0 k234 1 "[234, ]"
0 0 k235 1 "[235, ]"
0 0 k236 1 "[236, ]"
0 0 k237 1 "[237, ]"
0 0 k238 1 "[238, ]"
0 0 k239 1 "[239, ]"
0 0 k240 1 "[240, ]"
0 0 k241 1 "[241, ]"
0 0 k242 1 "[242, ]"
0 0 k243 1 "[243, ]"
0 0 k244 1 "[244, ]"
0 0 k245 1 "[245, ]"
0 0 k246 1 "[246, ]"
0 0 k247 1 "[247, ]"
0 0 k248 1 "[248, ]"
0 0 k249 1 "[249, ]"
0 0 k250 1 "[250, ]"
0 0 k251 1 "[251, ]"
0 0 k252 1 "[252, ]"
0 0 k253 1 "[253, ]"
0 0 k254 1 "[254, ]"
0 0 k255 1 "[255, ]"
0 1 k216 4 "[216, 217, 218, 219, ]"
0 1 k220 4 "[220, 221, 222, 223, ]"
0 1 k224 4 "[224, 225, 226, 227, ]"
0 1 k228 4 "[228, 229, 230, 231, ]"
0 1 k232 4 "[232, 233, 234, 235, ]"
0 1 k236 4 "[236, 237, 238, 239, ]"
0 1 k240 4 "[240, 241, 242, 243, ]"
0 1 k244 4 "[244, 245, 246, 247, ]"
0 1 k248 4 "[248, 249, 250, 251, ]"
0 1 k252 4 "[252, 253, 254, 255, ]"

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7cb503827ba17e9670296cc9531a1380

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b061f43e379e16f0617c05d3313d0078

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
81fc9489d6b163935b97433477dea63b

View File

@ -0,0 +1,20 @@
---
source: milli/src/update/facet/incremental.rs
---
0 0 k0 1 "[3, 435, 583, 849, ]"
0 0 k1 1 "[35, 494, 693, 796, ]"
0 0 k2 1 "[76, 420, 526, 909, ]"
0 0 k3 1 "[133, 451, 653, 806, ]"
0 0 k4 1 "[131, 464, 656, 853, ]"
0 0 k5 1 "[61, 308, 701, 903, ]"
0 0 k6 1 "[144, 449, 674, 794, ]"
0 0 k7 1 "[182, 451, 735, 941, ]"
0 0 k8 1 "[6, 359, 679, 1003, ]"
0 0 k9 1 "[197, 418, 659, 904, ]"
0 0 k10 1 "[88, 297, 567, 800, ]"
0 0 k11 1 "[150, 309, 530, 946, ]"
0 0 k12 1 "[156, 466, 567, 892, ]"
0 0 k13 1 "[46, 425, 610, 807, ]"
0 0 k14 1 "[236, 433, 549, 891, ]"
0 0 k15 1 "[207, 472, 603, 974, ]"

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7f8aa18d2b3a6422d55c03bede0563db

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
7f8aa18d2b3a6422d55c03bede0563db

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
b3e2de9020d9e0f3941bc3a179c795ba

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
5dbfa134cc44abeb3ab6242fc182e48e

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
4fc800f49201a336295af0542fdf01ab

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
9343355bf535ed4a0c956df2b229d5e6

View File

@ -0,0 +1,4 @@
---
source: milli/src/update/facet/incremental.rs
---
fd65ce7d96a07aafb0ef6cfb5bf016b8