searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-11 15:38:55 +01:00
Code Issues Releases Wiki Activity

Serializing a RoaringBitmap into a Vec cannot fail

Browse Source
This commit is contained in:
Kerollmops 2021-06-09 11:28:38 +02:00
parent ff9414a6ba
commit 93978ec38a
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs
View File

@ -23,18 +23,17 @@ impl CboRoaringBitmapCodec {
} }
} }
pub fn serialize_into(roaring: &RoaringBitmap, vec: &mut Vec<u8>) -> io::Result<()> { pub fn serialize_into(roaring: &RoaringBitmap, vec: &mut Vec<u8>) {
if roaring.len() <= THRESHOLD as u64 { if roaring.len() <= THRESHOLD as u64 {
// If the number of items (u32s) to encode is less than or equal to the threshold // If the number of items (u32s) to encode is less than or equal to the threshold
// it means that it would weigh the same or less than the RoaringBitmap // it means that it would weigh the same or less than the RoaringBitmap
// header, so we directly encode them using ByteOrder instead. // header, so we directly encode them using ByteOrder instead.
for integer in roaring { for integer in roaring {
vec.write_u32::<NativeEndian>(integer)?; vec.write_u32::<NativeEndian>(integer).unwrap();
} }
Ok(())
} else { } else {
// Otherwise, we use the classic RoaringBitmapCodec that writes a header. // Otherwise, we use the classic RoaringBitmapCodec that writes a header.
roaring.serialize_into(vec) roaring.serialize_into(vec).unwrap();
} }
} }
@ -68,7 +67,7 @@ impl heed::BytesEncode<'_> for CboRoaringBitmapCodec {
fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> { fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
let mut vec = Vec::with_capacity(Self::serialized_size(item)); let mut vec = Vec::with_capacity(Self::serialized_size(item));
Self::serialize_into(item, &mut vec).ok()?; Self::serialize_into(item, &mut vec);
Some(Cow::Owned(vec)) Some(Cow::Owned(vec))
} }
} }

2
milli/src/update/index_documents/merge_function.rs
View File

@ -112,6 +112,6 @@ fn cbo_roaring_bitmap_merge(values: &[Cow<[u8]>]) -> anyhow::Result<Vec<u8>> {
} }
let mut vec = Vec::new(); let mut vec = Vec::new();
CboRoaringBitmapCodec::serialize_into(&head, &mut vec)?; CboRoaringBitmapCodec::serialize_into(&head, &mut vec);
Ok(vec) Ok(vec)
} }

7
milli/src/update/index_documents/store.rs
View File

@ -407,7 +407,7 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// We serialize the document ids into a buffer // We serialize the document ids into a buffer
buffer.clear(); buffer.clear();
buffer.reserve(CboRoaringBitmapCodec::serialized_size(&docids)); buffer.reserve(CboRoaringBitmapCodec::serialized_size(&docids));
CboRoaringBitmapCodec::serialize_into(&docids, &mut buffer)?; CboRoaringBitmapCodec::serialize_into(&docids, &mut buffer);
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key) { if lmdb_key_valid_size(&key) {
sorter.insert(&key, &buffer)?; sorter.insert(&key, &buffer)?;
@ -469,8 +469,7 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
data_buffer.clear(); data_buffer.clear();
let positions = RoaringBitmap::from_iter(Some(document_id)); let positions = RoaringBitmap::from_iter(Some(document_id));
// We serialize the positions into a buffer. // We serialize the positions into a buffer.
CboRoaringBitmapCodec::serialize_into(&positions, &mut data_buffer) CboRoaringBitmapCodec::serialize_into(&positions, &mut data_buffer);
.with_context(|| "could not serialize positions")?;
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key_buffer) { if lmdb_key_valid_size(&key_buffer) {
@ -706,7 +705,7 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
let mut docids_buffer = Vec::new(); let mut docids_buffer = Vec::new();
for ((fid, count), docids) in self.field_id_word_count_docids { for ((fid, count), docids) in self.field_id_word_count_docids {
docids_buffer.clear(); docids_buffer.clear();
CboRoaringBitmapCodec::serialize_into(&docids, &mut docids_buffer)?; CboRoaringBitmapCodec::serialize_into(&docids, &mut docids_buffer);
self.field_id_word_count_docids_sorter.insert([fid, count], &docids_buffer)?; self.field_id_word_count_docids_sorter.insert([fid, count], &docids_buffer)?;
} }