searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Serialize docids bitmap one time

Browse Source
This commit is contained in:
ManyTheFish 2024-10-09 14:39:27 +02:00
parent d675e73af1
commit 4e97e38177
2 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
milli/src/update/new/channel.rs
View File

@ -5,6 +5,7 @@ use crossbeam_channel::{IntoIter, Receiver, SendError, Sender};
use grenad::Merger; use grenad::Merger;
use heed::types::Bytes; use heed::types::Bytes;
use memmap2::Mmap; use memmap2::Mmap;
use roaring::RoaringBitmap;
use super::extract::FacetKind; use super::extract::FacetKind;
use super::StdResult; use super::StdResult;
@ -46,6 +47,13 @@ impl KeyValueEntry {
KeyValueEntry::SmallInMemory { key_length: key.len(), data: data.into_boxed_slice() } KeyValueEntry::SmallInMemory { key_length: key.len(), data: data.into_boxed_slice() }
} }
pub fn from_small_key_bitmap(key: &[u8], bitmap: RoaringBitmap) -> Self {
let mut data = Vec::with_capacity(key.len() + bitmap.serialized_size());
data.extend_from_slice(key);
bitmap.serialize_into(&mut data).unwrap();
KeyValueEntry::SmallInMemory { key_length: key.len(), data: data.into_boxed_slice() }
}
pub fn from_large_key_value(key: &[u8], value: Mmap) -> Self { pub fn from_large_key_value(key: &[u8], value: Mmap) -> Self {
KeyValueEntry::LargeOnDisk { key: key.to_vec().into_boxed_slice(), value } KeyValueEntry::LargeOnDisk { key: key.to_vec().into_boxed_slice(), value }
} }
@ -232,10 +240,10 @@ impl MergerSender {
DocumentsSender(self) DocumentsSender(self)
} }
pub fn send_documents_ids(&self, bitmap: &[u8]) -> StdResult<(), SendError<()>> { pub fn send_documents_ids(&self, documents_ids: RoaringBitmap) -> StdResult<(), SendError<()>> {
let entry = EntryOperation::Write(KeyValueEntry::from_small_key_value( let entry = EntryOperation::Write(KeyValueEntry::from_small_key_bitmap(
DOCUMENTS_IDS_KEY.as_bytes(), DOCUMENTS_IDS_KEY.as_bytes(),
bitmap, documents_ids,
)); ));
match self.send(WriterOperation { database: Database::Main, entry }) { match self.send(WriterOperation { database: Database::Main, entry }) {
Ok(()) => Ok(()), Ok(()) => Ok(()),

11
milli/src/update/new/merger.rs
View File

@ -189,9 +189,7 @@ pub fn merge_grenad_entries(
let _entered = span.enter(); let _entered = span.enter();
// Send the documents ids unionized with the current one // Send the documents ids unionized with the current one
/// TODO return the slice of bytes directly sender.send_documents_ids(documents_ids).unwrap();
serialize_bitmap_into_vec(&documents_ids, &mut buffer);
sender.send_documents_ids(&buffer).unwrap();
} }
// ... // ...
@ -447,10 +445,3 @@ fn cbo_bitmap_serialize_into_vec<'b>(bitmap: &RoaringBitmap, buffer: &'b mut Vec
CboRoaringBitmapCodec::serialize_into(bitmap, buffer); CboRoaringBitmapCodec::serialize_into(bitmap, buffer);
buffer.as_slice() buffer.as_slice()
} }
/// TODO Return the slice directly from the serialize_into method
fn serialize_bitmap_into_vec(bitmap: &RoaringBitmap, buffer: &mut Vec<u8>) {
buffer.clear();
bitmap.serialize_into(buffer).unwrap();
// buffer.as_slice()
}