searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-09 06:28:55 +01:00
Code Issues Releases Wiki Activity

Prefer encoding the output size when compressing documents

Browse Source
This commit is contained in:
Clément Renault 2024-07-02 13:12:56 +02:00
parent bf5d9f68fa
commit 0d63d02ab2
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
1 changed files with 5 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
milli/src/heed_codec/compressed_obkv_codec.rs
View File

@ -30,15 +30,11 @@ impl<'a> CompressedKvReaderU16<'a> {
buffer: &'b mut Vec<u8>, buffer: &'b mut Vec<u8>,
dictionnary: &[u8], dictionnary: &[u8],
) -> Result<KvReaderU16<'b>, lz4_flex::block::DecompressError> { ) -> Result<KvReaderU16<'b>, lz4_flex::block::DecompressError> {
// TODO WHAT THE HECK!!! WHY DO I NEED TO INCREASE THE SIZE PROVIDED let (size, input) = lz4_flex::block::uncompressed_size(self.0)?;
let max_size = lz4_flex::block::get_maximum_output_size(self.0.len()) * 2; buffer.resize(size, 0);
buffer.resize(max_size, 0);
// TODO loop to increase the buffer size of need be // TODO loop to increase the buffer size of need be
let size = lz4_flex::block::decompress_into_with_dict( let size =
self.0, lz4_flex::block::decompress_into_with_dict(input, &mut buffer[..size], dictionnary)?;
&mut buffer[..max_size],
dictionnary,
)?;
Ok(KvReaderU16::new(&buffer[..size])) Ok(KvReaderU16::new(&buffer[..size]))
} }
@ -53,7 +49,7 @@ pub struct CompressedKvWriterU16(Vec<u8>);
impl CompressedKvWriterU16 { impl CompressedKvWriterU16 {
// TODO ask for a KvReaderU16 here // TODO ask for a KvReaderU16 here
pub fn new_with_dictionary(writer: &[u8], dictionary: &[u8]) -> Self { pub fn new_with_dictionary(writer: &[u8], dictionary: &[u8]) -> Self {
CompressedKvWriterU16(lz4_flex::block::compress_with_dict(writer, dictionary)) CompressedKvWriterU16(lz4_flex::block::compress_prepend_size_with_dict(writer, dictionary))
} }
pub fn as_bytes(&self) -> &[u8] { pub fn as_bytes(&self) -> &[u8] {