searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Take the BBQueue capacity into account in the max memory

Browse source
This commit is contained in:
Clément Renault 2024-11-28 15:43:14 +01:00
parent b57dd5c58e
commit 3c7ac093d3
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
2 changed files with 22 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/milli/src/update/new/channel.rs
View file

@ -27,8 +27,9 @@ use crate::{CboRoaringBitmapCodec, DocumentId, Index};
/// Creates a tuple of senders/receiver to be used by
/// the extractors and the writer loop.
///
/// The `bbqueue_capacity` represent the number of bytes allocated
/// to each BBQueue buffer and is not the sum of all of them.
/// The `total_bbbuffer_capacity` represent the number of bytes
/// allocated to all BBQueue buffer. It will be split by the
/// number of thread.
///
/// The `channel_capacity` parameter defines the number of
/// too-large-to-fit-in-BBQueue entries that can be sent through
@ -46,10 +47,12 @@ use crate::{CboRoaringBitmapCodec, DocumentId, Index};
/// to the number of available threads in the rayon threadpool.
pub fn extractor_writer_bbqueue(
bbbuffers: &mut Vec<BBBuffer>,
bbbuffer_capacity: usize,
total_bbbuffer_capacity: usize,
channel_capacity: usize,
) -> (ExtractorBbqueueSender, WriterBbqueueReceiver) {
bbbuffers.resize_with(rayon::current_num_threads(), || BBBuffer::new(bbbuffer_capacity));
let current_num_threads = rayon::current_num_threads();
let bbbuffer_capacity = total_bbbuffer_capacity.checked_div(current_num_threads).unwrap();
bbbuffers.resize_with(current_num_threads, || BBBuffer::new(bbbuffer_capacity));
let capacity = bbbuffers.first().unwrap().capacity();
// Read the field description to understand this