searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-09 22:48:54 +01:00
Code Issues Releases Wiki Activity

Increase some channel sizes

Browse Source
This commit is contained in:
Clément Renault 2024-09-05 15:12:07 +02:00
parent 73ce67862d
commit f6b3d1f9a5
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
milli/src/update/new/extract/searchable/mod.rs
View File

@ -66,7 +66,7 @@ pub trait SearchableExtractor {
fields_ids_map.clone(),
CboCachedSorter::new(
// TODO use a better value
100.try_into().unwrap(),
1_000_000.try_into().unwrap(),
create_sorter(
grenad::SortAlgorithm::Stable,
MergeDeladdCboRoaringBitmaps,

18
milli/src/update/new/indexer/mod.rs
View File

@ -54,7 +54,7 @@ where
PI: IntoParallelIterator<Item = Result<DocumentChange>> + Send,
PI::Iter: Clone,
{
let (merger_sender, writer_receiver) = merger_writer_channel(100);
let (merger_sender, writer_receiver) = merger_writer_channel(10_000);
// This channel acts as a rendezvous point to ensure that we are one task ahead
let (extractor_sender, merger_receiver) = extractors_merger_channels(0);
@ -89,10 +89,16 @@ where
Ok(()) as Result<_>
})?;
const TEN_GIB: usize = 10 * 1024 * 1024 * 1024;
let max_memory = TEN_GIB / dbg!(rayon::current_num_threads());
let grenad_parameters = GrenadParameters {
max_memory: Some(max_memory),
..GrenadParameters::default()
};
extract_and_send_docids::<WordDocidsExtractor, WordDocids>(
index,
&global_fields_ids_map,
GrenadParameters::default(),
grenad_parameters,
document_changes.clone(),
&extractor_sender,
)?;
@ -100,7 +106,7 @@ where
extract_and_send_docids::<WordFidDocidsExtractor, WordFidDocids>(
index,
&global_fields_ids_map,
GrenadParameters::default(),
grenad_parameters,
document_changes.clone(),
&extractor_sender,
)?;
@ -108,7 +114,7 @@ where
extract_and_send_docids::<ExactWordDocidsExtractor, ExactWordDocids>(
index,
&global_fields_ids_map,
GrenadParameters::default(),
grenad_parameters,
document_changes.clone(),
&extractor_sender,
)?;
@ -116,7 +122,7 @@ where
extract_and_send_docids::<WordPositionDocidsExtractor, WordPositionDocids>(
index,
&global_fields_ids_map,
GrenadParameters::default(),
grenad_parameters,
document_changes.clone(),
&extractor_sender,
)?;
@ -135,7 +141,7 @@ where
>(
index,
&global_fields_ids_map,
GrenadParameters::default(),
grenad_parameters,
document_changes.clone(),
&extractor_sender,
)?;