searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-24 13:40:31 +01:00
Code Issues Releases Wiki Activity

Increase some channel sizes

Browse Source
This commit is contained in:
Clément Renault 2024-09-05 15:12:07 +02:00
parent 73ce67862d
commit f6b3d1f9a5
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
milli/src/update/new/extract/searchable/mod.rs
View File

@ -66,7 +66,7 @@ pub trait SearchableExtractor {
fields_ids_map.clone(), fields_ids_map.clone(),
CboCachedSorter::new( CboCachedSorter::new(
// TODO use a better value // TODO use a better value
100.try_into().unwrap(), 1_000_000.try_into().unwrap(),
create_sorter( create_sorter(
grenad::SortAlgorithm::Stable, grenad::SortAlgorithm::Stable,
MergeDeladdCboRoaringBitmaps, MergeDeladdCboRoaringBitmaps,

18
milli/src/update/new/indexer/mod.rs
View File

@ -54,7 +54,7 @@ where
PI: IntoParallelIterator<Item = Result<DocumentChange>> + Send, PI: IntoParallelIterator<Item = Result<DocumentChange>> + Send,
PI::Iter: Clone, PI::Iter: Clone,
{ {
let (merger_sender, writer_receiver) = merger_writer_channel(100); let (merger_sender, writer_receiver) = merger_writer_channel(10_000);
// This channel acts as a rendezvous point to ensure that we are one task ahead // This channel acts as a rendezvous point to ensure that we are one task ahead
let (extractor_sender, merger_receiver) = extractors_merger_channels(0); let (extractor_sender, merger_receiver) = extractors_merger_channels(0);
@ -89,10 +89,16 @@ where
Ok(()) as Result<_> Ok(()) as Result<_>
})?; })?;
const TEN_GIB: usize = 10 * 1024 * 1024 * 1024;
let max_memory = TEN_GIB / dbg!(rayon::current_num_threads());
let grenad_parameters = GrenadParameters {
max_memory: Some(max_memory),
..GrenadParameters::default()
};
extract_and_send_docids::<WordDocidsExtractor, WordDocids>( extract_and_send_docids::<WordDocidsExtractor, WordDocids>(
index, index,
&global_fields_ids_map, &global_fields_ids_map,
GrenadParameters::default(), grenad_parameters,
document_changes.clone(), document_changes.clone(),
&extractor_sender, &extractor_sender,
)?; )?;
@ -100,7 +106,7 @@ where
extract_and_send_docids::<WordFidDocidsExtractor, WordFidDocids>( extract_and_send_docids::<WordFidDocidsExtractor, WordFidDocids>(
index, index,
&global_fields_ids_map, &global_fields_ids_map,
GrenadParameters::default(), grenad_parameters,
document_changes.clone(), document_changes.clone(),
&extractor_sender, &extractor_sender,
)?; )?;
@ -108,7 +114,7 @@ where
extract_and_send_docids::<ExactWordDocidsExtractor, ExactWordDocids>( extract_and_send_docids::<ExactWordDocidsExtractor, ExactWordDocids>(
index, index,
&global_fields_ids_map, &global_fields_ids_map,
GrenadParameters::default(), grenad_parameters,
document_changes.clone(), document_changes.clone(),
&extractor_sender, &extractor_sender,
)?; )?;
@ -116,7 +122,7 @@ where
extract_and_send_docids::<WordPositionDocidsExtractor, WordPositionDocids>( extract_and_send_docids::<WordPositionDocidsExtractor, WordPositionDocids>(
index, index,
&global_fields_ids_map, &global_fields_ids_map,
GrenadParameters::default(), grenad_parameters,
document_changes.clone(), document_changes.clone(),
&extractor_sender, &extractor_sender,
)?; )?;
@ -135,7 +141,7 @@ where
>( >(
index, index,
&global_fields_ids_map, &global_fields_ids_map,
GrenadParameters::default(), grenad_parameters,
document_changes.clone(), document_changes.clone(),
&extractor_sender, &extractor_sender,
)?; )?;