searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Fix the indexer::index to correctly use the rayon::ThreadPool

Browse source
This commit is contained in:
Clément Renault 2024-11-27 16:27:43 +01:00
parent 98d4a2909e
commit e83534a430
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
2 changed files with 19 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

17
crates/milli/src/update/new/indexer/mod.rs
View file

@ -77,17 +77,18 @@ where
MSP: Fn() -> bool + Sync,
SP: Fn(Progress) + Sync,
{
/// TODO restrict memory and remove this memory from the extractors bump allocators
let bbbuffers: Vec<_> = pool
let mut bbbuffers = Vec::new();
let finished_extraction = AtomicBool::new(false);
let (extractor_sender, mut writer_receiver) = pool
.install(|| {
(0..rayon::current_num_threads())
.map(|_| bbqueue::BBBuffer::new(100 * 1024 * 1024)) // 100 MiB by thread
.collect()
/// TODO restrict memory and remove this memory from the extractors bump allocators
extractor_writer_bbqueue(
&mut bbbuffers,
100 * 1024 * 1024, // 100 MiB
1000,
)
})
.unwrap();
let (extractor_sender, mut writer_receiver) =
pool.install(|| extractor_writer_bbqueue(&bbbuffers, 1000)).unwrap();
let finished_extraction = AtomicBool::new(false);
let metadata_builder = MetadataBuilder::from_index(index, wtxn)?;
let new_fields_ids_map = FieldIdMapWithMetadata::new(new_fields_ids_map, metadata_builder);