searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Sort entries using rayon in the transform sorters

Browse source
This commit is contained in:
Clément Renault 2023-11-01 10:39:16 +01:00
parent 0fc446c62f
commit c71b1d33ae
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
3 changed files with 40 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

51
milli/src/update/index_documents/transform.rs
View file

@ -114,24 +114,43 @@ impl<'a, 'i> Transform<'a, 'i> {
};
// We initialize the sorter with the user indexing settings.
let original_sorter = create_sorter(
grenad::SortAlgorithm::Stable,
merge_function,
indexer_settings.chunk_compression_type,
indexer_settings.chunk_compression_level,
indexer_settings.max_nb_chunks,
indexer_settings.max_memory.map(|mem| mem / 2),
);
let original_sorter = {
let mut builder = grenad::Sorter::builder(merge_function);
builder.chunk_compression_type(indexer_settings.chunk_compression_type);
if let Some(level) = indexer_settings.chunk_compression_level {
builder.chunk_compression_level(level);
}
if let Some(nb_chunks) = indexer_settings.max_nb_chunks {
builder.max_nb_chunks(nb_chunks);
}
if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) {
builder.dump_threshold(memory);
builder.allow_realloc(false);
}
builder.sort_algorithm(grenad::SortAlgorithm::Stable);
builder.sort_in_parallel(true);
builder.build()
};
// We initialize the sorter with the user indexing settings.
let flattened_sorter = create_sorter(
grenad::SortAlgorithm::Stable,
merge_function,
indexer_settings.chunk_compression_type,
indexer_settings.chunk_compression_level,
indexer_settings.max_nb_chunks,
indexer_settings.max_memory.map(|mem| mem / 2),
);
let flattened_sorter = {
let mut builder = grenad::Sorter::builder(merge_function);
builder.chunk_compression_type(indexer_settings.chunk_compression_type);
if let Some(level) = indexer_settings.chunk_compression_level {
builder.chunk_compression_level(level);
}
if let Some(nb_chunks) = indexer_settings.max_nb_chunks {
builder.max_nb_chunks(nb_chunks);
}
if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) {
builder.dump_threshold(memory);
builder.allow_realloc(false);
}
builder.sort_algorithm(grenad::SortAlgorithm::Stable);
builder.sort_in_parallel(true);
builder.build()
};
let documents_ids = index.documents_ids(wtxn)?;
Ok(Transform {