searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

Remove log in chunk generator

Browse Source
This commit is contained in:
many 2021-09-02 16:57:46 +02:00
parent 7f7fafb857
commit 741a4444a9
No known key found for this signature in database
GPG Key ID: 2CEF23B75189EACA
3 changed files with 2 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
milli/src/update/index_documents/helpers/grenad_helpers.rs
View File

@ -129,10 +129,8 @@ impl GrenadParameters {
pub fn grenad_obkv_into_chunks<R: io::Read>(
mut reader: grenad::Reader<R>,
indexer: GrenadParameters,
log_frequency: Option<usize>,
documents_chunk_size: usize,
) -> Result<impl Iterator<Item = Result<grenad::Reader<File>>>> {
let mut document_count = 0;
let mut continue_reading = true;
let indexer_clone = indexer.clone();
@ -154,11 +152,6 @@ pub fn grenad_obkv_into_chunks<R: io::Read>(
obkv_documents.insert(document_id, obkv)?;
current_chunk_size += document_id.len() as u64 + obkv.len() as u64;
document_count += 1;
if log_frequency.map_or(false, |log_frequency| document_count % log_frequency == 0) {
debug!("reached {} chunked documents", document_count);
}
if current_chunk_size >= documents_chunk_size as u64 {
return writer_into_reader(obkv_documents).map(Some);
}
@ -168,16 +161,7 @@ pub fn grenad_obkv_into_chunks<R: io::Read>(
writer_into_reader(obkv_documents).map(Some)
};
Ok(std::iter::from_fn(move || {
let result = transposer().transpose();
if result.as_ref().map_or(false, |r| r.is_ok()) {
debug!(
"A new chunk of approximately {:.2} MiB has been generated",
documents_chunk_size as f64 / 1024.0 / 1024.0,
);
}
result
}))
Ok(std::iter::from_fn(move || transposer().transpose()))
}
pub fn write_into_lmdb_database(

1
milli/src/update/index_documents/mod.rs
View File

@ -250,7 +250,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
let chunk_iter = grenad_obkv_into_chunks(
documents_file,
params.clone(),
self.log_every_n,
self.documents_chunk_size.unwrap_or(1024 * 1024 * 128), // 128MiB
);

2
milli/tests/search/mod.rs
View File

@ -5,7 +5,7 @@ use big_s::S;
use either::{Either, Left, Right};
use heed::EnvOpenOptions;
use maplit::{hashmap, hashset};
use milli::update::{IndexDocuments, Settings, UpdateBuilder, UpdateFormat};
use milli::update::{Settings, UpdateBuilder, UpdateFormat};
use milli::{AscDesc, Criterion, DocumentId, Index};
use serde::Deserialize;
use slice_group_by::GroupBy;