searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-23 13:24:27 +01:00
Code Issues Releases Wiki Activity

Display the time it takes to merge user documents ids

Browse Source
This commit is contained in:
Clément Renault 2020-11-22 11:28:35 +01:00
parent b0c5f59c07
commit f06355b0bb
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/update/index_documents/transform.rs
View File

@ -3,10 +3,12 @@ use std::convert::TryFrom;
use std::fs::File; use std::fs::File;
use std::io::{Read, Seek, SeekFrom}; use std::io::{Read, Seek, SeekFrom};
use std::iter::Peekable; use std::iter::Peekable;
use std::time::Instant;
use anyhow::{anyhow, Context}; use anyhow::{anyhow, Context};
use fst::{IntoStreamer, Streamer}; use fst::{IntoStreamer, Streamer};
use grenad::CompressionType; use grenad::CompressionType;
use log::info;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use serde_json::{Map, Value}; use serde_json::{Map, Value};
@ -457,19 +459,22 @@ impl Transform<'_, '_> {
// We create the union between the existing users ids documents ids with the new ones. // We create the union between the existing users ids documents ids with the new ones.
let new_users_ids_documents_ids = new_users_ids_documents_ids_builder.into_map(); let new_users_ids_documents_ids = new_users_ids_documents_ids_builder.into_map();
let union_ = fst::map::OpBuilder::new() let union_op = fst::map::OpBuilder::new()
.add(&users_ids_documents_ids) .add(&users_ids_documents_ids)
.add(&new_users_ids_documents_ids) .add(&new_users_ids_documents_ids)
.r#union(); .r#union();
// We stream and merge the new users ids documents ids map with the existing one. // We stream and merge the new users ids documents ids map with the existing one.
let before_docids_merging = Instant::now();
let mut users_ids_documents_ids_builder = fst::MapBuilder::memory(); let mut users_ids_documents_ids_builder = fst::MapBuilder::memory();
let mut iter = union_.into_stream(); let mut iter = union_op.into_stream();
while let Some((user_id, vals)) = iter.next() { while let Some((user_id, vals)) = iter.next() {
assert_eq!(vals.len(), 1, "there must be exactly one document id"); assert_eq!(vals.len(), 1, "there must be exactly one document id");
users_ids_documents_ids_builder.insert(user_id, vals[0].value)?; users_ids_documents_ids_builder.insert(user_id, vals[0].value)?;
} }
info!("Documents users ids merging took {:.02?}", before_docids_merging.elapsed());
Ok(TransformOutput { Ok(TransformOutput {
primary_key, primary_key,
fields_ids_map, fields_ids_map,