searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-25 06:00:08 +01:00
Code Issues Releases Wiki Activity

Make the final merge done in parallel

Browse Source
This commit is contained in:
Clément Renault 2020-08-07 15:44:04 +02:00
parent 91282c8b6a
commit ecd2b2f217
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/bin/indexer.rs
View File

@ -426,6 +426,7 @@ fn main() -> anyhow::Result<()> {
let index = Index::new(&env)?;
let documents_path = opt.database.join("documents.mtbl");
let num_threads = rayon::current_num_threads();
let arc_cache_size = opt.arc_cache_size;
let max_nb_chunks = opt.max_nb_chunks;
@ -483,25 +484,29 @@ fn main() -> anyhow::Result<()> {
docs_stores.push(d);
});
debug!("We are writing into LMDB...");
let mut wtxn = env.write_txn()?;
debug!("We are writing into LMDB and MTBL...");
// We merge the postings lists into LMDB.
merge_into_lmdb(stores, |k, v| lmdb_writer(&mut wtxn, &index, k, v))?;
// We run both merging steps in parallel.
let (lmdb, mtbl) = rayon::join(|| {
// We merge the postings lists into LMDB.
let mut wtxn = env.write_txn()?;
merge_into_lmdb(stores, |k, v| lmdb_writer(&mut wtxn, &index, k, v))?;
Ok(wtxn.commit()?) as anyhow::Result<_>
}, || {
// We also merge the documents into its own MTBL store.
let file = OpenOptions::new().create(true).truncate(true).write(true).read(true).open(documents_path)?;
let mut writer = Writer::builder().compression_type(CompressionType::Snappy).build(file);
let mut builder = Merger::builder(docs_merge);
builder.extend(docs_stores);
builder.build().write_into(&mut writer)?;
Ok(writer.into_inner()?) as anyhow::Result<_>
});
// We also merge the documents into its own MTBL store.
let path = opt.database.join("documents.mtbl");
let file = OpenOptions::new().create(true).truncate(true).write(true).read(true).open(path)?;
let mut writer = Writer::builder().compression_type(CompressionType::Snappy).build(file);
let mut builder = Merger::builder(docs_merge);
builder.extend(docs_stores);
builder.build().write_into(&mut writer)?;
let file = writer.into_inner()?;
let file = lmdb.and(mtbl)?;
let mmap = unsafe { Mmap::map(&file)? };
let documents = Reader::new(&mmap)?;
let documents = Reader::new(mmap)?;
let count = documents.metadata().count_entries;
wtxn.commit()?;
debug!("Wrote {} documents into LMDB", count);
Ok(())