searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-23 13:24:27 +01:00
Code Issues Releases Wiki Activity

Merge only 3 MTBL at the same time

Browse Source
This commit is contained in:
Kerollmops 2020-06-01 19:49:58 +02:00
parent 5cc81a0179
commit 2174042994
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/bin/indexer.rs
View File

@ -280,7 +280,7 @@ fn main() -> anyhow::Result<()> {
let index = Index::new(&env)?; let index = Index::new(&env)?;
let stores: Vec<_> = opt.files_to_index let mut stores: Vec<_> = opt.files_to_index
.into_par_iter() .into_par_iter()
.map(|path| { .map(|path| {
let rdr = csv::Reader::from_path(path)?; let rdr = csv::Reader::from_path(path)?;
@ -291,7 +291,14 @@ fn main() -> anyhow::Result<()> {
}) })
.collect::<Result<_, _>>()?; .collect::<Result<_, _>>()?;
let mtbl_store = MtblKvStore::from_many(stores)?; while stores.len() > 1 {
let s = std::mem::take(&mut stores);
stores = s.into_par_iter().chunks(3)
.map(MtblKvStore::from_many)
.collect::<Result<_, _>>()?;
}
let mtbl_store = stores.pop().unwrap_or_default();
eprintln!("We are writing into LMDB..."); eprintln!("We are writing into LMDB...");
let mut wtxn = env.write_txn()?; let mut wtxn = env.write_txn()?;