searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-06-29 07:42:55 +02:00
Code Issues Releases Wiki Activity

Merge only 3 MTBL at the same time

Browse Source
This commit is contained in:
Kerollmops 2020-06-01 19:49:58 +02:00
parent 5cc81a0179
commit 2174042994
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/bin/indexer.rs
View File

@ -280,7 +280,7 @@ fn main() -> anyhow::Result<()> {
let index = Index::new(&env)?;
let stores: Vec<_> = opt.files_to_index
let mut stores: Vec<_> = opt.files_to_index
.into_par_iter()
.map(|path| {
let rdr = csv::Reader::from_path(path)?;
@ -291,7 +291,14 @@ fn main() -> anyhow::Result<()> {
})
.collect::<Result<_, _>>()?;
let mtbl_store = MtblKvStore::from_many(stores)?;
while stores.len() > 1 {
let s = std::mem::take(&mut stores);
stores = s.into_par_iter().chunks(3)
.map(MtblKvStore::from_many)
.collect::<Result<_, _>>()?;
}
let mtbl_store = stores.pop().unwrap_or_default();
eprintln!("We are writing into LMDB...");
let mut wtxn = env.write_txn()?;