searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

Intersect document ids by inverse popularity of the words

Browse Source 
This reduces the worst request we had which took 56s to now took 3s ("the best of the do").
This commit is contained in:
Kerollmops 2020-07-05 19:33:51 +02:00
parent cd7e64b2b3
commit ec1023e790
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/lib.rs
View File

@ -228,20 +228,32 @@ impl Index {
for positions in positions {
let before = Instant::now();
let mut intersect_docids: Option<RoaringBitmap> = None;
for (word, pos) in positions.iter().enumerate() {
let before = Instant::now();
let union_docids = union_cache.entry((word, *pos)).or_insert_with(|| unions_word_pos(word, *pos));
let mut to_intersect: Vec<_> = positions.iter()
.enumerate()
.map(|(word, pos)| {
let docids = union_cache.entry((word, *pos)).or_insert_with(|| unions_word_pos(word, *pos));
// FIXME don't clone here
(docids.len(), docids.clone())
})
.collect();
let before_intersect = Instant::now();
match &mut intersect_docids {
Some(left) => left.intersect_with(&union_docids),
None => intersect_docids = Some(union_docids.clone()),
}
to_intersect.sort_unstable_by_key(|(l, _)| *l);
let elapsed_retrieving = before.elapsed();
eprintln!("retrieving words took {:.02?} and took {:.02?} to intersect",
before.elapsed(), before_intersect.elapsed());
}
let before_intersect = Instant::now();
let intersect_docids: Option<RoaringBitmap> = to_intersect.into_iter()
.fold(None, |acc, (_, union_docids)| {
match acc {
Some(mut left) => {
left.intersect_with(&union_docids);
Some(left)
},
None => Some(union_docids.clone()),
}
});
eprintln!("retrieving words took {:.02?} and took {:.02?} to intersect",
elapsed_retrieving, before_intersect.elapsed());
eprintln!("for proximity {:?} {:?} we took {:.02?} to find {} documents",
proximity, positions, before.elapsed(),