searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-02 09:35:45 +01:00
Code Issues Releases Wiki Activity

Reduce the number of KV lookups to the sucessfulls only

Browse Source
This commit is contained in:
Kerollmops 2020-06-16 12:58:29 +02:00
parent e974e6b3c9
commit 3577de04b8
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 20 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/lib.rs
View File

@ -102,11 +102,11 @@ impl Index {
// TODO re-enable the prefixes system // TODO re-enable the prefixes system
let mut stream = fst.search(&dfa).into_stream(); let mut stream = fst.search(&dfa).into_stream();
while let Some(word) = stream.next() { while let Some(word) = stream.next() {
derived_words.push(word.to_vec());
let word = std::str::from_utf8(word)?; let word = std::str::from_utf8(word)?;
if let Some(attrs) = self.postings_attrs.get(rtxn, word)? { if let Some(attrs) = self.postings_attrs.get(rtxn, word)? {
let right = RoaringBitmap::deserialize_from(attrs)?; let right = RoaringBitmap::deserialize_from(attrs)?;
union_positions.union_with(&right); union_positions.union_with(&right);
derived_words.push((word.as_bytes().to_vec(), right));
count += 1; count += 1;
} }
} }
@ -145,14 +145,16 @@ impl Index {
for (i, (derived_words, pos))in once(left).chain(once(right)).enumerate() { for (i, (derived_words, pos))in once(left).chain(once(right)).enumerate() {
let mut union_docids = RoaringBitmap::default(); let mut union_docids = RoaringBitmap::default();
// TODO re-enable the prefixes system // TODO re-enable the prefixes system
for word in derived_words.iter() { for (word, attrs) in derived_words.iter() {
if i == 0 { l_lookups += 1 } else { r_lookups += 1 } if attrs.contains(pos) {
let mut key = word.clone(); if i == 0 { l_lookups += 1 } else { r_lookups += 1 }
key.extend_from_slice(&pos.to_be_bytes()); let mut key = word.clone();
if let Some(attrs) = self.postings_ids.get(rtxn, &key).unwrap() { key.extend_from_slice(&pos.to_be_bytes());
if i == 0 { l_bitmaps += 1 } else { r_bitmaps += 1 } if let Some(attrs) = self.postings_ids.get(rtxn, &key).unwrap() {
let right = RoaringBitmap::deserialize_from(attrs).unwrap(); if i == 0 { l_bitmaps += 1 } else { r_bitmaps += 1 }
union_docids.union_with(&right); let right = RoaringBitmap::deserialize_from(attrs).unwrap();
union_docids.union_with(&right);
}
} }
} }
@ -189,13 +191,15 @@ impl Index {
let before = Instant::now(); let before = Instant::now();
// TODO re-enable the prefixes system // TODO re-enable the prefixes system
for word in derived_words.iter() { for (word, attrs) in derived_words.iter() {
let mut key = word.clone(); if attrs.contains(pos) {
key.extend_from_slice(&pos.to_be_bytes()); let mut key = word.clone();
if let Some(attrs) = self.postings_ids.get(rtxn, &key)? { key.extend_from_slice(&pos.to_be_bytes());
let right = RoaringBitmap::deserialize_from(attrs)?; if let Some(attrs) = self.postings_ids.get(rtxn, &key)? {
union_docids.union_with(&right); let right = RoaringBitmap::deserialize_from(attrs)?;
count += 1; union_docids.union_with(&right);
count += 1;
}
} }
} }