searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Retrieve the words before the intersect loops

Browse Source
This commit is contained in:
Kerollmops 2020-06-10 22:05:01 +02:00
parent 6ca3579cc0
commit 4e86ecf807
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/bin/search.rs
View File

@ -6,6 +6,10 @@ use heed::EnvOpenOptions;
use structopt::StructOpt; use structopt::StructOpt;
use mega_mini_indexer::{Index, BEU32}; use mega_mini_indexer::{Index, BEU32};
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
#[derive(Debug, StructOpt)] #[derive(Debug, StructOpt)]
#[structopt(name = "mm-search", about = "The server side of the MMI project.")] #[structopt(name = "mm-search", about = "The server side of the MMI project.")]
struct Opt { struct Opt {

4
src/bin/serve.rs
View File

@ -10,6 +10,10 @@ use warp::{Filter, http::Response};
use mega_mini_indexer::{BEU32, Index}; use mega_mini_indexer::{BEU32, Index};
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
#[derive(Debug, StructOpt)] #[derive(Debug, StructOpt)]
#[structopt(name = "mmi", about = "The server side of the mmi project.")] #[structopt(name = "mmi", about = "The server side of the mmi project.")]
struct Opt { struct Opt {

29
src/lib.rs
View File

@ -121,6 +121,17 @@ impl Index {
eprintln!("Retrieving words positions took {:.02?}", before.elapsed()); eprintln!("Retrieving words positions took {:.02?}", before.elapsed());
// TODO re-enable the prefixes system
let mut words = Vec::new();
for (_word, _is_prefix, dfa) in words_positions {
let mut stream = fst.search(dfa).into_stream();
let mut derived_words = Vec::new();
while let Some(word) = stream.next() {
derived_words.push(word.to_vec());
}
words.push(derived_words);
}
let mut documents = Vec::new(); let mut documents = Vec::new();
'outer: for (proximity, positions) in BestProximity::new(positions) { 'outer: for (proximity, positions) in BestProximity::new(positions) {
@ -131,26 +142,15 @@ impl Index {
let before = Instant::now(); let before = Instant::now();
let mut intersect_docids: Option<RoaringBitmap> = None; let mut intersect_docids: Option<RoaringBitmap> = None;
for ((word, is_prefix, dfa), pos) in words_positions.iter().zip(positions.clone()) { for (derived_words, pos) in words.iter().zip(positions.clone()) {
let mut count = 0; let mut count = 0;
let mut union_docids = RoaringBitmap::default(); let mut union_docids = RoaringBitmap::default();
let before = Instant::now(); let before = Instant::now();
// TODO re-enable the prefixes system // TODO re-enable the prefixes system
if false && word.len() <= 4 && *is_prefix { for word in derived_words.iter() {
let mut key = word.as_bytes()[..word.len().min(5)].to_vec(); let mut key = word.clone();
key.extend_from_slice(&pos.to_be_bytes());
if let Some(ids) = self.prefix_postings_ids.get(rtxn, &key)? {
let right = RoaringBitmap::deserialize_from(ids)?;
union_docids.union_with(&right);
count = 1;
}
} else {
let mut stream = fst.search(dfa).into_stream();
while let Some(word) = stream.next() {
let word = std::str::from_utf8(word)?;
let mut key = word.as_bytes().to_vec();
key.extend_from_slice(&pos.to_be_bytes()); key.extend_from_slice(&pos.to_be_bytes());
if let Some(attrs) = self.postings_ids.get(rtxn, &key)? { if let Some(attrs) = self.postings_ids.get(rtxn, &key)? {
let right = RoaringBitmap::deserialize_from(attrs)?; let right = RoaringBitmap::deserialize_from(attrs)?;
@ -158,7 +158,6 @@ impl Index {
count += 1; count += 1;
} }
} }
}
let before_intersect = Instant::now(); let before_intersect = Instant::now();