searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-25 06:00:08 +01:00
Code Issues Releases Wiki Activity

Introduce the new Query Tree creation supporting more operations

Browse Source
This commit is contained in:
Clément Renault 2020-01-22 17:46:46 +01:00
parent 789e05304c
commit 0b9fe2c072
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
1 changed files with 54 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
meilisearch-core/src/query_tree.rs
View File

@ -13,7 +13,7 @@ use log::debug;
use crate::database::MainT; use crate::database::MainT;
use crate::{store, DocumentId, DocIndex, MResult}; use crate::{store, DocumentId, DocIndex, MResult};
use crate::automaton::{build_dfa, build_prefix_dfa, build_exact_dfa}; use crate::automaton::{normalize_str, build_dfa, build_prefix_dfa, build_exact_dfa};
use crate::QueryWordsMapper; use crate::QueryWordsMapper;
#[derive(Clone, PartialEq, Eq, Hash)] #[derive(Clone, PartialEq, Eq, Hash)]
@ -144,7 +144,7 @@ fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'
} }
fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> { fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
let words = words.join(" "); let words = normalize_str(&words.join(" "));
let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default(); let set = ctx.synonyms.synonyms(reader, words.as_bytes())?.unwrap_or_default();
let mut strings = Vec::new(); let mut strings = Vec::new();
@ -159,13 +159,6 @@ fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) ->
Ok(strings) Ok(strings)
} }
fn is_last<I: IntoIterator>(iter: I) -> impl Iterator<Item=(bool, I::Item)> {
let mut iter = iter.into_iter().peekable();
core::iter::from_fn(move || {
iter.next().map(|item| (iter.peek().is_none(), item))
})
}
fn create_operation<I, F>(iter: I, f: F) -> Operation fn create_operation<I, F>(iter: I, f: F) -> Operation
where I: IntoIterator<Item=Operation>, where I: IntoIterator<Item=Operation>,
F: Fn(Vec<Operation>) -> Operation, F: Fn(Vec<Operation>) -> Operation,
@ -186,61 +179,61 @@ pub fn create_query_tree(
) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)> ) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)>
{ {
let words = split_query_string(query).map(str::to_lowercase); let words = split_query_string(query).map(str::to_lowercase);
let words: Vec<_> = words.filter(|s| !s.contains(char::is_whitespace)).enumerate().collect(); let words: Vec<_> = words.into_iter().enumerate().collect();
let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w)); let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w));
let mut ngrams = Vec::new();
for ngram in 1..=MAX_NGRAM {
let ngiter = words.windows(ngram).enumerate().map(|(i, group)| { fn create_inner(
let before = words[0..i].windows(1).enumerate().map(|(i, g)| (i..i+1, g)); reader: &heed::RoTxn<MainT>,
let after = words[i + ngram..].windows(1) ctx: &Context,
.enumerate() mapper: &mut QueryWordsMapper,
.map(move |(j, g)| (i + j + ngram..i + j + ngram + 1, g)); words: &[(usize, String)],
before.chain(Some((i..i + ngram, group))).chain(after) ) -> MResult<Vec<Operation>>
}); {
let mut alts = Vec::new();
for group in ngiter { for ngram in 1..=MAX_NGRAM {
if let Some(group) = words.get(..ngram) {
let mut group_ops = Vec::new();
let mut ops = Vec::new(); let tail = &words[ngram..];
for (is_last, (range, words)) in is_last(group) { let is_last = tail.is_empty();
let mut alts = Vec::new(); let mut group_alts = Vec::new();
match words { match group {
[(id, word)] => { [(id, word)] => {
let mut idgen = ((id + 1) * 100)..; let mut idgen = ((id + 1) * 100)..;
let range = (*id)..id+1;
let phrase = split_best_frequency(reader, ctx, word)? let phrase = split_best_frequency(reader, ctx, word)?.map(|ws| {
.map(|ws| { let id = idgen.next().unwrap();
idgen.next().unwrap();
mapper.declare(range.clone(), id, &[ws.0, ws.1]);
Operation::phrase2(id, is_last, ws)
});
let synonyms = fetch_synonyms(reader, ctx, &[word])?.into_iter().map(|alts| {
let id = idgen.next().unwrap();
mapper.declare(range.clone(), id, &alts);
let mut idgen = once(id).chain(&mut idgen);
let iter = alts.into_iter().map(|w| {
let id = idgen.next().unwrap(); let id = idgen.next().unwrap();
idgen.next().unwrap(); Operation::exact(id, false, &w)
mapper.declare(range.clone(), id, &[ws.0, ws.1]);
Operation::phrase2(id, is_last, ws)
}); });
let synonyms = fetch_synonyms(reader, ctx, &[word])? create_operation(iter, Operation::And)
.into_iter() });
.map(|alts| {
let id = idgen.next().unwrap();
mapper.declare(range.clone(), id, &alts);
let mut idgen = once(id).chain(&mut idgen); let original = Operation::tolerant(*id, is_last, word);
let iter = alts.into_iter().map(|w| {
let id = idgen.next().unwrap();
Operation::exact(id, false, &w)
});
create_operation(iter, Operation::And) group_alts.push(original);
}); group_alts.extend(synonyms.chain(phrase));
let query = Operation::tolerant(*id, is_last, word);
alts.push(query);
alts.extend(synonyms.chain(phrase));
}, },
words => { words => {
let id = words[0].0; let id = words[0].0;
let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..; let mut idgen = ((id + 1) * 100_usize.pow(ngram as u32))..;
let range = id..id+ngram;
let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect(); let words: Vec<_> = words.iter().map(|(_, s)| s.as_str()).collect();
@ -253,25 +246,32 @@ pub fn create_query_tree(
let id = idgen.next().unwrap(); let id = idgen.next().unwrap();
Operation::exact(id, false, &s) Operation::exact(id, false, &s)
}); });
alts.push(create_operation(synonym, Operation::And)); group_alts.push(create_operation(synonym, Operation::And));
} }
let id = idgen.next().unwrap(); let id = idgen.next().unwrap();
let concat = words.concat(); let concat = words.concat();
alts.push(Operation::exact(id, is_last, &concat)); mapper.declare(range.clone(), id, &[&concat]);
mapper.declare(range.clone(), id, &[concat]); group_alts.push(Operation::exact(id, is_last, &concat));
} }
} }
ops.push(create_operation(alts, Operation::Or)); group_ops.push(create_operation(group_alts, Operation::Or));
}
ngrams.push(create_operation(ops, Operation::And)); if !tail.is_empty() {
if ngram == 1 { break } let tail_ops = create_inner(reader, ctx, mapper, tail)?;
group_ops.push(create_operation(tail_ops, Operation::Or));
}
alts.push(create_operation(group_ops, Operation::And));
}
} }
Ok(alts)
} }
let operation = create_operation(ngrams, Operation::Or); let alternatives = create_inner(reader, ctx, &mut mapper, &words)?;
let operation = Operation::Or(alternatives);
let mapping = mapper.mapping(); let mapping = mapper.mapping();
Ok((operation, mapping)) Ok((operation, mapping))