2018-07-07 14:43:29 +02:00
|
|
|
mod sum_of_typos;
|
|
|
|
mod number_of_words;
|
|
|
|
mod words_proximity;
|
|
|
|
mod sum_of_words_attribute;
|
|
|
|
mod sum_of_words_position;
|
|
|
|
mod exact;
|
|
|
|
|
|
|
|
use std::cmp::Ordering;
|
2018-09-09 11:13:58 +02:00
|
|
|
use std::rc::Rc;
|
2018-07-07 14:43:29 +02:00
|
|
|
use std::{mem, vec};
|
2018-09-09 13:35:12 +02:00
|
|
|
use fst::Streamer;
|
2018-08-23 21:32:31 +02:00
|
|
|
use fnv::FnvHashMap;
|
2018-07-07 14:43:29 +02:00
|
|
|
use group_by::GroupByMut;
|
2018-09-09 11:13:58 +02:00
|
|
|
use crate::automaton::{DfaExt, AutomatonExt};
|
2018-09-09 13:35:12 +02:00
|
|
|
use crate::metadata::Metadata;
|
|
|
|
use crate::metadata::ops::{OpBuilder, Union};
|
2018-09-09 11:13:58 +02:00
|
|
|
use crate::{Match, DocumentId};
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-08-25 13:15:04 +02:00
|
|
|
use self::{
|
|
|
|
sum_of_typos::sum_of_typos,
|
|
|
|
number_of_words::number_of_words,
|
|
|
|
words_proximity::words_proximity,
|
|
|
|
sum_of_words_attribute::sum_of_words_attribute,
|
|
|
|
sum_of_words_position::sum_of_words_position,
|
|
|
|
exact::exact,
|
|
|
|
};
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-06-25 22:26:49 +02:00
|
|
|
#[inline]
|
|
|
|
fn match_query_index(a: &Match, b: &Match) -> bool {
|
|
|
|
a.query_index == b.query_index
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
2018-05-27 15:23:43 +02:00
|
|
|
pub struct Document {
|
2018-07-06 22:05:51 +02:00
|
|
|
pub document_id: DocumentId,
|
|
|
|
pub matches: Vec<Match>,
|
2018-05-27 15:23:43 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Document {
|
|
|
|
pub fn new(doc: DocumentId, match_: Match) -> Self {
|
|
|
|
Self::from_sorted_matches(doc, vec![match_])
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn from_sorted_matches(doc: DocumentId, matches: Vec<Match>) -> Self {
|
|
|
|
Self {
|
|
|
|
document_id: doc,
|
|
|
|
matches: matches,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-24 21:01:53 +02:00
|
|
|
fn matches_into_iter(matches: FnvHashMap<DocumentId, Vec<Match>>, limit: usize) -> vec::IntoIter<Document> {
|
|
|
|
let mut documents: Vec<_> = matches.into_iter().map(|(id, mut matches)| {
|
|
|
|
matches.sort_unstable();
|
|
|
|
Document::from_sorted_matches(id, matches)
|
|
|
|
}).collect();
|
|
|
|
|
|
|
|
let sorts = &[
|
|
|
|
sum_of_typos,
|
|
|
|
number_of_words,
|
|
|
|
words_proximity,
|
|
|
|
sum_of_words_attribute,
|
|
|
|
sum_of_words_position,
|
|
|
|
exact,
|
|
|
|
];
|
|
|
|
|
2018-08-25 12:35:29 +02:00
|
|
|
let mut groups = vec![documents.as_mut_slice()];
|
|
|
|
|
|
|
|
for sort in sorts {
|
|
|
|
let temp = mem::replace(&mut groups, Vec::new());
|
|
|
|
let mut computed = 0;
|
|
|
|
|
2018-09-09 13:40:37 +02:00
|
|
|
'grp: for group in temp {
|
2018-08-25 12:35:29 +02:00
|
|
|
group.sort_unstable_by(sort);
|
|
|
|
for group in GroupByMut::new(group, |a, b| sort(a, b) == Ordering::Equal) {
|
|
|
|
computed += group.len();
|
|
|
|
groups.push(group);
|
2018-09-09 13:40:37 +02:00
|
|
|
if computed >= limit { break 'grp }
|
2018-06-25 22:26:49 +02:00
|
|
|
}
|
|
|
|
}
|
2018-05-27 15:23:43 +02:00
|
|
|
}
|
2018-08-24 21:01:53 +02:00
|
|
|
|
|
|
|
documents.truncate(limit);
|
|
|
|
documents.into_iter()
|
2018-05-27 15:23:43 +02:00
|
|
|
}
|
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
pub struct RankedStream<'m>(RankedStreamInner<'m>);
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
impl<'m> RankedStream<'m> {
|
|
|
|
pub fn new(metadata: &'m Metadata, automatons: Vec<DfaExt>, limit: usize) -> Self {
|
2018-09-09 11:13:58 +02:00
|
|
|
let automatons: Vec<_> = automatons.into_iter().map(Rc::new).collect();
|
2018-09-09 13:35:12 +02:00
|
|
|
let mut builder = OpBuilder::with_automatons(automatons.clone());
|
|
|
|
builder.push(metadata);
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-08-25 12:35:29 +02:00
|
|
|
let inner = RankedStreamInner::Fed {
|
2018-09-09 13:35:12 +02:00
|
|
|
inner: builder.union(),
|
2018-05-27 15:23:43 +02:00
|
|
|
automatons: automatons,
|
2018-08-24 21:01:53 +02:00
|
|
|
limit: limit,
|
|
|
|
matches: FnvHashMap::default(),
|
2018-08-25 12:35:29 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
RankedStream(inner)
|
2018-05-27 15:23:43 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
impl<'m, 'a> fst::Streamer<'a> for RankedStream<'m> {
|
2018-07-06 22:05:51 +02:00
|
|
|
type Item = Document;
|
2018-05-27 15:23:43 +02:00
|
|
|
|
|
|
|
fn next(&'a mut self) -> Option<Self::Item> {
|
2018-08-25 12:35:29 +02:00
|
|
|
self.0.next()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
enum RankedStreamInner<'m> {
|
2018-08-25 12:35:29 +02:00
|
|
|
Fed {
|
2018-09-09 13:35:12 +02:00
|
|
|
inner: Union<'m>,
|
2018-09-09 11:13:58 +02:00
|
|
|
automatons: Vec<Rc<DfaExt>>,
|
2018-08-25 12:35:29 +02:00
|
|
|
limit: usize,
|
|
|
|
matches: FnvHashMap<DocumentId, Vec<Match>>,
|
|
|
|
},
|
|
|
|
Pours {
|
|
|
|
inner: vec::IntoIter<Document>,
|
|
|
|
},
|
|
|
|
}
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
impl<'m, 'a> fst::Streamer<'a> for RankedStreamInner<'m> {
|
2018-08-25 12:35:29 +02:00
|
|
|
type Item = Document;
|
|
|
|
|
|
|
|
fn next(&'a mut self) -> Option<Self::Item> {
|
|
|
|
loop {
|
2018-05-27 15:23:43 +02:00
|
|
|
match self {
|
2018-08-25 12:35:29 +02:00
|
|
|
RankedStreamInner::Fed { inner, automatons, limit, matches } => {
|
2018-05-27 15:23:43 +02:00
|
|
|
match inner.next() {
|
2018-07-06 20:58:06 +02:00
|
|
|
Some((string, indexed_values)) => {
|
2018-05-27 15:23:43 +02:00
|
|
|
for iv in indexed_values {
|
|
|
|
|
2018-07-06 20:58:06 +02:00
|
|
|
let automaton = &automatons[iv.index];
|
2018-09-09 11:13:58 +02:00
|
|
|
let distance = automaton.eval(string).to_u8();
|
|
|
|
let same_length = string.len() == automaton.query_len();
|
2018-05-27 15:23:43 +02:00
|
|
|
|
2018-09-09 13:35:12 +02:00
|
|
|
for di in iv.doc_indexes.as_slice() {
|
2018-05-27 15:23:43 +02:00
|
|
|
let match_ = Match {
|
|
|
|
query_index: iv.index as u32,
|
|
|
|
distance: distance,
|
|
|
|
attribute: di.attribute,
|
|
|
|
attribute_index: di.attribute_index,
|
2018-09-09 11:13:58 +02:00
|
|
|
is_exact: distance == 0 && same_length,
|
2018-05-27 15:23:43 +02:00
|
|
|
};
|
|
|
|
matches.entry(di.document)
|
2018-09-09 11:13:58 +02:00
|
|
|
.or_insert_with(Vec::new)
|
|
|
|
.push(match_);
|
2018-05-27 15:23:43 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
None => {
|
2018-08-25 12:35:29 +02:00
|
|
|
let matches = mem::replace(matches, FnvHashMap::default());
|
|
|
|
*self = RankedStreamInner::Pours {
|
|
|
|
inner: matches_into_iter(matches, *limit).into_iter()
|
|
|
|
};
|
2018-05-27 15:23:43 +02:00
|
|
|
},
|
|
|
|
}
|
|
|
|
},
|
2018-08-25 12:35:29 +02:00
|
|
|
RankedStreamInner::Pours { inner } => {
|
2018-07-06 22:05:51 +02:00
|
|
|
return inner.next()
|
2018-05-27 15:23:43 +02:00
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|