Merge pull request #106 from Kerollmops/fix-criterion

Fix the SumOfTypos and WordsProximity criteria
This commit is contained in:
Clément Renault 2019-02-12 22:06:32 +01:00 committed by GitHub
commit b108f1e6c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 62 additions and 45 deletions

View File

@ -19,7 +19,7 @@ sdset = "0.3"
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
serde_json = { version = "1.0", features = ["preserve_order"] } serde_json = { version = "1.0", features = ["preserve_order"] }
slice-group-by = "0.2" slice-group-by = "0.2.3-alpha"
unidecode = "0.3" unidecode = "0.3"
rayon = "1.0" rayon = "1.0"
lockfree = "0.5.1" lockfree = "0.5.1"

View File

@ -10,7 +10,7 @@ fn number_exact_matches(query_index: &[u32], is_exact: &[bool]) -> usize {
let mut count = 0; let mut count = 0;
let mut index = 0; let mut index = 0;
for group in query_index.linear_group_by(PartialEq::eq) { for group in query_index.linear_group() {
let len = group.len(); let len = group.len();
count += is_exact[index..index + len].contains(&true) as usize; count += is_exact[index..index + len].contains(&true) as usize;
index += len; index += len;

View File

@ -7,7 +7,7 @@ use crate::rank::RawDocument;
#[inline] #[inline]
fn number_of_query_words(query_index: &[u32]) -> usize { fn number_of_query_words(query_index: &[u32]) -> usize {
query_index.linear_group_by(PartialEq::eq).count() query_index.linear_group().count()
} }
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]

View File

@ -7,17 +7,20 @@ use crate::rank::RawDocument;
#[inline] #[inline]
fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize { fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize {
let mut sum_typos = 0; let mut number_words = 0.0;
let mut number_words = 0; let mut sum_typos = 0.0;
let mut index = 0; let mut index = 0;
for group in query_index.linear_group_by(PartialEq::eq) { for group in query_index.linear_group() {
sum_typos += distance[index] as isize; let typo = distance[index] as f32;
number_words += 1; sum_typos += (typo + 1.0).log10();
number_words += 1.0_f32;
index += group.len(); index += group.len();
} }
sum_typos - number_words let out = number_words / (sum_typos + 1.0);
(out * 1000.0) as isize
} }
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
@ -37,7 +40,7 @@ impl Criterion for SumOfTypos {
sum_matches_typos(query_index, distance) sum_matches_typos(query_index, distance)
}; };
lhs.cmp(&rhs) lhs.cmp(&rhs).reverse()
} }
} }
@ -57,9 +60,9 @@ mod tests {
let query_index1 = &[0, 1]; let query_index1 = &[0, 1];
let distance1 = &[1, 0]; let distance1 = &[1, 0];
let lhs = sum_matches_typos(query_index0, distance0); let doc0 = sum_matches_typos(query_index0, distance0);
let rhs = sum_matches_typos(query_index1, distance1); let doc1 = sum_matches_typos(query_index1, distance1);
assert_eq!(lhs.cmp(&rhs), Ordering::Less); assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
} }
// typing: "bouton manchette" // typing: "bouton manchette"
@ -74,9 +77,9 @@ mod tests {
let query_index1 = &[0]; let query_index1 = &[0];
let distance1 = &[0]; let distance1 = &[0];
let lhs = sum_matches_typos(query_index0, distance0); let doc0 = sum_matches_typos(query_index0, distance0);
let rhs = sum_matches_typos(query_index1, distance1); let doc1 = sum_matches_typos(query_index1, distance1);
assert_eq!(lhs.cmp(&rhs), Ordering::Less); assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
} }
// typing: "bouton manchztte" // typing: "bouton manchztte"
@ -91,8 +94,8 @@ mod tests {
let query_index1 = &[0]; let query_index1 = &[0];
let distance1 = &[0]; let distance1 = &[0];
let lhs = sum_matches_typos(query_index0, distance0); let doc0 = sum_matches_typos(query_index0, distance0);
let rhs = sum_matches_typos(query_index1, distance1); let doc1 = sum_matches_typos(query_index1, distance1);
assert_eq!(lhs.cmp(&rhs), Ordering::Equal); assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
} }
} }

View File

@ -10,7 +10,7 @@ fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
let mut sum_attributes = 0; let mut sum_attributes = 0;
let mut index = 0; let mut index = 0;
for group in query_index.linear_group_by(PartialEq::eq) { for group in query_index.linear_group() {
sum_attributes += attribute[index] as usize; sum_attributes += attribute[index] as usize;
index += group.len(); index += group.len();
} }

View File

@ -10,7 +10,7 @@ fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u32]) -> usize
let mut sum_word_index = 0; let mut sum_word_index = 0;
let mut index = 0; let mut index = 0;
for group in query_index.linear_group_by(PartialEq::eq) { for group in query_index.linear_group() {
sum_word_index += word_index[index] as usize; sum_word_index += word_index[index] as usize;
index += group.len(); index += group.len();
} }

View File

@ -27,6 +27,7 @@ fn attribute_proximity((lattr, lwi): (u16, u32), (rattr, rwi): (u16, u32)) -> u3
fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 { fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 {
let mut min_prox = u32::max_value(); let mut min_prox = u32::max_value();
for a in lattr.iter().zip(lwi) { for a in lattr.iter().zip(lwi) {
for b in rattr.iter().zip(rwi) { for b in rattr.iter().zip(rwi) {
let a = clone_tuple(a); let a = clone_tuple(a);
@ -34,34 +35,43 @@ fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32]))
min_prox = cmp::min(min_prox, attribute_proximity(a, b)); min_prox = cmp::min(min_prox, attribute_proximity(a, b));
} }
} }
min_prox min_prox
} }
fn matches_proximity(query_index: &[u32], attribute: &[u16], word_index: &[u32]) -> u32 { fn matches_proximity(
query_index: &[u32],
distance: &[u8],
attribute: &[u16],
word_index: &[u32],
) -> u32
{
let mut query_index_groups = query_index.linear_group();
let mut proximity = 0; let mut proximity = 0;
let mut index = 0; let mut index = 0;
let mut iter = query_index.linear_group_by(PartialEq::eq);
let mut last = iter.next().map(|group| { let get_attr_wi = |index: usize, group_len: usize| {
let len = group.len(); // retrieve the first distance group (with the lowest values)
let len = distance[index..index + group_len].linear_group().next().unwrap().len();
let rattr = &attribute[index..index + len]; let rattr = &attribute[index..index + len];
let rwi = &word_index[index..index + len]; let rwi = &word_index[index..index + len];
index += len;
(rattr, rwi) (rattr, rwi)
};
let mut last = query_index_groups.next().map(|group| {
let attr_wi = get_attr_wi(index, group.len());
index += group.len();
attr_wi
}); });
while let (Some(lhs), Some(rhs)) = (last, iter.next()) { // iter by windows of size 2
let len = rhs.len(); while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
let attr_wi = get_attr_wi(index, rhs.len());
let rattr = &attribute[index..index + len]; proximity += min_proximity(lhs, attr_wi);
let rwi = &word_index[index..index + len]; last = Some(attr_wi);
let rhs = (rattr, rwi); index += rhs.len();
proximity += min_proximity(lhs, rhs);
last = Some(rhs);
index += len;
} }
proximity proximity
@ -74,16 +84,18 @@ impl Criterion for WordsProximity {
fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
let lhs = { let lhs = {
let query_index = lhs.query_index(); let query_index = lhs.query_index();
let distance = lhs.distance();
let attribute = lhs.attribute(); let attribute = lhs.attribute();
let word_index = lhs.word_index(); let word_index = lhs.word_index();
matches_proximity(query_index, attribute, word_index) matches_proximity(query_index, distance, attribute, word_index)
}; };
let rhs = { let rhs = {
let query_index = rhs.query_index(); let query_index = rhs.query_index();
let distance = rhs.distance();
let attribute = rhs.attribute(); let attribute = rhs.attribute();
let word_index = rhs.word_index(); let word_index = rhs.word_index();
matches_proximity(query_index, attribute, word_index) matches_proximity(query_index, distance, attribute, word_index)
}; };
lhs.cmp(&rhs) lhs.cmp(&rhs)
@ -106,13 +118,14 @@ mod tests {
// { id: 3, attr: 3, attr_index: 1 } // { id: 3, attr: 3, attr_index: 1 }
let query_index = &[0, 1, 2, 2, 3]; let query_index = &[0, 1, 2, 2, 3];
let attribute = &[0, 1, 1, 2, 3]; let distance = &[0, 0, 0, 0, 0];
let word_index = &[0, 0, 1, 0, 1]; let attribute = &[0, 1, 1, 2, 3];
let word_index = &[0, 0, 1, 0, 1];
// soup -> of = 8 // soup -> of = 8
// + of -> the = 1 // + of -> the = 1
// + the -> day = 8 (not 1) // + the -> day = 8 (not 1)
assert_eq!(matches_proximity(query_index, attribute, word_index), 17); assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 17);
} }
#[test] #[test]
@ -128,12 +141,13 @@ mod tests {
// { id: 3, attr: 1, attr_index: 3 } // { id: 3, attr: 1, attr_index: 3 }
let query_index = &[0, 0, 1, 2, 3, 3]; let query_index = &[0, 0, 1, 2, 3, 3];
let attribute = &[0, 1, 1, 1, 0, 1]; let distance = &[0, 0, 0, 0, 0, 0];
let word_index = &[0, 0, 1, 2, 1, 3]; let attribute = &[0, 1, 1, 1, 0, 1];
let word_index = &[0, 0, 1, 2, 1, 3];
// soup -> of = 1 // soup -> of = 1
// + of -> the = 1 // + of -> the = 1
// + the -> day = 1 // + the -> day = 1
assert_eq!(matches_proximity(query_index, attribute, word_index), 3); assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 3);
} }
} }