Merge pull request #106 from Kerollmops/fix-criterion

Fix the SumOfTypos and WordsProximity criteria

Cargo.toml

@@ -19,7 +19,7 @@ sdset = "0.3"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = { version = "1.0", features = ["preserve_order"] }
-slice-group-by = "0.2"
+slice-group-by = "0.2.3-alpha"
 unidecode = "0.3"
 rayon = "1.0"
 lockfree = "0.5.1"

src/rank/criterion/exact.rs

@@ -10,7 +10,7 @@ fn number_exact_matches(query_index: &[u32], is_exact: &[bool]) -> usize {
     let mut count = 0;
     let mut index = 0;
 
-    for group in query_index.linear_group_by(PartialEq::eq) {
+    for group in query_index.linear_group() {
         let len = group.len();
         count += is_exact[index..index + len].contains(&true) as usize;
         index += len;

src/rank/criterion/number_of_words.rs

@@ -7,7 +7,7 @@ use crate::rank::RawDocument;
 
 #[inline]
 fn number_of_query_words(query_index: &[u32]) -> usize {
-    query_index.linear_group_by(PartialEq::eq).count()
+    query_index.linear_group().count()
 }
 
 #[derive(Debug, Clone, Copy)]

src/rank/criterion/sum_of_typos.rs

@@ -7,17 +7,20 @@ use crate::rank::RawDocument;
 
 #[inline]
 fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize {
-    let mut sum_typos = 0;
-    let mut number_words = 0;
+    let mut number_words = 0.0;
+    let mut sum_typos = 0.0;
     let mut index = 0;
 
-    for group in query_index.linear_group_by(PartialEq::eq) {
-        sum_typos += distance[index] as isize;
-        number_words += 1;
+    for group in query_index.linear_group() {
+        let typo = distance[index] as f32;
+        sum_typos += (typo + 1.0).log10();
+        number_words += 1.0_f32;
         index += group.len();
     }
 
-    sum_typos - number_words
+    let out = number_words / (sum_typos + 1.0);
+
+    (out * 1000.0) as isize
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -37,7 +40,7 @@ impl Criterion for SumOfTypos {
             sum_matches_typos(query_index, distance)
         };
 
-        lhs.cmp(&rhs)
+        lhs.cmp(&rhs).reverse()
     }
 }
 
@@ -57,9 +60,9 @@ mod tests {
         let query_index1 = &[0, 1];
         let distance1 = &[1, 0];
 
-        let lhs = sum_matches_typos(query_index0, distance0);
-        let rhs = sum_matches_typos(query_index1, distance1);
-        assert_eq!(lhs.cmp(&rhs), Ordering::Less);
+        let doc0 = sum_matches_typos(query_index0, distance0);
+        let doc1 = sum_matches_typos(query_index1, distance1);
+        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
     }
 
     // typing: "bouton manchette"
@@ -74,9 +77,9 @@ mod tests {
         let query_index1 = &[0];
         let distance1 = &[0];
 
-        let lhs = sum_matches_typos(query_index0, distance0);
-        let rhs = sum_matches_typos(query_index1, distance1);
-        assert_eq!(lhs.cmp(&rhs), Ordering::Less);
+        let doc0 = sum_matches_typos(query_index0, distance0);
+        let doc1 = sum_matches_typos(query_index1, distance1);
+        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
     }
 
     // typing: "bouton manchztte"
@@ -91,8 +94,8 @@ mod tests {
         let query_index1 = &[0];
         let distance1 = &[0];
 
-        let lhs = sum_matches_typos(query_index0, distance0);
-        let rhs = sum_matches_typos(query_index1, distance1);
-        assert_eq!(lhs.cmp(&rhs), Ordering::Equal);
+        let doc0 = sum_matches_typos(query_index0, distance0);
+        let doc1 = sum_matches_typos(query_index1, distance1);
+        assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
     }
 }

src/rank/criterion/sum_of_words_attribute.rs

@@ -10,7 +10,7 @@ fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
     let mut sum_attributes = 0;
     let mut index = 0;
 
-    for group in query_index.linear_group_by(PartialEq::eq) {
+    for group in query_index.linear_group() {
         sum_attributes += attribute[index] as usize;
         index += group.len();
     }

src/rank/criterion/sum_of_words_position.rs

@@ -10,7 +10,7 @@ fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u32]) -> usize
     let mut sum_word_index = 0;
     let mut index = 0;
 
-    for group in query_index.linear_group_by(PartialEq::eq) {
+    for group in query_index.linear_group() {
         sum_word_index += word_index[index] as usize;
         index += group.len();
     }

src/rank/criterion/words_proximity.rs

@@ -27,6 +27,7 @@ fn attribute_proximity((lattr, lwi): (u16, u32), (rattr, rwi): (u16, u32)) -> u3
 
 fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 {
     let mut min_prox = u32::max_value();
+
     for a in lattr.iter().zip(lwi) {
         for b in rattr.iter().zip(rwi) {
             let a = clone_tuple(a);
@@ -34,34 +35,43 @@ fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32]))
             min_prox = cmp::min(min_prox, attribute_proximity(a, b));
         }
     }
+
     min_prox
 }
 
-fn matches_proximity(query_index: &[u32], attribute: &[u16], word_index: &[u32]) -> u32 {
+fn matches_proximity(
+    query_index: &[u32],
+    distance: &[u8],
+    attribute: &[u16],
+    word_index: &[u32],
+) -> u32
+{
+    let mut query_index_groups = query_index.linear_group();
     let mut proximity = 0;
-
     let mut index = 0;
-    let mut iter = query_index.linear_group_by(PartialEq::eq);
-    let mut last = iter.next().map(|group| {
-        let len = group.len();
+
+    let get_attr_wi = |index: usize, group_len: usize| {
+        // retrieve the first distance group (with the lowest values)
+        let len = distance[index..index + group_len].linear_group().next().unwrap().len();
 
         let rattr = &attribute[index..index + len];
         let rwi = &word_index[index..index + len];
-        index += len;
 
         (rattr, rwi)
+    };
+
+    let mut last = query_index_groups.next().map(|group| {
+        let attr_wi = get_attr_wi(index, group.len());
+        index += group.len();
+        attr_wi
     });
 
-    while let (Some(lhs), Some(rhs)) = (last, iter.next()) {
-        let len = rhs.len();
-
-        let rattr = &attribute[index..index + len];
-        let rwi = &word_index[index..index + len];
-        let rhs = (rattr, rwi);
-
-        proximity += min_proximity(lhs, rhs);
-        last = Some(rhs);
-        index += len;
+    // iter by windows of size 2
+    while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
+        let attr_wi = get_attr_wi(index, rhs.len());
+        proximity += min_proximity(lhs, attr_wi);
+        last = Some(attr_wi);
+        index += rhs.len();
     }
 
     proximity
@@ -74,16 +84,18 @@ impl Criterion for WordsProximity {
     fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
         let lhs = {
             let query_index = lhs.query_index();
+            let distance = lhs.distance();
             let attribute = lhs.attribute();
             let word_index = lhs.word_index();
-            matches_proximity(query_index, attribute, word_index)
+            matches_proximity(query_index, distance, attribute, word_index)
         };
 
         let rhs = {
             let query_index = rhs.query_index();
+            let distance = rhs.distance();
             let attribute = rhs.attribute();
             let word_index = rhs.word_index();
-            matches_proximity(query_index, attribute, word_index)
+            matches_proximity(query_index, distance, attribute, word_index)
         };
 
         lhs.cmp(&rhs)
@@ -106,13 +118,14 @@ mod tests {
         // { id: 3, attr: 3, attr_index: 1 }
 
         let query_index = &[0, 1, 2, 2, 3];
-        let attribute = &[0, 1, 1, 2, 3];
-        let word_index = &[0, 0, 1, 0, 1];
+        let distance    = &[0, 0, 0, 0, 0];
+        let attribute   = &[0, 1, 1, 2, 3];
+        let word_index  = &[0, 0, 1, 0, 1];
 
         //   soup -> of = 8
         // + of -> the  = 1
         // + the -> day = 8 (not 1)
-        assert_eq!(matches_proximity(query_index, attribute, word_index), 17);
+        assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 17);
     }
 
     #[test]
@@ -128,12 +141,13 @@ mod tests {
         // { id: 3, attr: 1, attr_index: 3 }
 
         let query_index = &[0, 0, 1, 2, 3, 3];
-        let attribute = &[0, 1, 1, 1, 0, 1];
-        let word_index = &[0, 0, 1, 2, 1, 3];
+        let distance    = &[0, 0, 0, 0, 0, 0];
+        let attribute   = &[0, 1, 1, 1, 0, 1];
+        let word_index  = &[0, 0, 1, 2, 1, 3];
 
         //   soup -> of = 1
         // + of -> the  = 1
         // + the -> day = 1
-        assert_eq!(matches_proximity(query_index, attribute, word_index), 3);
+        assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 3);
     }
 }