From 85a1f126bfe82c62ddb6afcc4c879ea4b9914346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Feb 2019 15:36:45 +0100 Subject: [PATCH 1/4] fix: Make the SumOfTypos criterion use a more clever algorithm --- src/rank/criterion/sum_of_typos.rs | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/rank/criterion/sum_of_typos.rs b/src/rank/criterion/sum_of_typos.rs index 5d98a42e7..0f7edbbd5 100644 --- a/src/rank/criterion/sum_of_typos.rs +++ b/src/rank/criterion/sum_of_typos.rs @@ -7,17 +7,20 @@ use crate::rank::RawDocument; #[inline] fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize { - let mut sum_typos = 0; - let mut number_words = 0; + let mut number_words = 0.0; + let mut sum_typos = 0.0; let mut index = 0; for group in query_index.linear_group_by(PartialEq::eq) { - sum_typos += distance[index] as isize; - number_words += 1; + let typo = distance[index] as f32; + sum_typos += (typo + 1.0).log10(); + number_words += 1.0_f32; index += group.len(); } - sum_typos - number_words + let out = number_words / (sum_typos + 1.0); + + (out * 1000.0) as isize } #[derive(Debug, Clone, Copy)] @@ -37,7 +40,7 @@ impl Criterion for SumOfTypos { sum_matches_typos(query_index, distance) }; - lhs.cmp(&rhs) + lhs.cmp(&rhs).reverse() } } @@ -57,9 +60,9 @@ mod tests { let query_index1 = &[0, 1]; let distance1 = &[1, 0]; - let lhs = sum_matches_typos(query_index0, distance0); - let rhs = sum_matches_typos(query_index1, distance1); - assert_eq!(lhs.cmp(&rhs), Ordering::Less); + let doc0 = sum_matches_typos(query_index0, distance0); + let doc1 = sum_matches_typos(query_index1, distance1); + assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less); } // typing: "bouton manchette" @@ -74,9 +77,9 @@ mod tests { let query_index1 = &[0]; let distance1 = &[0]; - let lhs = sum_matches_typos(query_index0, distance0); - let rhs = sum_matches_typos(query_index1, distance1); - assert_eq!(lhs.cmp(&rhs), Ordering::Less); + let doc0 = sum_matches_typos(query_index0, distance0); + let doc1 = sum_matches_typos(query_index1, distance1); + assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less); } // typing: "bouton manchztte" @@ -91,8 +94,8 @@ mod tests { let query_index1 = &[0]; let distance1 = &[0]; - let lhs = sum_matches_typos(query_index0, distance0); - let rhs = sum_matches_typos(query_index1, distance1); - assert_eq!(lhs.cmp(&rhs), Ordering::Equal); + let doc0 = sum_matches_typos(query_index0, distance0); + let doc1 = sum_matches_typos(query_index1, distance1); + assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less); } } From 092d446a7ec758105642909d7761041033c4a363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Feb 2019 15:37:22 +0100 Subject: [PATCH 2/4] chore: Update the slice-group-by dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8a02b2657..73de6f2eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ sdset = "0.3" serde = "1.0" serde_derive = "1.0" serde_json = { version = "1.0", features = ["preserve_order"] } -slice-group-by = "0.2" +slice-group-by = "0.2.3-alpha" unidecode = "0.3" rayon = "1.0" lockfree = "0.5.1" From 2e5a616d8e16c93e6785d8b6809de295fe08afe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Feb 2019 15:42:17 +0100 Subject: [PATCH 3/4] fix: Compute the proximity on the words with the min distance --- src/rank/criterion/words_proximity.rs | 62 ++++++++++++++++----------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/rank/criterion/words_proximity.rs b/src/rank/criterion/words_proximity.rs index 614d8f7ff..42cc738ce 100644 --- a/src/rank/criterion/words_proximity.rs +++ b/src/rank/criterion/words_proximity.rs @@ -27,6 +27,7 @@ fn attribute_proximity((lattr, lwi): (u16, u32), (rattr, rwi): (u16, u32)) -> u3 fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 { let mut min_prox = u32::max_value(); + for a in lattr.iter().zip(lwi) { for b in rattr.iter().zip(rwi) { let a = clone_tuple(a); @@ -34,34 +35,43 @@ fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) min_prox = cmp::min(min_prox, attribute_proximity(a, b)); } } + min_prox } -fn matches_proximity(query_index: &[u32], attribute: &[u16], word_index: &[u32]) -> u32 { +fn matches_proximity( + query_index: &[u32], + distance: &[u8], + attribute: &[u16], + word_index: &[u32], +) -> u32 +{ + let mut query_index_groups = query_index.linear_group(); let mut proximity = 0; - let mut index = 0; - let mut iter = query_index.linear_group_by(PartialEq::eq); - let mut last = iter.next().map(|group| { - let len = group.len(); + + let get_attr_wi = |index: usize, group_len: usize| { + // retrieve the first distance group (with the lowest values) + let len = distance[index..index + group_len].linear_group().next().unwrap().len(); let rattr = &attribute[index..index + len]; let rwi = &word_index[index..index + len]; - index += len; (rattr, rwi) + }; + + let mut last = query_index_groups.next().map(|group| { + let attr_wi = get_attr_wi(index, group.len()); + index += group.len(); + attr_wi }); - while let (Some(lhs), Some(rhs)) = (last, iter.next()) { - let len = rhs.len(); - - let rattr = &attribute[index..index + len]; - let rwi = &word_index[index..index + len]; - let rhs = (rattr, rwi); - - proximity += min_proximity(lhs, rhs); - last = Some(rhs); - index += len; + // iter by windows of size 2 + while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) { + let attr_wi = get_attr_wi(index, rhs.len()); + proximity += min_proximity(lhs, attr_wi); + last = Some(attr_wi); + index += rhs.len(); } proximity @@ -74,16 +84,18 @@ impl Criterion for WordsProximity { fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { let lhs = { let query_index = lhs.query_index(); + let distance = lhs.distance(); let attribute = lhs.attribute(); let word_index = lhs.word_index(); - matches_proximity(query_index, attribute, word_index) + matches_proximity(query_index, distance, attribute, word_index) }; let rhs = { let query_index = rhs.query_index(); + let distance = rhs.distance(); let attribute = rhs.attribute(); let word_index = rhs.word_index(); - matches_proximity(query_index, attribute, word_index) + matches_proximity(query_index, distance, attribute, word_index) }; lhs.cmp(&rhs) @@ -106,13 +118,14 @@ mod tests { // { id: 3, attr: 3, attr_index: 1 } let query_index = &[0, 1, 2, 2, 3]; - let attribute = &[0, 1, 1, 2, 3]; - let word_index = &[0, 0, 1, 0, 1]; + let distance = &[0, 0, 0, 0, 0]; + let attribute = &[0, 1, 1, 2, 3]; + let word_index = &[0, 0, 1, 0, 1]; // soup -> of = 8 // + of -> the = 1 // + the -> day = 8 (not 1) - assert_eq!(matches_proximity(query_index, attribute, word_index), 17); + assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 17); } #[test] @@ -128,12 +141,13 @@ mod tests { // { id: 3, attr: 1, attr_index: 3 } let query_index = &[0, 0, 1, 2, 3, 3]; - let attribute = &[0, 1, 1, 1, 0, 1]; - let word_index = &[0, 0, 1, 2, 1, 3]; + let distance = &[0, 0, 0, 0, 0, 0]; + let attribute = &[0, 1, 1, 1, 0, 1]; + let word_index = &[0, 0, 1, 2, 1, 3]; // soup -> of = 1 // + of -> the = 1 // + the -> day = 1 - assert_eq!(matches_proximity(query_index, attribute, word_index), 3); + assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 3); } } From 58b417e04531d0a9669a88f964a6dda47fe441bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Feb 2019 15:43:43 +0100 Subject: [PATCH 4/4] feat: Replace the linear_group_by by the new linear_group method --- src/rank/criterion/exact.rs | 2 +- src/rank/criterion/number_of_words.rs | 2 +- src/rank/criterion/sum_of_typos.rs | 2 +- src/rank/criterion/sum_of_words_attribute.rs | 2 +- src/rank/criterion/sum_of_words_position.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rank/criterion/exact.rs b/src/rank/criterion/exact.rs index 54b5b7b9f..6933aaff5 100644 --- a/src/rank/criterion/exact.rs +++ b/src/rank/criterion/exact.rs @@ -10,7 +10,7 @@ fn number_exact_matches(query_index: &[u32], is_exact: &[bool]) -> usize { let mut count = 0; let mut index = 0; - for group in query_index.linear_group_by(PartialEq::eq) { + for group in query_index.linear_group() { let len = group.len(); count += is_exact[index..index + len].contains(&true) as usize; index += len; diff --git a/src/rank/criterion/number_of_words.rs b/src/rank/criterion/number_of_words.rs index c8dd1edb4..0c6f5a200 100644 --- a/src/rank/criterion/number_of_words.rs +++ b/src/rank/criterion/number_of_words.rs @@ -7,7 +7,7 @@ use crate::rank::RawDocument; #[inline] fn number_of_query_words(query_index: &[u32]) -> usize { - query_index.linear_group_by(PartialEq::eq).count() + query_index.linear_group().count() } #[derive(Debug, Clone, Copy)] diff --git a/src/rank/criterion/sum_of_typos.rs b/src/rank/criterion/sum_of_typos.rs index 0f7edbbd5..c218293e3 100644 --- a/src/rank/criterion/sum_of_typos.rs +++ b/src/rank/criterion/sum_of_typos.rs @@ -11,7 +11,7 @@ fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize { let mut sum_typos = 0.0; let mut index = 0; - for group in query_index.linear_group_by(PartialEq::eq) { + for group in query_index.linear_group() { let typo = distance[index] as f32; sum_typos += (typo + 1.0).log10(); number_words += 1.0_f32; diff --git a/src/rank/criterion/sum_of_words_attribute.rs b/src/rank/criterion/sum_of_words_attribute.rs index 5c42f8552..0a5303490 100644 --- a/src/rank/criterion/sum_of_words_attribute.rs +++ b/src/rank/criterion/sum_of_words_attribute.rs @@ -10,7 +10,7 @@ fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize { let mut sum_attributes = 0; let mut index = 0; - for group in query_index.linear_group_by(PartialEq::eq) { + for group in query_index.linear_group() { sum_attributes += attribute[index] as usize; index += group.len(); } diff --git a/src/rank/criterion/sum_of_words_position.rs b/src/rank/criterion/sum_of_words_position.rs index ad93dc4a8..73ea5978c 100644 --- a/src/rank/criterion/sum_of_words_position.rs +++ b/src/rank/criterion/sum_of_words_position.rs @@ -10,7 +10,7 @@ fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u32]) -> usize let mut sum_word_index = 0; let mut index = 0; - for group in query_index.linear_group_by(PartialEq::eq) { + for group in query_index.linear_group() { sum_word_index += word_index[index] as usize; index += group.len(); }