mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-26 23:04:26 +01:00
Merge pull request #106 from Kerollmops/fix-criterion
Fix the SumOfTypos and WordsProximity criteria
This commit is contained in:
commit
b108f1e6c9
@ -19,7 +19,7 @@ sdset = "0.3"
|
|||||||
serde = "1.0"
|
serde = "1.0"
|
||||||
serde_derive = "1.0"
|
serde_derive = "1.0"
|
||||||
serde_json = { version = "1.0", features = ["preserve_order"] }
|
serde_json = { version = "1.0", features = ["preserve_order"] }
|
||||||
slice-group-by = "0.2"
|
slice-group-by = "0.2.3-alpha"
|
||||||
unidecode = "0.3"
|
unidecode = "0.3"
|
||||||
rayon = "1.0"
|
rayon = "1.0"
|
||||||
lockfree = "0.5.1"
|
lockfree = "0.5.1"
|
||||||
|
@ -10,7 +10,7 @@ fn number_exact_matches(query_index: &[u32], is_exact: &[bool]) -> usize {
|
|||||||
let mut count = 0;
|
let mut count = 0;
|
||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
|
|
||||||
for group in query_index.linear_group_by(PartialEq::eq) {
|
for group in query_index.linear_group() {
|
||||||
let len = group.len();
|
let len = group.len();
|
||||||
count += is_exact[index..index + len].contains(&true) as usize;
|
count += is_exact[index..index + len].contains(&true) as usize;
|
||||||
index += len;
|
index += len;
|
||||||
|
@ -7,7 +7,7 @@ use crate::rank::RawDocument;
|
|||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn number_of_query_words(query_index: &[u32]) -> usize {
|
fn number_of_query_words(query_index: &[u32]) -> usize {
|
||||||
query_index.linear_group_by(PartialEq::eq).count()
|
query_index.linear_group().count()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
@ -7,17 +7,20 @@ use crate::rank::RawDocument;
|
|||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize {
|
fn sum_matches_typos(query_index: &[u32], distance: &[u8]) -> isize {
|
||||||
let mut sum_typos = 0;
|
let mut number_words = 0.0;
|
||||||
let mut number_words = 0;
|
let mut sum_typos = 0.0;
|
||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
|
|
||||||
for group in query_index.linear_group_by(PartialEq::eq) {
|
for group in query_index.linear_group() {
|
||||||
sum_typos += distance[index] as isize;
|
let typo = distance[index] as f32;
|
||||||
number_words += 1;
|
sum_typos += (typo + 1.0).log10();
|
||||||
|
number_words += 1.0_f32;
|
||||||
index += group.len();
|
index += group.len();
|
||||||
}
|
}
|
||||||
|
|
||||||
sum_typos - number_words
|
let out = number_words / (sum_typos + 1.0);
|
||||||
|
|
||||||
|
(out * 1000.0) as isize
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
@ -37,7 +40,7 @@ impl Criterion for SumOfTypos {
|
|||||||
sum_matches_typos(query_index, distance)
|
sum_matches_typos(query_index, distance)
|
||||||
};
|
};
|
||||||
|
|
||||||
lhs.cmp(&rhs)
|
lhs.cmp(&rhs).reverse()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -57,9 +60,9 @@ mod tests {
|
|||||||
let query_index1 = &[0, 1];
|
let query_index1 = &[0, 1];
|
||||||
let distance1 = &[1, 0];
|
let distance1 = &[1, 0];
|
||||||
|
|
||||||
let lhs = sum_matches_typos(query_index0, distance0);
|
let doc0 = sum_matches_typos(query_index0, distance0);
|
||||||
let rhs = sum_matches_typos(query_index1, distance1);
|
let doc1 = sum_matches_typos(query_index1, distance1);
|
||||||
assert_eq!(lhs.cmp(&rhs), Ordering::Less);
|
assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
|
||||||
}
|
}
|
||||||
|
|
||||||
// typing: "bouton manchette"
|
// typing: "bouton manchette"
|
||||||
@ -74,9 +77,9 @@ mod tests {
|
|||||||
let query_index1 = &[0];
|
let query_index1 = &[0];
|
||||||
let distance1 = &[0];
|
let distance1 = &[0];
|
||||||
|
|
||||||
let lhs = sum_matches_typos(query_index0, distance0);
|
let doc0 = sum_matches_typos(query_index0, distance0);
|
||||||
let rhs = sum_matches_typos(query_index1, distance1);
|
let doc1 = sum_matches_typos(query_index1, distance1);
|
||||||
assert_eq!(lhs.cmp(&rhs), Ordering::Less);
|
assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
|
||||||
}
|
}
|
||||||
|
|
||||||
// typing: "bouton manchztte"
|
// typing: "bouton manchztte"
|
||||||
@ -91,8 +94,8 @@ mod tests {
|
|||||||
let query_index1 = &[0];
|
let query_index1 = &[0];
|
||||||
let distance1 = &[0];
|
let distance1 = &[0];
|
||||||
|
|
||||||
let lhs = sum_matches_typos(query_index0, distance0);
|
let doc0 = sum_matches_typos(query_index0, distance0);
|
||||||
let rhs = sum_matches_typos(query_index1, distance1);
|
let doc1 = sum_matches_typos(query_index1, distance1);
|
||||||
assert_eq!(lhs.cmp(&rhs), Ordering::Equal);
|
assert_eq!(doc0.cmp(&doc1).reverse(), Ordering::Less);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ fn sum_matches_attributes(query_index: &[u32], attribute: &[u16]) -> usize {
|
|||||||
let mut sum_attributes = 0;
|
let mut sum_attributes = 0;
|
||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
|
|
||||||
for group in query_index.linear_group_by(PartialEq::eq) {
|
for group in query_index.linear_group() {
|
||||||
sum_attributes += attribute[index] as usize;
|
sum_attributes += attribute[index] as usize;
|
||||||
index += group.len();
|
index += group.len();
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ fn sum_matches_attribute_index(query_index: &[u32], word_index: &[u32]) -> usize
|
|||||||
let mut sum_word_index = 0;
|
let mut sum_word_index = 0;
|
||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
|
|
||||||
for group in query_index.linear_group_by(PartialEq::eq) {
|
for group in query_index.linear_group() {
|
||||||
sum_word_index += word_index[index] as usize;
|
sum_word_index += word_index[index] as usize;
|
||||||
index += group.len();
|
index += group.len();
|
||||||
}
|
}
|
||||||
|
@ -27,6 +27,7 @@ fn attribute_proximity((lattr, lwi): (u16, u32), (rattr, rwi): (u16, u32)) -> u3
|
|||||||
|
|
||||||
fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 {
|
fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32])) -> u32 {
|
||||||
let mut min_prox = u32::max_value();
|
let mut min_prox = u32::max_value();
|
||||||
|
|
||||||
for a in lattr.iter().zip(lwi) {
|
for a in lattr.iter().zip(lwi) {
|
||||||
for b in rattr.iter().zip(rwi) {
|
for b in rattr.iter().zip(rwi) {
|
||||||
let a = clone_tuple(a);
|
let a = clone_tuple(a);
|
||||||
@ -34,34 +35,43 @@ fn min_proximity((lattr, lwi): (&[u16], &[u32]), (rattr, rwi): (&[u16], &[u32]))
|
|||||||
min_prox = cmp::min(min_prox, attribute_proximity(a, b));
|
min_prox = cmp::min(min_prox, attribute_proximity(a, b));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
min_prox
|
min_prox
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_proximity(query_index: &[u32], attribute: &[u16], word_index: &[u32]) -> u32 {
|
fn matches_proximity(
|
||||||
|
query_index: &[u32],
|
||||||
|
distance: &[u8],
|
||||||
|
attribute: &[u16],
|
||||||
|
word_index: &[u32],
|
||||||
|
) -> u32
|
||||||
|
{
|
||||||
|
let mut query_index_groups = query_index.linear_group();
|
||||||
let mut proximity = 0;
|
let mut proximity = 0;
|
||||||
|
|
||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
let mut iter = query_index.linear_group_by(PartialEq::eq);
|
|
||||||
let mut last = iter.next().map(|group| {
|
let get_attr_wi = |index: usize, group_len: usize| {
|
||||||
let len = group.len();
|
// retrieve the first distance group (with the lowest values)
|
||||||
|
let len = distance[index..index + group_len].linear_group().next().unwrap().len();
|
||||||
|
|
||||||
let rattr = &attribute[index..index + len];
|
let rattr = &attribute[index..index + len];
|
||||||
let rwi = &word_index[index..index + len];
|
let rwi = &word_index[index..index + len];
|
||||||
index += len;
|
|
||||||
|
|
||||||
(rattr, rwi)
|
(rattr, rwi)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut last = query_index_groups.next().map(|group| {
|
||||||
|
let attr_wi = get_attr_wi(index, group.len());
|
||||||
|
index += group.len();
|
||||||
|
attr_wi
|
||||||
});
|
});
|
||||||
|
|
||||||
while let (Some(lhs), Some(rhs)) = (last, iter.next()) {
|
// iter by windows of size 2
|
||||||
let len = rhs.len();
|
while let (Some(lhs), Some(rhs)) = (last, query_index_groups.next()) {
|
||||||
|
let attr_wi = get_attr_wi(index, rhs.len());
|
||||||
let rattr = &attribute[index..index + len];
|
proximity += min_proximity(lhs, attr_wi);
|
||||||
let rwi = &word_index[index..index + len];
|
last = Some(attr_wi);
|
||||||
let rhs = (rattr, rwi);
|
index += rhs.len();
|
||||||
|
|
||||||
proximity += min_proximity(lhs, rhs);
|
|
||||||
last = Some(rhs);
|
|
||||||
index += len;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
proximity
|
proximity
|
||||||
@ -74,16 +84,18 @@ impl Criterion for WordsProximity {
|
|||||||
fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
|
fn evaluate(&self, lhs: &RawDocument, rhs: &RawDocument) -> Ordering {
|
||||||
let lhs = {
|
let lhs = {
|
||||||
let query_index = lhs.query_index();
|
let query_index = lhs.query_index();
|
||||||
|
let distance = lhs.distance();
|
||||||
let attribute = lhs.attribute();
|
let attribute = lhs.attribute();
|
||||||
let word_index = lhs.word_index();
|
let word_index = lhs.word_index();
|
||||||
matches_proximity(query_index, attribute, word_index)
|
matches_proximity(query_index, distance, attribute, word_index)
|
||||||
};
|
};
|
||||||
|
|
||||||
let rhs = {
|
let rhs = {
|
||||||
let query_index = rhs.query_index();
|
let query_index = rhs.query_index();
|
||||||
|
let distance = rhs.distance();
|
||||||
let attribute = rhs.attribute();
|
let attribute = rhs.attribute();
|
||||||
let word_index = rhs.word_index();
|
let word_index = rhs.word_index();
|
||||||
matches_proximity(query_index, attribute, word_index)
|
matches_proximity(query_index, distance, attribute, word_index)
|
||||||
};
|
};
|
||||||
|
|
||||||
lhs.cmp(&rhs)
|
lhs.cmp(&rhs)
|
||||||
@ -106,13 +118,14 @@ mod tests {
|
|||||||
// { id: 3, attr: 3, attr_index: 1 }
|
// { id: 3, attr: 3, attr_index: 1 }
|
||||||
|
|
||||||
let query_index = &[0, 1, 2, 2, 3];
|
let query_index = &[0, 1, 2, 2, 3];
|
||||||
let attribute = &[0, 1, 1, 2, 3];
|
let distance = &[0, 0, 0, 0, 0];
|
||||||
let word_index = &[0, 0, 1, 0, 1];
|
let attribute = &[0, 1, 1, 2, 3];
|
||||||
|
let word_index = &[0, 0, 1, 0, 1];
|
||||||
|
|
||||||
// soup -> of = 8
|
// soup -> of = 8
|
||||||
// + of -> the = 1
|
// + of -> the = 1
|
||||||
// + the -> day = 8 (not 1)
|
// + the -> day = 8 (not 1)
|
||||||
assert_eq!(matches_proximity(query_index, attribute, word_index), 17);
|
assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 17);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -128,12 +141,13 @@ mod tests {
|
|||||||
// { id: 3, attr: 1, attr_index: 3 }
|
// { id: 3, attr: 1, attr_index: 3 }
|
||||||
|
|
||||||
let query_index = &[0, 0, 1, 2, 3, 3];
|
let query_index = &[0, 0, 1, 2, 3, 3];
|
||||||
let attribute = &[0, 1, 1, 1, 0, 1];
|
let distance = &[0, 0, 0, 0, 0, 0];
|
||||||
let word_index = &[0, 0, 1, 2, 1, 3];
|
let attribute = &[0, 1, 1, 1, 0, 1];
|
||||||
|
let word_index = &[0, 0, 1, 2, 1, 3];
|
||||||
|
|
||||||
// soup -> of = 1
|
// soup -> of = 1
|
||||||
// + of -> the = 1
|
// + of -> the = 1
|
||||||
// + the -> day = 1
|
// + the -> day = 1
|
||||||
assert_eq!(matches_proximity(query_index, attribute, word_index), 3);
|
assert_eq!(matches_proximity(query_index, distance, attribute, word_index), 3);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user