From 628c835a220c4b29f12bc23b40ff90ca5292a620 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Wed, 2 Feb 2022 18:45:11 +0100 Subject: [PATCH] fix tests --- Cargo.toml | 3 +++ milli/src/search/mod.rs | 30 +++++++++++++++++++--------- milli/src/search/query_tree.rs | 17 +++++++++------- milli/tests/assets/test_set.ndjson | 2 +- milli/tests/search/query_criteria.rs | 1 + 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6b3e12f07..52599b1bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,6 @@ opt-level = 3 opt-level = 3 [profile.test.build-override] opt-level = 3 + +[patch.crates-io] +fst = { git = "https://github.com/MarinPostma/fst.git", rev = "e6c606b7507e8cb5e502d1609f9b909b8690bac5" } diff --git a/milli/src/search/mod.rs b/milli/src/search/mod.rs index 67b86d6bf..bfe5e023c 100644 --- a/milli/src/search/mod.rs +++ b/milli/src/search/mod.rs @@ -70,6 +70,7 @@ impl<'a> Search<'a> { pub fn offset(&mut self, offset: usize) -> &mut Search<'a> { self.offset = offset; + self } @@ -301,23 +302,34 @@ pub fn word_derivations<'c>( if max_typo == 1 { let dfa = build_dfa(word, 1, is_prefix); let starts = Str::new(get_first(word)).starts_with(); - let mut stream = fst.search(starts.intersection(&dfa)).into_stream(); + let mut stream = fst.search_with_state(starts.intersection(&dfa)).into_stream(); - while let Some(word) = stream.next() { + while let Some((word, state)) = stream.next() { let word = std::str::from_utf8(word)?; - derived_words.push((word.to_string(), 1)); + let d = dfa.distance(state.1); + derived_words.push((word.to_string(), d.to_u8())); } } else { let starts = Str::new(get_first(word)).starts_with(); let first = build_dfa(word, 1, is_prefix).intersection((&starts).complement()); - let second = build_dfa(word, 2, is_prefix).intersection(&starts); - let automaton = first.union(second); + let second_dfa = build_dfa(word, 2, is_prefix); + let second = (&second_dfa).intersection(&starts); + let automaton = first.union(&second); - let mut stream = fst.search(automaton).into_stream(); + let mut stream = fst.search_with_state(automaton).into_stream(); - while let Some(word) = stream.next() { - let word = std::str::from_utf8(word)?; - derived_words.push((word.to_string(), 2)); + while let Some((found_word, state)) = stream.next() { + let found_word = std::str::from_utf8(found_word)?; + // in the case the typo is on the first letter, we know the number of typo + // is two + if get_first(found_word) != get_first(word) { + derived_words.push((word.to_string(), 2)); + } else { + // Else, we know that it is the second dfa that matched and compute the + // correct distance + let d = second_dfa.distance((state.1).0); + derived_words.push((word.to_string(), d.to_u8())); + } } } } diff --git a/milli/src/search/query_tree.rs b/milli/src/search/query_tree.rs index 355e42663..a7285ccaa 100644 --- a/milli/src/search/query_tree.rs +++ b/milli/src/search/query_tree.rs @@ -365,7 +365,10 @@ fn create_query_tree( .collect(); let mut operations = synonyms(ctx, &words)?.unwrap_or_default(); let concat = words.concat(); - let query = Query { prefix: is_prefix, kind: typos(concat, true, 1) }; + let query = Query { + prefix: is_prefix, + kind: typos(concat, authorize_typos, 1), + }; operations.push(Operation::Query(query)); and_op_children.push(Operation::or(false, operations)); } @@ -657,7 +660,7 @@ mod test { ]), Operation::Query(Query { prefix: true, - kind: QueryKind::tolerant(2, "heyfriends".to_string()), + kind: QueryKind::tolerant(1, "heyfriends".to_string()), }), ], ); @@ -690,7 +693,7 @@ mod test { ]), Operation::Query(Query { prefix: false, - kind: QueryKind::tolerant(2, "heyfriends".to_string()), + kind: QueryKind::tolerant(1, "heyfriends".to_string()), }), ], ); @@ -755,7 +758,7 @@ mod test { ]), Operation::Query(Query { prefix: false, - kind: QueryKind::tolerant(2, "helloworld".to_string()), + kind: QueryKind::tolerant(1, "helloworld".to_string()), }), ], ); @@ -853,7 +856,7 @@ mod test { ]), Operation::Query(Query { prefix: false, - kind: QueryKind::tolerant(2, "newyorkcity".to_string()), + kind: QueryKind::tolerant(1, "newyorkcity".to_string()), }), ], ), @@ -927,7 +930,7 @@ mod test { ]), Operation::Query(Query { prefix: false, - kind: QueryKind::tolerant(2, "wordsplitfish".to_string()), + kind: QueryKind::tolerant(1, "wordsplitfish".to_string()), }), ], ); @@ -1047,7 +1050,7 @@ mod test { ]), Operation::Query(Query { prefix: false, - kind: QueryKind::tolerant(2, "heymyfriend".to_string()), + kind: QueryKind::tolerant(1, "heymyfriend".to_string()), }), ], ), diff --git a/milli/tests/assets/test_set.ndjson b/milli/tests/assets/test_set.ndjson index 9a0fe5b0a..6383d274e 100644 --- a/milli/tests/assets/test_set.ndjson +++ b/milli/tests/assets/test_set.ndjson @@ -8,7 +8,7 @@ {"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":202182,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","_geo": { "lat": 48.875617484531965, "lng": 2.346747821504194 },"":""} {"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"sort_by_rank":0,"geo_rank":740667,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","_geo": { "lat": 43.973998070351065, "lng": 3.4661837318345032 },"":""} {"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":739020,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","_geo": { "lat": 43.98920130353838, "lng": 3.480519311627928 },"":""} -{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":738830,"title":"ello creation system","description":"in few word ello was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","_geo": { "lat": 43.99155030238669, "lng": 3.503453528249425 },"":""} +{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":738830,"title":"hallo creation system","description":"in few word hallo was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","_geo": { "lat": 43.99155030238669, "lng": 3.503453528249425 },"":""} {"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":737861,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","_geo": { "lat": 44.000507750283695, "lng": 3.5116812040621572 },"":""} {"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"sort_by_rank":2,"geo_rank":739203,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","_geo": { "lat": 43.99150729038736, "lng": 3.606143957295055 },"":""} {"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":9499586,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","_geo": { "lat": 35.511540843367115, "lng": 138.764368875787 },"":""} diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index 0dcbf660e..ef080db9f 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -61,6 +61,7 @@ test_criterion!( vec![Attribute], vec![] ); +test_criterion!(typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, vec![Typo], vec![]); test_criterion!( attribute_disallow_typo, DISALLOW_OPTIONAL_WORDS,