searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

typos on first letter counts as 2

Browse Source
This commit is contained in:
mpostma 2022-01-20 18:35:11 +01:00 committed by ad hoc
parent 642c01d0dc
commit 55e6cb9c7b
No known key found for this signature in database
GPG Key ID: 4F00A782990CC643
2 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@ -18,3 +18,6 @@ opt-level = 3
opt-level = 3 opt-level = 3
[profile.test.build-override] [profile.test.build-override]
opt-level = 3 opt-level = 3
[patch.crates-io]
fst = { path = "/Users/mpostma/Documents/code/rust/fst/" }

37
milli/src/search/mod.rs
View File

@ -7,7 +7,8 @@ use std::str::Utf8Error;
use std::time::Instant; use std::time::Instant;
use distinct::{Distinct, DocIter, FacetDistinct, NoopDistinct}; use distinct::{Distinct, DocIter, FacetDistinct, NoopDistinct};
use fst::{IntoStreamer, Streamer}; use fst::automaton::Str;
use fst::{Automaton, IntoStreamer, Streamer};
use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA}; use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA};
use log::debug; use log::debug;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
@ -285,19 +286,39 @@ pub fn word_derivations<'c>(
Entry::Vacant(entry) => { Entry::Vacant(entry) => {
let mut derived_words = Vec::new(); let mut derived_words = Vec::new();
let dfa = build_dfa(word, max_typo, is_prefix); let dfa = build_dfa(word, max_typo, is_prefix);
let mut stream = fst.search_with_state(&dfa).into_stream(); if max_typo == 1 {
let starts = Str::new(get_first(word));
let mut stream = fst.search_with_state(starts.intersection(&dfa)).into_stream();
while let Some((word, state)) = stream.next() { while let Some((word, state)) = stream.next() {
let word = std::str::from_utf8(word)?; let word = std::str::from_utf8(word)?;
let distance = dfa.distance(state); let distance = dfa.distance(state.1);
derived_words.push((word.to_string(), distance.to_u8())); derived_words.push((word.to_string(), distance.to_u8()));
}
Ok(entry.insert(derived_words))
} else {
let mut stream = fst.search_with_state(&dfa).into_stream();
while let Some((word, state)) = stream.next() {
let word = std::str::from_utf8(word)?;
let distance = dfa.distance(state);
derived_words.push((word.to_string(), distance.to_u8()));
}
Ok(entry.insert(derived_words))
} }
Ok(entry.insert(derived_words))
} }
} }
} }
fn get_first(s: &str) -> &str {
match s.chars().next() {
Some(c) => &s[..c.len_utf8()],
None => s,
}
}
pub fn build_dfa(word: &str, typos: u8, is_prefix: bool) -> DFA { pub fn build_dfa(word: &str, typos: u8, is_prefix: bool) -> DFA {
let lev = match typos { let lev = match typos {
0 => &LEVDIST0, 0 => &LEVDIST0,