searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 14:54:27 +01:00
Code Issues Releases Wiki Activity

Use Charabia in milli

Browse Source
This commit is contained in:
ManyTheFish 2022-06-02 15:47:28 +02:00
parent 192e024ada
commit 86ac8568e6
7 changed files with 127 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
milli/src/lib.rs
View File

@ -21,7 +21,7 @@ pub use filter_parser::{Condition, FilterCondition};
use fxhash::{FxHasher32, FxHasher64};
pub use grenad::CompressionType;
use serde_json::{Map, Value};
pub use {heed, meilisearch_tokenizer as tokenizer};
pub use {charabia as tokenizer, heed};
pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError};
pub use self::criterion::{default_criteria, Criterion, CriterionError};

61
milli/src/search/matches/matching_words.rs
View File

@ -3,8 +3,8 @@ use std::collections::BTreeMap;
use std::fmt;
use std::ops::{Index, IndexMut};
use charabia::Token;
use levenshtein_automata::{Distance, DFA};
use meilisearch_tokenizer::Token;
use crate::search::build_dfa;
@ -99,13 +99,13 @@ impl MatchingWord {
/// Returns the lenght in chars of the match in case of the token matches the term.
pub fn match_token(&self, token: &Token) -> Option<usize> {
match self.dfa.eval(token.text()) {
match self.dfa.eval(token.lemma()) {
Distance::Exact(t) if t <= self.typo => {
if self.prefix {
let len = bytes_to_highlight(token.text(), &self.word);
Some(token.num_chars_from_bytes(len))
let len = bytes_to_highlight(token.lemma(), &self.word);
Some(token.original_lengths(len).0)
} else {
Some(token.num_chars_from_bytes(token.text().len()))
Some(token.original_lengths(token.lemma().len()).0)
}
}
_otherwise => None,
@ -262,7 +262,7 @@ mod tests {
use std::borrow::Cow;
use std::str::from_utf8;
use meilisearch_tokenizer::TokenKind;
use charabia::TokenKind;
use super::*;
use crate::MatchingWords;
@ -344,11 +344,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("word"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("word"),
char_end: "word".chars().count(),
byte_end: "word".len(),
char_map: None,
..Default::default()
})
.next(),
Some(MatchType::Full { char_len: 3, ids: &[2] })
@ -357,11 +356,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("nyc"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("nyc"),
char_end: "nyc".chars().count(),
byte_end: "nyc".len(),
char_map: None,
..Default::default()
})
.next(),
None
@ -370,11 +368,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("world"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("world"),
char_end: "world".chars().count(),
byte_end: "world".len(),
char_map: None,
..Default::default()
})
.next(),
Some(MatchType::Full { char_len: 5, ids: &[2] })
@ -383,11 +380,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("splitted"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("splitted"),
char_end: "splitted".chars().count(),
byte_end: "splitted".len(),
char_map: None,
..Default::default()
})
.next(),
Some(MatchType::Full { char_len: 5, ids: &[0] })
@ -396,11 +392,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("thisnew"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("thisnew"),
char_end: "thisnew".chars().count(),
byte_end: "thisnew".len(),
char_map: None,
..Default::default()
})
.next(),
None
@ -409,11 +404,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("borld"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("borld"),
char_end: "borld".chars().count(),
byte_end: "borld".len(),
char_map: None,
..Default::default()
})
.next(),
Some(MatchType::Full { char_len: 5, ids: &[2] })
@ -422,11 +416,10 @@ mod tests {
matching_words
.match_token(&Token {
kind: TokenKind::Word,
word: Cow::Borrowed("wordsplit"),
byte_start: 0,
char_index: 0,
lemma: Cow::Borrowed("wordsplit"),
char_end: "wordsplit".chars().count(),
byte_end: "wordsplit".len(),
char_map: None,
..Default::default()
})
.next(),
Some(MatchType::Full { char_len: 4, ids: &[2] })

108
milli/src/search/matches/mod.rs
View File

@ -1,8 +1,8 @@
use std::borrow::Cow;
use charabia::{SeparatorKind, Token};
use matching_words::{MatchType, PartialMatch, PrimitiveWordId};
pub use matching_words::{MatchingWord, MatchingWords};
use meilisearch_tokenizer::token::{SeparatorKind, Token};
use serde::Serialize;
pub mod matching_words;
@ -168,13 +168,13 @@ impl<'t> Matcher<'t, '_> {
let current_token_position = *token_position;
let current_word_position = *word_position;
*token_position += 1;
if token.is_separator().is_none() {
if !token.is_separator() {
*word_position += 1;
}
Some((current_token_position, current_word_position, token))
})
.filter(|(_, _, token)| token.is_separator().is_none());
.filter(|(_, _, token)| !token.is_separator());
while let Some((token_position, word_position, word)) = words_positions.next() {
for match_type in self.matching_words.match_token(word) {
@ -243,8 +243,8 @@ impl<'t> Matcher<'t, '_> {
let mut after_tokens = self.tokens[last_match_token_position..].iter().peekable();
while remaining_words > 0 {
let before_token = before_tokens.peek().map(|t| t.is_separator());
let after_token = after_tokens.peek().map(|t| t.is_separator());
let before_token = before_tokens.peek().map(|t| t.separator_kind());
let after_token = after_tokens.peek().map(|t| t.separator_kind());
match (before_token, after_token) {
// we can expand both sides.
@ -470,7 +470,7 @@ impl<'t> Matcher<'t, '_> {
#[cfg(test)]
mod tests {
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use charabia::Tokenize;
use super::*;
use crate::search::matches::matching_words::MatchingWord;
@ -490,30 +490,26 @@ mod tests {
let matching_words = matching_words();
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: false, crop: None };
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
@ -524,44 +520,38 @@ mod tests {
let matching_words = matching_words();
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: true, crop: None };
// empty text.
let text = "";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text, because there is no matches.
assert_eq!(&matcher.format(format_options), &text);
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "Natalie risk her future to build a <em>world</em> with <em>the</em> boy she loves. Emily Henry: <em>The</em> Love That <em>Split</em> <em>The</em> <em>World</em>.");
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text with highlighted matches.
assert_eq!(
@ -580,30 +570,26 @@ mod tests {
let matching_words = MatchingWords::new(matching_words);
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: true, crop: None };
// Text containing prefix match.
let text = "Ŵôřlḑôle";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "<em>Ŵôřlḑ</em>ôle");
// Text containing unicode match.
let text = "Ŵôřlḑ";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "<em>Ŵôřlḑ</em>");
// Text containing unicode match.
let text = "Westfália";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "<em>Westfáli</em>a");
@ -614,28 +600,24 @@ mod tests {
let matching_words = matching_words();
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: false, crop: Some(10) };
// empty text.
let text = "";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no highlight should return 10 first words with a marker at the end.
assert_eq!(
@ -645,8 +627,7 @@ mod tests {
// Text without any match starting by a separator.
let text = "(A quick brown fox can not jump 32 feet, right? Brr, it is cold!)";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no highlight should return 10 first words with a marker at the end.
assert_eq!(
@ -656,19 +637,17 @@ mod tests {
// Test phrase propagation
let text = "Natalie risk her future. Split The World is a book written by Emily Henry. I never read it.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// should crop the phrase instead of croping around the match.
assert_eq!(
&matcher.format(format_options),
"…Split The World is a book written by Emily Henry…"
"… Split The World is a book written by Emily Henry…",
);
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no highlight should return 10 last words with a marker at the start.
assert_eq!(
@ -678,8 +657,7 @@ mod tests {
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// no highlight should return 10 last words with a marker at the start.
assert_eq!(
@ -689,8 +667,7 @@ mod tests {
// Text containing a match unordered and a match ordered.
let text = "The world split void void void void void void void void void split the world void void";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
@ -700,8 +677,7 @@ mod tests {
// Text containing matches with diferent density.
let text = "split void the void void world void void void void void void void void void void split the world void void";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
@ -711,8 +687,7 @@ mod tests {
// Text containing matches with same word.
let text = "split split split split split split void void void void void void void void void void split the world void void";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
@ -726,28 +701,24 @@ mod tests {
let matching_words = matching_words();
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: true, crop: Some(10) };
// empty text.
let text = "";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// both should return 10 first words with a marker at the end.
assert_eq!(
@ -757,8 +728,7 @@ mod tests {
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// both should return 10 last words with a marker at the start and highlighted matches.
assert_eq!(
@ -768,16 +738,14 @@ mod tests {
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// both should return 10 last words with a marker at the start and highlighted matches.
assert_eq!(&matcher.format(format_options), "…she loves. Emily Henry: <em>The</em> Love That <em>Split</em> <em>The</em> <em>World</em>.");
// Text containing a match unordered and a match ordered.
let text = "The world split void void void void void void void void void split the world void void";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
@ -792,11 +760,9 @@ mod tests {
let matching_words = matching_words();
let builder = MatcherBuilder::from_matching_words(matching_words);
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let text = "void void split the world void void.";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
// set a smaller crop size
let format_options = FormatOptions { highlight: false, crop: Some(2) };
@ -847,13 +813,11 @@ mod tests {
let mut builder = MatcherBuilder::from_matching_words(matching_words);
builder.highlight_prefix("_".to_string());
builder.highlight_suffix("_".to_string());
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let format_options = FormatOptions { highlight: true, crop: None };
let text = "the do or die can't be he do and or isn't he";
let analyzed = analyzer.analyze(&text);
let tokens: Vec<_> = analyzed.tokens().collect();
let tokens: Vec<_> = text.tokenize().collect();
let mut matcher = builder.build(&tokens[..], text);
assert_eq!(

12
milli/src/search/mod.rs
View File

@ -6,12 +6,12 @@ use std::result::Result as StdResult;
use std::str::Utf8Error;
use std::time::Instant;
use charabia::TokenizerBuilder;
use distinct::{Distinct, DocIter, FacetDistinct, NoopDistinct};
use fst::automaton::Str;
use fst::{Automaton, IntoStreamer, Streamer};
use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA};
use log::debug;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use once_cell::sync::Lazy;
use roaring::bitmap::RoaringBitmap;
@ -126,14 +126,14 @@ impl<'a> Search<'a> {
builder.words_limit(self.words_limit);
// We make sure that the analyzer is aware of the stop words
// this ensures that the query builder is able to properly remove them.
let mut config = AnalyzerConfig::default();
let mut tokbuilder = TokenizerBuilder::new();
let stop_words = self.index.stop_words(self.rtxn)?;
if let Some(ref stop_words) = stop_words {
config.stop_words(stop_words);
tokbuilder.stop_words(stop_words);
}
let analyzer = Analyzer::new(config);
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokenizer = tokbuilder.build();
let tokens = tokenizer.tokenize(query);
builder
.build(tokens)?
.map_or((None, None, None), |(qt, pq, mw)| (Some(qt), Some(pq), Some(mw)))

93
milli/src/search/query_tree.rs
View File

@ -1,10 +1,9 @@
use std::borrow::Cow;
use std::{cmp, fmt, mem};
use charabia::classifier::ClassifiedTokenIter;
use charabia::{SeparatorKind, TokenKind};
use fst::Set;
use meilisearch_tokenizer::token::SeparatorKind;
use meilisearch_tokenizer::tokenizer::TokenStream;
use meilisearch_tokenizer::TokenKind;
use roaring::RoaringBitmap;
use slice_group_by::GroupBy;
@ -235,9 +234,9 @@ impl<'a> QueryTreeBuilder<'a> {
/// - if `authorize_typos` is set to `false` the query tree will be generated
/// forcing all query words to match documents without any typo
/// (the criterion `typo` will be ignored)
pub fn build(
pub fn build<A: AsRef<[u8]>>(
&self,
query: TokenStream,
query: ClassifiedTokenIter<A>,
) -> Result<Option<(Operation, PrimitiveQuery, MatchingWords)>> {
let stop_words = self.index.stop_words(self.rtxn)?;
let primitive_query = create_primitive_query(query, stop_words, self.words_limit);
@ -649,11 +648,14 @@ impl PrimitiveQueryPart {
/// Create primitive query from tokenized query string,
/// the primitive query is an intermediate state to build the query tree.
fn create_primitive_query(
query: TokenStream,
fn create_primitive_query<A>(
query: ClassifiedTokenIter<A>,
stop_words: Option<Set<&[u8]>>,
words_limit: Option<usize>,
) -> PrimitiveQuery {
) -> PrimitiveQuery
where
A: AsRef<[u8]>,
{
let mut primitive_query = Vec::new();
let mut phrase = Vec::new();
let mut quoted = false;
@ -673,21 +675,18 @@ fn create_primitive_query(
// 2. if the word is not the last token of the query and is not a stop_word we push it as a non-prefix word,
// 3. if the word is the last token of the query we push it as a prefix word.
if quoted {
phrase.push(token.word.to_string());
phrase.push(token.lemma().to_string());
} else if peekable.peek().is_some() {
if !stop_words
.as_ref()
.map_or(false, |swords| swords.contains(token.word.as_ref()))
{
if !stop_words.as_ref().map_or(false, |swords| swords.contains(token.lemma())) {
primitive_query
.push(PrimitiveQueryPart::Word(token.word.to_string(), false));
.push(PrimitiveQueryPart::Word(token.lemma().to_string(), false));
}
} else {
primitive_query.push(PrimitiveQueryPart::Word(token.word.to_string(), true));
primitive_query.push(PrimitiveQueryPart::Word(token.lemma().to_string(), true));
}
}
TokenKind::Separator(separator_kind) => {
let quote_count = token.word.chars().filter(|&s| s == '"').count();
let quote_count = token.lemma().chars().filter(|&s| s == '"').count();
// swap quoted state if we encounter a double quote
if quote_count % 2 != 0 {
quoted = !quoted;
@ -738,8 +737,8 @@ pub fn maximum_proximity(operation: &Operation) -> usize {
mod test {
use std::collections::HashMap;
use charabia::Tokenize;
use maplit::hashmap;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
@ -754,12 +753,12 @@ mod test {
}
impl TestContext {
fn build(
fn build<A: AsRef<[u8]>>(
&self,
optional_words: bool,
authorize_typos: bool,
words_limit: Option<usize>,
query: TokenStream,
query: ClassifiedTokenIter<A>,
) -> Result<Option<(Operation, PrimitiveQuery)>> {
let primitive_query = create_primitive_query(query, None, words_limit);
if !primitive_query.is_empty() {
@ -856,9 +855,7 @@ mod test {
#[test]
fn prefix() {
let query = "hey friends";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -889,9 +886,7 @@ mod test {
#[test]
fn no_prefix() {
let query = "hey friends ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -922,9 +917,7 @@ mod test {
#[test]
fn synonyms() {
let query = "hello world ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -987,9 +980,7 @@ mod test {
#[test]
fn complex_synonyms() {
let query = "new york city ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -1087,9 +1078,7 @@ mod test {
#[test]
fn ngrams() {
let query = "n grams ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -1120,9 +1109,7 @@ mod test {
#[test]
fn word_split() {
let query = "wordsplit fish ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -1159,9 +1146,7 @@ mod test {
#[test]
fn phrase() {
let query = "\"hey friends\" \" \" \"wooop";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::And(vec![
Operation::Phrase(vec!["hey".to_string(), "friends".to_string()]),
@ -1177,9 +1162,7 @@ mod test {
#[test]
fn phrase_with_hard_separator() {
let query = "\"hey friends. wooop wooop\"";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::And(vec![
Operation::Phrase(vec!["hey".to_string(), "friends".to_string()]),
@ -1195,9 +1178,7 @@ mod test {
#[test]
fn optional_word() {
let query = "hey my friend ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
true,
@ -1280,9 +1261,7 @@ mod test {
#[test]
fn optional_word_phrase() {
let query = "\"hey my\"";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Phrase(vec!["hey".to_string(), "my".to_string()]);
let (query_tree, _) =
@ -1294,9 +1273,7 @@ mod test {
#[test]
fn optional_word_multiple_phrases() {
let query = r#""hey" my good "friend""#;
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
true,
@ -1365,9 +1342,7 @@ mod test {
#[test]
fn no_typo() {
let query = "hey friends ";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::Or(
false,
@ -1397,9 +1372,7 @@ mod test {
#[test]
fn words_limit() {
let query = "\"hey my\" good friend";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = result.tokens();
let tokens = query.tokenize();
let expected = Operation::And(vec![
Operation::Phrase(vec!["hey".to_string(), "my".to_string()]),
@ -1441,10 +1414,8 @@ mod test {
#[test]
fn disable_typo_on_word() {
let query = "goodbye";
let analyzer = Analyzer::new(AnalyzerConfig::<Vec<u8>>::default());
let result = analyzer.analyze(query);
let tokens = query.tokenize();
let tokens = result.tokens();
let exact_words = fst::Set::from_iter(Some("goodbye")).unwrap().into_fst().into_inner();
let exact_words = Some(fst::Set::new(exact_words).unwrap().map_data(Cow::Owned).unwrap());
let context = TestContext { exact_words, ..Default::default() };

16
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -3,8 +3,7 @@ use std::convert::TryInto;
use std::fs::File;
use std::{io, mem, str};
use meilisearch_tokenizer::token::SeparatorKind;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token, TokenKind};
use charabia::{SeparatorKind, Token, TokenKind, TokenizerBuilder};
use roaring::RoaringBitmap;
use serde_json::Value;
@ -40,11 +39,11 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
let mut key_buffer = Vec::new();
let mut field_buffer = String::new();
let mut config = AnalyzerConfig::default();
let mut builder = TokenizerBuilder::new();
if let Some(stop_words) = stop_words {
config.stop_words(stop_words);
builder.stop_words(stop_words);
}
let analyzer = Analyzer::<Vec<u8>>::new(AnalyzerConfig::default());
let tokenizer = builder.build();
let mut cursor = obkv_documents.into_cursor()?;
while let Some((key, value)) = cursor.move_on_next()? {
@ -64,12 +63,11 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
serde_json::from_slice(field_bytes).map_err(InternalError::SerdeJson)?;
field_buffer.clear();
if let Some(field) = json_to_string(&value, &mut field_buffer) {
let analyzed = analyzer.analyze(field);
let tokens = process_tokens(analyzed.tokens())
let tokens = process_tokens(tokenizer.tokenize(field))
.take_while(|(p, _)| (*p as u32) < max_positions_per_attributes);
for (index, token) in tokens {
let token = token.text().trim();
let token = token.lemma().trim();
if !token.is_empty() {
key_buffer.truncate(mem::size_of::<u32>());
key_buffer.extend_from_slice(token.as_bytes());
@ -146,7 +144,7 @@ fn process_tokens<'a>(
tokens: impl Iterator<Item = Token<'a>>,
) -> impl Iterator<Item = (usize, Token<'a>)> {
tokens
.skip_while(|token| token.is_separator().is_some())
.skip_while(|token| token.is_separator())
.scan((0, None), |(offset, prev_kind), token| {
match token.kind {
TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {

37
milli/src/update/settings.rs
View File

@ -1,8 +1,8 @@
use std::collections::{BTreeSet, HashMap, HashSet};
use std::result::Result as StdResult;
use charabia::{Tokenizer, TokenizerBuilder};
use itertools::Itertools;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use time::OffsetDateTime;
@ -385,13 +385,12 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
fn update_synonyms(&mut self) -> Result<bool> {
match self.synonyms {
Setting::Set(ref synonyms) => {
fn normalize(analyzer: &Analyzer<&[u8]>, text: &str) -> Vec<String> {
analyzer
.analyze(text)
.tokens()
fn normalize(tokenizer: &Tokenizer<&[u8]>, text: &str) -> Vec<String> {
tokenizer
.tokenize(text)
.filter_map(|token| {
if token.is_word() {
Some(token.text().to_string())
Some(token.lemma().to_string())
} else {
None
}
@ -399,19 +398,19 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
.collect::<Vec<_>>()
}
let mut config = AnalyzerConfig::default();
let mut builder = TokenizerBuilder::new();
let stop_words = self.index.stop_words(self.wtxn)?;
if let Some(stop_words) = &stop_words {
config.stop_words(stop_words);
if let Some(ref stop_words) = stop_words {
builder.stop_words(stop_words);
}
let analyzer = Analyzer::new(config);
let tokenizer = builder.build();
let mut new_synonyms = HashMap::new();
for (word, synonyms) in synonyms {
// Normalize both the word and associated synonyms.
let normalized_word = normalize(&analyzer, word);
let normalized_word = normalize(&tokenizer, word);
let normalized_synonyms =
synonyms.iter().map(|synonym| normalize(&analyzer, synonym));
synonyms.iter().map(|synonym| normalize(&tokenizer, synonym));
// Store the normalized synonyms under the normalized word,
// merging the possible duplicate words.
@ -584,19 +583,19 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
fn update_exact_words(&mut self) -> Result<()> {
match self.exact_words {
Setting::Set(ref mut words) => {
fn normalize(analyzer: &Analyzer<&[u8]>, text: &str) -> String {
analyzer.analyze(text).tokens().map(|token| token.text().to_string()).collect()
fn normalize(tokenizer: &Tokenizer<&[u8]>, text: &str) -> String {
tokenizer.tokenize(text).map(|token| token.lemma().to_string()).collect()
}
let mut config = AnalyzerConfig::default();
let mut builder = TokenizerBuilder::new();
let stop_words = self.index.stop_words(self.wtxn)?;
if let Some(stop_words) = &stop_words {
config.stop_words(stop_words);
if let Some(ref stop_words) = stop_words {
builder.stop_words(stop_words);
}
let analyzer = Analyzer::new(config);
let tokenizer = builder.build();
let mut words: Vec<_> =
words.iter().map(|word| normalize(&analyzer, word)).collect();
words.iter().map(|word| normalize(&tokenizer, word)).collect();
// normalization could reorder words
words.sort_unstable();