diff --git a/CHANGELOG.md b/CHANGELOG.md index 20cad9d34..312e6645a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.10.1 (unreleased) + + - Add '@' character as tokenizer separator (#607) + ## v0.10 - Refined filtering (#592) diff --git a/meilisearch-tokenizer/src/lib.rs b/meilisearch-tokenizer/src/lib.rs index 784b58116..b13a64c05 100644 --- a/meilisearch-tokenizer/src/lib.rs +++ b/meilisearch-tokenizer/src/lib.rs @@ -54,7 +54,7 @@ fn classify_separator(c: char) -> Option { c if c.is_whitespace() => Some(Soft), // whitespaces c if deunicode_char(c) == Some("'") => Some(Soft), // quotes c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes - '-' | '_' | '\'' | ':' | '/' | '\\' => Some(Soft), + '-' | '_' | '\'' | ':' | '/' | '\\' | '@' => Some(Soft), '.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard), _ => None, }