Merge pull request #607 from tpayet/add-separators-tokenizer

Add '@' char as a tokenizer separator

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.10.1 (unreleased)
+
+  - Add '@' character as tokenizer separator (#607)
+
 ## v0.10
 
   - Refined filtering (#592)

meilisearch-tokenizer/src/lib.rs

@@ -54,7 +54,7 @@ fn classify_separator(c: char) -> Option<SeparatorCategory> {
         c if c.is_whitespace() => Some(Soft), // whitespaces
         c if deunicode_char(c) == Some("'") => Some(Soft), // quotes
         c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes
-        '-' | '_' | '\'' | ':' | '/' | '\\' => Some(Soft),
+        '-' | '_' | '\'' | ':' | '/' | '\\' | '@' => Some(Soft),
         '.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard),
         _ => None,
     }