Add '@' char as a tokenizer separator

Update CHANGELOG.md

Co-Authored-By: Clément Renault <renault.cle@gmail.com>
This commit is contained in:
Thomas Payet 2020-04-15 13:57:10 +02:00
parent 45e55bc054
commit 10211737c5
2 changed files with 5 additions and 1 deletions

View File

@ -1,3 +1,7 @@
## v0.10.1 (unreleased)
- Add '@' character as tokenizer separator (#607)
## v0.10
- Refined filtering (#592)

View File

@ -54,7 +54,7 @@ fn classify_separator(c: char) -> Option<SeparatorCategory> {
c if c.is_whitespace() => Some(Soft), // whitespaces
c if deunicode_char(c) == Some("'") => Some(Soft), // quotes
c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes
'-' | '_' | '\'' | ':' | '/' | '\\' => Some(Soft),
'-' | '_' | '\'' | ':' | '/' | '\\' | '@' => Some(Soft),
'.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard),
_ => None,
}