Add support for back/slashes

2019-11-11 21:23:08 +01:00 · 2019-11-11 21:23:08 +01:00 · 2d8f9a9849
parent 3f41681b18
commit 2d8f9a9849
1 changed files with 1 additions and 1 deletions
--- a/meilidb-tokenizer/src/lib.rs
+++ b/meilidb-tokenizer/src/lib.rs
@ -47,7 +47,7 @@ fn classify_separator(c: char) -> Option<SeparatorCategory> {
        c if c.is_whitespace() => Some(Soft), // whitespaces
        c if deunicode_char(c) == Some("'") => Some(Soft), // quotes
        c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes
-        '-' | '_' | '\'' | ':' => Some(Soft),
+        '-' | '_' | '\'' | ':' | '/' | '\\' => Some(Soft),
        '.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard),
        _ => None,
    }