diff --git a/meilidb-tokenizer/src/lib.rs b/meilidb-tokenizer/src/lib.rs index 88e389a46..50d577db8 100644 --- a/meilidb-tokenizer/src/lib.rs +++ b/meilidb-tokenizer/src/lib.rs @@ -39,8 +39,8 @@ fn is_separator(c: char) -> bool { fn classify_separator(c: char) -> Option { match c { - ' ' | '\'' | '"' => Some(Soft), - '.' | ';' | ',' | '!' | '?' | '-' | '(' | ')' => Some(Hard), + ' ' | '\'' | ':' | '"' => Some(Soft), + '.' | ';' | ',' | '!' | '?' | '-' | '_' | '(' | ')' => Some(Hard), _ => None, } }