From 6b9426a0510d9c71427cd6ef8c3b2b56d6281269 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Tue, 24 Sep 2019 13:56:26 +0200
Subject: [PATCH] feat: Support underscore as a split character

---
 meilidb-tokenizer/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meilidb-tokenizer/src/lib.rs b/meilidb-tokenizer/src/lib.rs
index 88e389a46..50d577db8 100644
--- a/meilidb-tokenizer/src/lib.rs
+++ b/meilidb-tokenizer/src/lib.rs
@@ -39,8 +39,8 @@ fn is_separator(c: char) -> bool {
 
 fn classify_separator(c: char) -> Option<SeparatorCategory> {
     match c {
-        ' ' | '\'' | '"' => Some(Soft),
-        '.' | ';' | ',' | '!' | '?' | '-' | '(' | ')' => Some(Hard),
+        ' ' | '\'' | ':' | '"' => Some(Soft),
+        '.' | ';' | ',' | '!' | '?' | '-' | '_' | '(' | ')' => Some(Hard),
         _ => None,
     }
 }