From 98899d3ea074ea8c313763a0d275f55b2fa4624c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <renault.cle@gmail.com>
Date: Wed, 17 Oct 2018 17:00:49 +0200
Subject: [PATCH] fix: Change the tokenizer to accept quotes

---
 src/tokenizer/mod.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs
index 23bb118a4..1fa8051e5 100644
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -24,7 +24,7 @@ impl<'a> Tokens<'a> {
     fn new(string: &str) -> Tokens {
         Tokens {
             index: 0,
-            inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-'][..]),
+            inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-', '\'', '"'][..]),
         }
     }
 }
@@ -62,7 +62,7 @@ impl<'a> Iterator for Tokens<'a> {
         for (i, c) in self.inner.char_indices() {
             let separator = match c {
                 '.' | ';' | ',' | '!' | '?' | '-' => Some(Long),
-                ' ' => Some(Short),
+                ' ' | '\'' | '"' => Some(Short),
                 _   => None,
             };