From 10414791a2d42b124617e984222d13589b484c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Fri, 22 Feb 2019 22:34:37 +0100 Subject: [PATCH] fix: Remove debug println from the tokenizer --- src/tokenizer/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index a2fd96311..bdca8c4a4 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -105,8 +105,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("no-cjk with start_word returns: {:?}", token); - self.char_index += word.chars().count(); return Some(token) } @@ -143,8 +141,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("cjk with start_word returns: {:?}", token); - self.word_index += 1; self.char_index += word.chars().count(); @@ -164,8 +160,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("cjk without start_word returns: {:?}", token); - if tail.chars().next().and_then(detect_separator).is_none() { self.word_index += 1; }