diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index a2fd96311..bdca8c4a4 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -105,8 +105,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("no-cjk with start_word returns: {:?}", token); - self.char_index += word.chars().count(); return Some(token) } @@ -143,8 +141,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("cjk with start_word returns: {:?}", token); - self.word_index += 1; self.char_index += word.chars().count(); @@ -164,8 +160,6 @@ impl<'a> Iterator for Tokenizer<'a> { char_index: self.char_index, }; - println!("cjk without start_word returns: {:?}", token); - if tail.chars().next().and_then(detect_separator).is_none() { self.word_index += 1; }