searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

fix: Change the tokenizer to accept quotes

Browse Source
This commit is contained in:
Clément Renault 2018-10-17 17:00:49 +02:00
parent af91bfa11f
commit 98899d3ea0
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer/mod.rs
View File

@ -24,7 +24,7 @@ impl<'a> Tokens<'a> {
fn new(string: &str) -> Tokens { fn new(string: &str) -> Tokens {
Tokens { Tokens {
index: 0, index: 0,
inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-'][..]), inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-', '\'', '"'][..]),
} }
} }
} }
@ -62,7 +62,7 @@ impl<'a> Iterator for Tokens<'a> {
for (i, c) in self.inner.char_indices() { for (i, c) in self.inner.char_indices() {
let separator = match c { let separator = match c {
'.' | ';' | ',' | '!' | '?' | '-' => Some(Long), '.' | ';' | ',' | '!' | '?' | '-' => Some(Long),
' ' => Some(Short), ' ' | '\'' | '"' => Some(Short),
_ => None, _ => None,
}; };