searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-06-13 08:09:50 +02:00
Code Issues Releases Wiki Activity

fix: Change the tokenizer to accept quotes

Browse Source
This commit is contained in:
Clément Renault 2018-10-17 17:00:49 +02:00
parent af91bfa11f
commit 98899d3ea0
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer/mod.rs
View File

@ -24,7 +24,7 @@ impl<'a> Tokens<'a> {
fn new(string: &str) -> Tokens {
Tokens {
index: 0,
inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-'][..]),
inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-', '\'', '"'][..]),
}
}
}
@ -62,7 +62,7 @@ impl<'a> Iterator for Tokens<'a> {
for (i, c) in self.inner.char_indices() {
let separator = match c {
'.' | ';' | ',' | '!' | '?' | '-' => Some(Long),
' ' => Some(Short),
' ' | '\'' | '"' => Some(Short),
_ => None,
};