searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 08:44:27 +01:00
Code Issues Releases Wiki Activity

Merge pull request #121 from Kerollmops/no-cjk-unidecode

Browse Source 
Do not save unidecoded cjk kanjis
This commit is contained in:
Clément Renault 2019-02-23 22:34:47 +01:00 committed by GitHub
commit 19724e5af9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/database/serde/indexer_serializer.rs
View File

@ -8,7 +8,7 @@ use crate::database::serde::SerializerError;
use crate::database::schema::SchemaAttr; use crate::database::schema::SchemaAttr;
use crate::tokenizer::TokenizerBuilder; use crate::tokenizer::TokenizerBuilder;
use crate::tokenizer::Token; use crate::tokenizer::Token;
use crate::{DocumentId, DocIndex}; use crate::{is_cjk, DocumentId, DocIndex};
pub struct IndexerSerializer<'a, 'b, B> { pub struct IndexerSerializer<'a, 'b, B> {
pub tokenizer_builder: &'a B, pub tokenizer_builder: &'a B,
@ -65,13 +65,16 @@ where B: TokenizerBuilder
if self.stop_words.contains(&word_lower) { continue } if self.stop_words.contains(&word_lower) { continue }
// and the unidecoded lowercased version // and the unidecoded lowercased version
if !word_lower.chars().any(is_cjk) {
let word_unidecoded = unidecode::unidecode(word).to_lowercase(); let word_unidecoded = unidecode::unidecode(word).to_lowercase();
let word_unidecoded = word_unidecoded.trim();
if word_lower != word_unidecoded { if word_lower != word_unidecoded {
let char_index = char_index as u32; let char_index = char_index as u32;
let char_length = length; let char_length = length;
let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length }; let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length };
self.update.insert_doc_index(word_unidecoded.into_bytes(), doc_index)?; self.update.insert_doc_index(word_unidecoded.as_bytes().to_vec(), doc_index)?;
}
} }
let char_index = char_index as u32; let char_index = char_index as u32;