feat: Introduce an Index system based on RocksDB

This commit is contained in:
Clément Renault 2018-11-15 17:55:20 +01:00
parent cc52d5dda5
commit b3249d515d
No known key found for this signature in database
GPG key ID: 0151CDAB43460DAE
14 changed files with 569 additions and 97 deletions

View file

@ -79,8 +79,7 @@ impl CsvIndexer {
}
{
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
let title = Tokenizer::new(&product.title).filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 1, title);
let key = format!("{}-title", product.id);
@ -89,8 +88,7 @@ impl CsvIndexer {
}
{
let description = Tokenizer::new(&product.description);
let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
let description = Tokenizer::new(&product.description).filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 2, description);
let key = format!("{}-description", product.id);

View file

@ -84,8 +84,7 @@ impl JsonLinesIndexer {
}
{
let title = Tokenizer::new(&product.title);
let title = title.iter().filter(|&(_, w)| !self.common_words.contains(w));
let title = Tokenizer::new(&product.title).filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 1, title);
let key = format!("{}-title", product.id);
@ -94,8 +93,7 @@ impl JsonLinesIndexer {
}
{
let description = Tokenizer::new(&product.description);
let description = description.iter().filter(|&(_, w)| !self.common_words.contains(w));
let description = Tokenizer::new(&product.description).filter(|&(_, w)| !self.common_words.contains(w));
insert_document_words(&mut builder, product.id, 2, description);
let key = format!("{}-description", product.id);