searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

feat: Introduce RocksDB in this project

Browse source 
in order to save fields of the products
This commit is contained in:
Clément Renault 2018-07-06 21:26:07 +02:00
parent b9a4be10c6
commit d6e113c683
6 changed files with 354 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

19
raptor-indexer/src/main.rs
View file

@ -2,6 +2,7 @@
// make only one binary
extern crate raptor;
extern crate rocksdb;
extern crate serde_json;
#[macro_use] extern crate serde_derive;
extern crate unidecode;
@ -13,6 +14,7 @@ use std::io::{self, BufReader, BufRead};
use std::iter;
use raptor::{DocIndexMapBuilder, DocIndexMap, DocIndex};
use rocksdb::{DB, WriteBatch, Writable};
use serde_json::from_str;
use unidecode::unidecode;
@ -62,8 +64,9 @@ fn main() {
let map_file = "map.fst";
let values_file = "values.vecs";
let rocksdb_file = "rocksdb/storage";
for file in &[map_file, values_file] {
for file in &[map_file, values_file, rocksdb_file] {
match is_readonly(file) {
Ok(true) => panic!("the {:?} file is readonly, please make it writeable", file),
Err(ref e) if e.kind() == io::ErrorKind::NotFound => (),
@ -72,6 +75,9 @@ fn main() {
}
}
fs::remove_file(rocksdb_file);
let db = DB::open_default(rocksdb_file).unwrap();
let mut builder = DocIndexMapBuilder::new();
for line in data.lines() {
let line = line.unwrap();
@ -81,6 +87,16 @@ fn main() {
let title = iter::repeat(0).zip(product.title.split_whitespace()).filter(|&(_, w)| !common_words.contains(w)).enumerate();
let description = iter::repeat(1).zip(product.ft.split_whitespace()).filter(|&(_, w)| !common_words.contains(w)).enumerate();
let mut batch = WriteBatch::new();
let title_key = format!("{}-title", product.product_id);
let _ = batch.put(title_key.as_bytes(), product.title.as_bytes());
let description_key = format!("{}-description", product.product_id);
let _ = batch.put(description_key.as_bytes(), product.ft.as_bytes());
db.write(batch).unwrap();
let words = title.chain(description);
for (i, (attr, word)) in words {
let doc_index = DocIndex {
@ -108,6 +124,7 @@ fn main() {
set_readonly(map_file, true).unwrap();
set_readonly(values_file, true).unwrap();
set_readonly(rocksdb_file, true).unwrap();
println!("Checking the dump consistency...");
unsafe { DocIndexMap::from_paths("map.fst", "values.vecs").unwrap() };