From effbbc737081c794dd13051f0fd4dd6acab496cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Fri, 4 Oct 2019 13:26:33 +0200 Subject: [PATCH] Load the indexes at startup --- meilidb-core/Cargo.toml | 5 +-- meilidb-core/src/lib.rs | 83 +++++++++++++++++++++++++++++++++++++++- meilidb-core/src/main.rs | 25 +++++------- 3 files changed, 92 insertions(+), 21 deletions(-) diff --git a/meilidb-core/Cargo.toml b/meilidb-core/Cargo.toml index 7f56c0903..9e0bbd71f 100644 --- a/meilidb-core/Cargo.toml +++ b/meilidb-core/Cargo.toml @@ -9,6 +9,8 @@ bincode = "1.1.4" byteorder = "1.3.2" deunicode = "1.0.0" hashbrown = { version = "0.6.0", features = ["serde"] } +meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" } +meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" } once_cell = "1.2.0" ordered-float = { version = "1.0.2", features = ["serde"] } rkv = "0.10.2" @@ -19,9 +21,6 @@ siphasher = "0.3.0" slice-group-by = "0.2.6" zerocopy = "0.2.8" -meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" } -meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" } - [dependencies.rmp-serde] git = "https://github.com/3Hren/msgpack-rust.git" rev = "40b3d48" diff --git a/meilidb-core/src/lib.rs b/meilidb-core/src/lib.rs index f917f652b..53bf47d8d 100644 --- a/meilidb-core/src/lib.rs +++ b/meilidb-core/src/lib.rs @@ -14,9 +14,88 @@ pub mod store; pub use self::query_builder::QueryBuilder; pub use self::raw_document::RawDocument; pub use self::error::{Error, MResult}; +pub use self::number::{Number, ParseNumberError}; +pub use self::ranked_map::RankedMap; +pub use self::store::Index; -use self::number::{Number, ParseNumberError}; -use self::ranked_map::RankedMap; +use std::collections::HashMap; +use std::io; +use std::path::Path; +use std::sync::{Arc, RwLock}; + +pub struct Database { + rkv: Arc>, + main_store: rkv::SingleStore, + indexes: RwLock>, +} + +impl Database { + pub fn open_or_create(path: impl AsRef) -> io::Result { + let manager = rkv::Manager::singleton(); + let mut rkv_write = manager.write().unwrap(); + let rkv = rkv_write + .get_or_create(path.as_ref(), |path| { + let mut builder = rkv::Rkv::environment_builder(); + builder.set_max_dbs(3000).set_map_size(10 * 1024 * 1024 * 1024); // 10GB + rkv::Rkv::from_env(path, builder) + }) + .unwrap(); + + drop(rkv_write); + + let mut indexes = HashMap::new(); + let main_store; + + { + let rkv_read = rkv.read().unwrap(); + main_store = rkv_read + .open_single("indexes", rkv::store::Options::create()) + .unwrap(); + + let mut must_open = Vec::new(); + + let reader = rkv_read.read().unwrap(); + for result in main_store.iter_start(&reader).unwrap() { + let (key, _) = result.unwrap(); + if let Ok(index_name) = std::str::from_utf8(key) { + println!("{:?}", index_name); + must_open.push(index_name.to_owned()); + } + } + + drop(reader); + + for index_name in must_open { + let index = store::open(&rkv_read, &index_name).unwrap(); + indexes.insert(index_name, index); + } + } + + Ok(Database { rkv, main_store, indexes: RwLock::new(indexes) }) + } + + pub fn open_index(&self, name: impl Into) -> MResult { + let read = self.indexes.read().unwrap(); + let name = name.into(); + + match read.get(&name) { + Some(index) => Ok(*index), + None => { + drop(read); + let rkv = self.rkv.read().unwrap(); + let mut write = self.indexes.write().unwrap(); + let index = store::create(&rkv, &name).unwrap(); + + let mut writer = rkv.write().unwrap(); + let value = rkv::Value::Blob(&[]); + self.main_store.put(&mut writer, &name, &value).unwrap(); + writer.commit().unwrap(); + + Ok(*write.entry(name.clone()).or_insert(index)) + }, + } + } +} use zerocopy::{AsBytes, FromBytes}; use ::serde::{Serialize, Deserialize}; diff --git a/meilidb-core/src/main.rs b/meilidb-core/src/main.rs index 0b50f3ce7..f40353f11 100644 --- a/meilidb-core/src/main.rs +++ b/meilidb-core/src/main.rs @@ -1,22 +1,15 @@ use rkv::{Manager, Rkv, SingleStore, Value, StoreOptions}; use std::{fs, path::Path}; - -use meilidb_schema::SchemaAttr; -use meilidb_core::{store, QueryBuilder, DocumentId}; -use meilidb_core::raw_indexer::{RawIndexer, Indexed}; +use meilidb_core::{Database, QueryBuilder}; fn main() { let path = Path::new("test.rkv"); fs::create_dir_all(path).unwrap(); - // The Manager enforces that each process opens the same environment - // at most once by caching a handle to each environment that it opens. - // Use it to retrieve the handle to an opened environment—or create one - // if it hasn't already been opened: - let created_arc = Manager::singleton().write().unwrap().get_or_create(path, Rkv::new).unwrap(); - let env = created_arc.read().unwrap(); - - let index = store::create(&env, "test").unwrap(); + let database = Database::open_or_create(path).unwrap(); + let hello = database.open_index("hello").unwrap(); + let hello1 = database.open_index("hello1").unwrap(); + let hello2 = database.open_index("hello2").unwrap(); // { // let mut writer = env.write().unwrap(); @@ -44,9 +37,9 @@ fn main() { // writer.commit().unwrap(); // } - let reader = env.read().unwrap(); - let builder = QueryBuilder::new(index.main, index.postings_lists, index.synonyms); - let documents = builder.query(&reader, "oubli", 0..20).unwrap(); + // let reader = env.read().unwrap(); + // let builder = QueryBuilder::new(index.main, index.postings_lists, index.synonyms); + // let documents = builder.query(&reader, "oubli", 0..20).unwrap(); - println!("{:?}", documents); + // println!("{:?}", documents); }