Load the indexes at startup

meilidb-core/Cargo.toml

@@ -9,6 +9,8 @@ bincode = "1.1.4"
 byteorder = "1.3.2"
 deunicode = "1.0.0"
 hashbrown = { version = "0.6.0", features = ["serde"] }
+meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" }
+meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" }
 once_cell = "1.2.0"
 ordered-float = { version = "1.0.2", features = ["serde"] }
 rkv = "0.10.2"
@@ -19,9 +21,6 @@ siphasher = "0.3.0"
 slice-group-by = "0.2.6"
 zerocopy = "0.2.8"
 
-meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" }
-meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" }
-
 [dependencies.rmp-serde]
 git = "https://github.com/3Hren/msgpack-rust.git"
 rev = "40b3d48"

meilidb-core/src/lib.rs

@@ -14,9 +14,88 @@ pub mod store;
 pub use self::query_builder::QueryBuilder;
 pub use self::raw_document::RawDocument;
 pub use self::error::{Error, MResult};
+pub use self::number::{Number, ParseNumberError};
+pub use self::ranked_map::RankedMap;
+pub use self::store::Index;
 
-use self::number::{Number, ParseNumberError};
+use std::collections::HashMap;
-use self::ranked_map::RankedMap;
+use std::io;
+use std::path::Path;
+use std::sync::{Arc, RwLock};
+
+pub struct Database {
+    rkv: Arc<RwLock<rkv::Rkv>>,
+    main_store: rkv::SingleStore,
+    indexes: RwLock<HashMap<String, Index>>,
+}
+
+impl Database {
+    pub fn open_or_create(path: impl AsRef<Path>) -> io::Result<Database> {
+        let manager = rkv::Manager::singleton();
+        let mut rkv_write = manager.write().unwrap();
+        let rkv = rkv_write
+            .get_or_create(path.as_ref(), |path| {
+                let mut builder = rkv::Rkv::environment_builder();
+                builder.set_max_dbs(3000).set_map_size(10 * 1024 * 1024 * 1024); // 10GB
+                rkv::Rkv::from_env(path, builder)
+            })
+            .unwrap();
+
+        drop(rkv_write);
+
+        let mut indexes = HashMap::new();
+        let main_store;
+
+        {
+            let rkv_read = rkv.read().unwrap();
+            main_store = rkv_read
+                .open_single("indexes", rkv::store::Options::create())
+                .unwrap();
+
+            let mut must_open = Vec::new();
+
+            let reader = rkv_read.read().unwrap();
+            for result in main_store.iter_start(&reader).unwrap() {
+                let (key, _) = result.unwrap();
+                if let Ok(index_name) = std::str::from_utf8(key) {
+                    println!("{:?}", index_name);
+                    must_open.push(index_name.to_owned());
+                }
+            }
+
+            drop(reader);
+
+            for index_name in must_open {
+                let index = store::open(&rkv_read, &index_name).unwrap();
+                indexes.insert(index_name, index);
+            }
+        }
+
+        Ok(Database { rkv, main_store, indexes: RwLock::new(indexes) })
+    }
+
+    pub fn open_index(&self, name: impl Into<String>) -> MResult<Index> {
+        let read = self.indexes.read().unwrap();
+        let name = name.into();
+
+        match read.get(&name) {
+            Some(index) => Ok(*index),
+            None => {
+                drop(read);
+                let rkv = self.rkv.read().unwrap();
+                let mut write = self.indexes.write().unwrap();
+                let index = store::create(&rkv, &name).unwrap();
+
+                let mut writer = rkv.write().unwrap();
+                let value = rkv::Value::Blob(&[]);
+                self.main_store.put(&mut writer, &name, &value).unwrap();
+                writer.commit().unwrap();
+
+                Ok(*write.entry(name.clone()).or_insert(index))
+            },
+        }
+    }
+}
 
 use zerocopy::{AsBytes, FromBytes};
 use ::serde::{Serialize, Deserialize};

meilidb-core/src/main.rs

@@ -1,22 +1,15 @@
 use rkv::{Manager, Rkv, SingleStore, Value, StoreOptions};
 use std::{fs, path::Path};
-
+use meilidb_core::{Database, QueryBuilder};
-use meilidb_schema::SchemaAttr;
-use meilidb_core::{store, QueryBuilder, DocumentId};
-use meilidb_core::raw_indexer::{RawIndexer, Indexed};
 
 fn main() {
     let path = Path::new("test.rkv");
     fs::create_dir_all(path).unwrap();
 
-    // The Manager enforces that each process opens the same environment
+    let database = Database::open_or_create(path).unwrap();
-    // at most once by caching a handle to each environment that it opens.
+    let hello = database.open_index("hello").unwrap();
-    // Use it to retrieve the handle to an opened environment—or create one
+    let hello1 = database.open_index("hello1").unwrap();
-    // if it hasn't already been opened:
+    let hello2 = database.open_index("hello2").unwrap();
-    let created_arc = Manager::singleton().write().unwrap().get_or_create(path, Rkv::new).unwrap();
-    let env = created_arc.read().unwrap();
-
-    let index = store::create(&env, "test").unwrap();
 
     // {
     //     let mut writer = env.write().unwrap();
@@ -44,9 +37,9 @@ fn main() {
     //     writer.commit().unwrap();
     // }
 
-    let reader = env.read().unwrap();
+    // let reader = env.read().unwrap();
-    let builder = QueryBuilder::new(index.main, index.postings_lists, index.synonyms);
+    // let builder = QueryBuilder::new(index.main, index.postings_lists, index.synonyms);
-    let documents = builder.query(&reader, "oubli", 0..20).unwrap();
+    // let documents = builder.query(&reader, "oubli", 0..20).unwrap();
 
-    println!("{:?}", documents);
+    // println!("{:?}", documents);
 }