diff --git a/Cargo.lock b/Cargo.lock index 40528367e..e0457b4d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,7 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", + "serde", "synchronoise", "url", "zerocopy", @@ -1636,6 +1637,7 @@ dependencies = [ "memmap", "milli", "mime", + "obkv", "once_cell", "page_size", "rand 0.7.3", diff --git a/Cargo.toml b/Cargo.toml index 47bbc09df..c06847253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } +heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -60,6 +60,7 @@ walkdir = "2.3.1" whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" +obkv = "0.1.1" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 944690e0c..4600531bf 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,11 +3,9 @@ mod updates; pub use search::{SearchQuery, SearchResult}; -use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; -use milli::Index; use sha2::Digest; use crate::{option::Opt, updates::Settings}; @@ -29,8 +27,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub indexes: Arc, - pub update_queue: Arc, + pub indexes: Arc>, api_keys: ApiKeys, options: Opt, } diff --git a/src/data/search.rs b/src/data/search.rs index 8c8e0da69..69029d8a9 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -19,18 +19,18 @@ const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { - q: Option, - offset: Option, + pub q: Option, + pub offset: Option, #[serde(default = "default_search_limit")] - limit: usize, - attributes_to_retrieve: Option>, - attributes_to_crop: Option>, - crop_length: Option, - attributes_to_highlight: Option>, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option>, + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + pub crop_length: Option, + pub attributes_to_highlight: Option>, + pub filters: Option, + pub matches: Option, + pub facet_filters: Option, + pub facets_distribution: Option>, } #[derive(Serialize)] diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 25e5d4787..a3b29879a 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,25 +1,26 @@ use std::fs::File; use std::io::{Read, Write}; -use std::path::Path; -use std::ops::Deref; +use std::path::{Path, PathBuf}; use anyhow::Result; use chrono::{DateTime, Utc}; use dashmap::DashMap; +use dashmap::mapref::one::Ref; use heed::types::{Str, SerdeBincode}; use heed::{EnvOpenOptions, Env, Database}; -use milli::{Index, FieldsIdsMap}; +use milli::{Index, FieldsIdsMap, SearchResult, FieldId}; use serde::{Serialize, Deserialize}; -use crate::data::{SearchQuery, SearchResult}; +use crate::data::SearchQuery; const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; const INDEXES_DB_NAME: &str = "indexes_db"; -trait UpdateStore {} +pub trait UpdateStore {} pub struct IndexController { + path: PathBuf, update_store: U, env: Env, indexes_db: Database>, @@ -34,7 +35,8 @@ struct IndexControllerMeta { impl IndexControllerMeta { fn from_path(path: impl AsRef) -> Result> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { let mut file = File::open(path)?; let mut buffer = Vec::new(); @@ -47,7 +49,8 @@ impl IndexControllerMeta { } fn to_path(self, path: impl AsRef) -> Result<()> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { Err(anyhow::anyhow!("Index controller metadata already exists")) } else { @@ -67,39 +70,24 @@ struct IndexMetadata { } impl IndexMetadata { - fn open_index(&self, path: impl AsRef) -> Result { - let path = path.as_ref().to_path_buf().push("indexes").push(&self.id); - Ok(Index::new(self.options, path)?) + fn open_index(&self, path: impl AsRef) -> Result { + // create a path in the form "db_path/indexes/index_id" + let mut path = path.as_ref().to_path_buf(); + path.push("indexes"); + path.push(&self.id); + Ok(Index::new(self.open_options, path)?) } } struct IndexView<'a, U> { txn: heed::RoTxn<'a>, - index: &'a Index, + index: Ref<'a, String, Index>, update_store: &'a U, } -struct IndexViewMut<'a, U> { - txn: heed::RwTxn<'a, 'a>, - index: &'a Index, - update_store: &'a U, -} - -impl<'a, U> Deref for IndexViewMut<'a, U> { - type Target = IndexView<'a, U>; - - fn deref(&self) -> &Self::Target { - IndexView { - txn: *self.txn, - index: self.index, - update_store: self.update_store, - } - } -} - impl<'a, U: UpdateStore> IndexView<'a, U> { pub fn search(&self, search_query: SearchQuery) -> Result { - let mut search = self.index.search(self.txn); + let mut search = self.index.search(&self.txn); if let Some(query) = &search_query.q { search.query(query); } @@ -115,15 +103,15 @@ impl<'a, U: UpdateStore> IndexView<'a, U> { } pub fn fields_ids_map(&self) -> Result { - self.index.fields_ids_map(self.txn) + Ok(self.index.fields_ids_map(&self.txn)?) } - pub fn fields_displayed_fields_ids(&self) -> Result { - self.index.fields_displayed_fields_ids(self.txn) + pub fn fields_displayed_fields_ids(&self) -> Result>> { + Ok(self.index.displayed_fields_ids(&self.txn)?) } - pub fn documents(&self, ids: &[u32]) -> Result> { - self.index.documents(self.txn, ids) + pub fn documents(&self, ids: Vec) -> Result)>> { + Ok(self.index.documents(&self.txn, ids)?) } } @@ -133,28 +121,29 @@ impl IndexController { pub fn new(path: impl AsRef, update_store: U) -> Result { // If index controller metadata is present, we return the env, otherwise, we create a new // metadata from scratch before returning a new env. - let env = match IndexControllerMeta::from_path(path)? { + let path = path.as_ref().to_path_buf(); + let env = match IndexControllerMeta::from_path(&path)? { Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, None => { let open_options = EnvOpenOptions::new() .map_size(page_size::get() * 1000); let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; let created_at = Utc::now(); - let meta = IndexControllerMeta { open_options, created_at }; + let meta = IndexControllerMeta { open_options: open_options.clone(), created_at }; meta.to_path(path)?; env } }; let indexes = DashMap::new(); - let indexes_db = match env.open_database(INDEXES_DB_NAME)? { + let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { Some(indexes_db) => indexes_db, - None => env.create_database(INDEXES_DB_NAME)?, + None => env.create_database(Some(INDEXES_DB_NAME))?, }; - Ok(Self { env, indexes, indexes_db, update_store }) + Ok(Self { env, indexes, indexes_db, update_store, path }) } - pub fn get_or_create>(&mut self, name: S) -> Result> { + pub fn get_or_create>(&mut self, name: S) -> Result> { todo!() } @@ -162,19 +151,26 @@ impl IndexController { /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check /// for metadata to launch the index. pub fn get>(&self, name: S) -> Result>> { + let update_store = &self.update_store; match self.indexes.get(name.as_ref()) { Some(index) => { let txn = index.read_txn()?; - let update_store = &self.update_store; Ok(Some(IndexView { index, update_store, txn })) } None => { let txn = self.env.read_txn()?; match self.indexes_db.get(&txn, name.as_ref())? { Some(meta) => { - let index = meta.open_index()?; + let index = meta.open_index(self.path)?; self.indexes.insert(name.as_ref().to_owned(), index); - Ok(self.indexes.get(name.as_ref())) + // TODO: create index view + match self.indexes.get(name.as_ref()) { + Some(index) => { + let txn = index.read_txn()?; + Ok(Some(IndexView { index, txn, update_store })) + } + None => Ok(None) + } } None => Ok(None) } @@ -182,7 +178,7 @@ impl IndexController { } } - pub fn get_mut>(&self, name: S) -> Result>> { + pub fn get_mut>(&self, name: S) -> Result>> { todo!() } diff --git a/src/updates/mod.rs b/src/updates/mod.rs index d92a3b16f..faefe7804 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -55,6 +55,8 @@ pub struct UpdateQueue { inner: Arc>, } +impl crate::index_controller::UpdateStore for UpdateQueue {} + impl Deref for UpdateQueue { type Target = Arc>;