diff --git a/Cargo.lock b/Cargo.lock index 5b3d81828..f7bf0807c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1658,20 +1658,29 @@ dependencies = [ name = "index-scheduler" version = "0.1.0" dependencies = [ + "actix-rt", "anyhow", "bincode", "derivative", "either", "fst", "indexmap", + "lazy_static", "log", "meilisearch-types", "milli 0.33.0", + "mockall", + "nelson", + "obkv", + "paste", "permissive-json-pointer", + "proptest", + "proptest-derive", "regex", "roaring 0.9.0", "serde", "serde_json", + "tempfile", "thiserror", "time", "uuid", diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index 4de98cdb0..536c8f0f1 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -9,17 +9,29 @@ edition = "2021" anyhow = "1.0.64" bincode = "1.3.3" derivative = "2.2.0" +either = { version = "1.6.1", features = ["serde"] } fst = "0.4.7" indexmap = { version = "1.8.0", features = ["serde-1"] } +lazy_static = "1.4.0" log = "0.4.14" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.33.0" } -permissive-json-pointer = { path = "../permissive-json-pointer" } meilisearch-types = { path = "../meilisearch-types" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.33.0" } +obkv = "0.2.0" +permissive-json-pointer = { path = "../permissive-json-pointer" } regex = "1.5.5" -either = { version = "1.6.1", features = ["serde"] } roaring = "0.9.0" serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.85", features = ["preserve_order"] } +tempfile = "3.3.0" thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } uuid = { version = "1.1.2", features = ["serde", "v4"] } + +[dev-dependencies] +actix-rt = "2.7.0" +meilisearch-types = { path = "../meilisearch-types", features = ["test-traits"] } +mockall = "0.11.0" +nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} +paste = "1.0.6" +proptest = "1.0.0" +proptest-derive = "0.3.0" diff --git a/index-scheduler/src/error.rs b/index-scheduler/src/error.rs index 212a9b04d..563e5a7d3 100644 --- a/index-scheduler/src/error.rs +++ b/index-scheduler/src/error.rs @@ -1,6 +1,8 @@ use milli::heed; use thiserror::Error; +use crate::index; + #[derive(Error, Debug)] pub enum Error { #[error("Index `{}` not found", .0)] @@ -13,6 +15,8 @@ pub enum Error { Heed(#[from] heed::Error), #[error(transparent)] Milli(#[from] milli::Error), + #[error("{0}")] + IndexError(#[from] index::error::IndexError), #[error(transparent)] Anyhow(#[from] anyhow::Error), diff --git a/index-scheduler/src/index/error.rs b/index-scheduler/src/index/error.rs index f795ceaa4..ee0ff8abf 100644 --- a/index-scheduler/src/index/error.rs +++ b/index-scheduler/src/index/error.rs @@ -4,7 +4,7 @@ use meilisearch_types::error::{Code, ErrorCode}; use meilisearch_types::internal_error; use serde_json::Value; -use crate::{error::MilliError, update_file_store}; +use crate::update_file_store; pub type Result = std::result::Result; @@ -29,6 +29,7 @@ internal_error!( milli::documents::Error ); +/* impl ErrorCode for IndexError { fn error_code(&self) -> Code { match self { @@ -39,6 +40,7 @@ impl ErrorCode for IndexError { } } } +*/ impl From for IndexError { fn from(error: milli::UserError) -> IndexError { diff --git a/index-scheduler/src/index/index.rs b/index-scheduler/src/index/index.rs index f32c842ed..fbb76bbdf 100644 --- a/index-scheduler/src/index/index.rs +++ b/index-scheduler/src/index/index.rs @@ -12,7 +12,6 @@ use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use time::OffsetDateTime; -use uuid::Uuid; use crate::index::search::DEFAULT_PAGINATION_MAX_TOTAL_HITS; @@ -298,7 +297,7 @@ impl Index { } pub fn size(&self) -> Result { - self.inner.on_disk_size() + Ok(self.inner.on_disk_size()?) } pub fn snapshot(&self, path: impl AsRef) -> Result<()> { diff --git a/index-scheduler/src/index/mod.rs b/index-scheduler/src/index/mod.rs index 98c25366d..505417dca 100644 --- a/index-scheduler/src/index/mod.rs +++ b/index-scheduler/src/index/mod.rs @@ -4,7 +4,7 @@ pub use search::{ }; pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked}; -mod dump; +// mod dump; pub mod error; mod search; pub mod updates; @@ -52,14 +52,15 @@ pub mod test { pub fn open( path: impl AsRef, + name: String, size: usize, - uuid: Uuid, update_handler: Arc, ) -> Result { - let index = Index::open(path, size, uuid, update_handler)?; + let index = Index::open(path, name, size, update_handler)?; Ok(Self::Real(index)) } + /* pub fn load_dump( src: impl AsRef, dst: impl AsRef, @@ -68,13 +69,7 @@ pub mod test { ) -> anyhow::Result<()> { Index::load_dump(src, dst, size, update_handler) } - - pub fn uuid(&self) -> Uuid { - match self { - MockIndex::Real(index) => index.uuid(), - MockIndex::Mock(m) => unsafe { m.get("uuid").call(()) }, - } - } + */ pub fn stats(&self) -> Result { match self { @@ -121,7 +116,7 @@ pub mod test { } } - pub fn size(&self) -> u64 { + pub fn size(&self) -> Result { match self { MockIndex::Real(index) => index.size(), MockIndex::Mock(_) => todo!(), @@ -149,12 +144,14 @@ pub mod test { } } + /* pub fn dump(&self, path: impl AsRef) -> Result<()> { match self { MockIndex::Real(index) => index.dump(path), MockIndex::Mock(m) => unsafe { m.get("dump").call(path.as_ref()) }, } } + */ pub fn update_documents( &self, diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 61ddcb882..485946014 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -240,7 +240,7 @@ impl IndexScheduler { new_name, } => { if self.available_index.get(wtxn, &new_name)?.unwrap_or(false) { - return Err(Error::IndexAlreadyExists); + return Err(Error::IndexAlreadyExists(new_name.to_string())); } todo!("wait for @guigui insight"); } @@ -275,7 +275,10 @@ impl IndexScheduler { return Err(Error::IndexNotFound(rhs.to_string())); } - let index_map = self.index_map.write()?; + let index_map = self + .index_map + .write() + .map_err(|_| Error::CorruptedTaskQueue)?; // index_map.remove. }