fix most of the index module

Cargo.lock

@@ -1658,20 +1658,29 @@ dependencies = [
 name = "index-scheduler"
 version = "0.1.0"
 dependencies = [
+ "actix-rt",
  "anyhow",
  "bincode",
  "derivative",
  "either",
  "fst",
  "indexmap",
+ "lazy_static",
  "log",
  "meilisearch-types",
  "milli 0.33.0",
+ "mockall",
+ "nelson",
+ "obkv",
+ "paste",
  "permissive-json-pointer",
+ "proptest",
+ "proptest-derive",
  "regex",
  "roaring 0.9.0",
  "serde",
  "serde_json",
+ "tempfile",
  "thiserror",
  "time",
  "uuid",

index-scheduler/Cargo.toml

@@ -9,17 +9,29 @@ edition = "2021"
 anyhow = "1.0.64"
 bincode = "1.3.3"
 derivative = "2.2.0"
+either = { version = "1.6.1", features = ["serde"] }
 fst = "0.4.7"
 indexmap = { version = "1.8.0", features = ["serde-1"] }
+lazy_static = "1.4.0"
 log = "0.4.14"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.33.0" }
-permissive-json-pointer = { path = "../permissive-json-pointer" }
 meilisearch-types = { path = "../meilisearch-types" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.33.0" }
+obkv = "0.2.0"
+permissive-json-pointer = { path = "../permissive-json-pointer" }
 regex = "1.5.5"
-either = { version = "1.6.1", features = ["serde"] }
 roaring = "0.9.0"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { version = "1.0.85", features = ["preserve_order"] }
+tempfile = "3.3.0"
 thiserror = "1.0.30"
 time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 uuid = { version = "1.1.2", features = ["serde", "v4"] }
+
+[dev-dependencies]
+actix-rt = "2.7.0"
+meilisearch-types = { path = "../meilisearch-types", features = ["test-traits"] }
+mockall = "0.11.0"
+nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}
+paste = "1.0.6"
+proptest = "1.0.0"
+proptest-derive = "0.3.0"

index-scheduler/src/error.rs

@@ -1,6 +1,8 @@
 use milli::heed;
 use thiserror::Error;
 
+use crate::index;
+
 #[derive(Error, Debug)]
 pub enum Error {
     #[error("Index `{}` not found", .0)]
@@ -13,6 +15,8 @@ pub enum Error {
     Heed(#[from] heed::Error),
     #[error(transparent)]
     Milli(#[from] milli::Error),
+    #[error("{0}")]
+    IndexError(#[from] index::error::IndexError),
 
     #[error(transparent)]
     Anyhow(#[from] anyhow::Error),

index-scheduler/src/index/error.rs

@@ -4,7 +4,7 @@ use meilisearch_types::error::{Code, ErrorCode};
 use meilisearch_types::internal_error;
 use serde_json::Value;
 
-use crate::{error::MilliError, update_file_store};
+use crate::update_file_store;
 
 pub type Result<T> = std::result::Result<T, IndexError>;
 
@@ -29,6 +29,7 @@ internal_error!(
     milli::documents::Error
 );
 
+/*
 impl ErrorCode for IndexError {
     fn error_code(&self) -> Code {
         match self {
@@ -39,6 +40,7 @@ impl ErrorCode for IndexError {
         }
     }
 }
+*/
 
 impl From<milli::UserError> for IndexError {
     fn from(error: milli::UserError) -> IndexError {

index-scheduler/src/index/index.rs

@@ -12,7 +12,6 @@ use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET};
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 use time::OffsetDateTime;
-use uuid::Uuid;
 
 use crate::index::search::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
 
@@ -298,7 +297,7 @@ impl Index {
     }
 
     pub fn size(&self) -> Result<u64> {
-        self.inner.on_disk_size()
+        Ok(self.inner.on_disk_size()?)
     }
 
     pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {

index-scheduler/src/index/mod.rs

@@ -4,7 +4,7 @@ pub use search::{
 };
 pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
 
-mod dump;
+// mod dump;
 pub mod error;
 mod search;
 pub mod updates;
@@ -52,14 +52,15 @@ pub mod test {
 
         pub fn open(
             path: impl AsRef<Path>,
+            name: String,
             size: usize,
-            uuid: Uuid,
             update_handler: Arc<IndexerConfig>,
         ) -> Result<Self> {
-            let index = Index::open(path, size, uuid, update_handler)?;
+            let index = Index::open(path, name, size, update_handler)?;
             Ok(Self::Real(index))
         }
 
+        /*
         pub fn load_dump(
             src: impl AsRef<Path>,
             dst: impl AsRef<Path>,
@@ -68,13 +69,7 @@ pub mod test {
         ) -> anyhow::Result<()> {
             Index::load_dump(src, dst, size, update_handler)
         }
-
-        pub fn uuid(&self) -> Uuid {
-            match self {
-                MockIndex::Real(index) => index.uuid(),
-                MockIndex::Mock(m) => unsafe { m.get("uuid").call(()) },
-            }
-        }
+        */
 
         pub fn stats(&self) -> Result<IndexStats> {
             match self {
@@ -121,7 +116,7 @@ pub mod test {
             }
         }
 
-        pub fn size(&self) -> u64 {
+        pub fn size(&self) -> Result<u64> {
             match self {
                 MockIndex::Real(index) => index.size(),
                 MockIndex::Mock(_) => todo!(),
@@ -149,12 +144,14 @@ pub mod test {
             }
         }
 
+        /*
         pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
             match self {
                 MockIndex::Real(index) => index.dump(path),
                 MockIndex::Mock(m) => unsafe { m.get("dump").call(path.as_ref()) },
             }
         }
+        */
 
         pub fn update_documents(
             &self,

index-scheduler/src/lib.rs

@@ -240,7 +240,7 @@ impl IndexScheduler {
                     new_name,
                 } => {
                     if self.available_index.get(wtxn, &new_name)?.unwrap_or(false) {
-                        return Err(Error::IndexAlreadyExists);
+                        return Err(Error::IndexAlreadyExists(new_name.to_string()));
                     }
                     todo!("wait for @guigui insight");
                 }
@@ -275,7 +275,10 @@ impl IndexScheduler {
                         return Err(Error::IndexNotFound(rhs.to_string()));
                     }
 
-                    let index_map = self.index_map.write()?;
+                    let index_map = self
+                        .index_map
+                        .write()
+                        .map_err(|_| Error::CorruptedTaskQueue)?;
 
                     // index_map.remove.
                 }