Introduce a background thread that manage updates to do

meilidb-core/src/store/mod.rs

@@ -14,6 +14,8 @@ pub use self::synonyms::Synonyms;
 pub use self::updates::Updates;
 pub use self::updates_results::UpdatesResults;
 
+use crate::update;
+
 fn aligned_to(bytes: &[u8], align: usize) -> bool {
     (bytes as *const _ as *const () as usize) % align == 0
 }
@@ -46,31 +48,62 @@ fn updates_results_name(name: &str) -> String {
     format!("store-{}-updates-results", name)
 }
 
-#[derive(Copy, Clone)]
+#[derive(Clone)]
 pub struct Index {
     pub main: Main,
     pub postings_lists: PostingsLists,
     pub documents_fields: DocumentsFields,
     pub synonyms: Synonyms,
     pub docs_words: DocsWords,
+
     pub updates: Updates,
     pub updates_results: UpdatesResults,
+    updates_notifier: crossbeam_channel::Sender<()>,
 }
 
-pub fn create(env: &rkv::Rkv, name: &str) -> Result<Index, rkv::StoreError> {
-    open_options(env, name, rkv::StoreOptions::create())
+impl Index {
+    pub fn documents_addition<D>(&self) -> update::DocumentsAddition<D> {
+        update::DocumentsAddition::new(
+            self.updates,
+            self.updates_results,
+            self.updates_notifier.clone(),
+        )
+    }
+
+    pub fn documents_deletion<D>(&self) -> update::DocumentsDeletion {
+        update::DocumentsDeletion::new(
+            self.updates,
+            self.updates_results,
+            self.updates_notifier.clone(),
+        )
+    }
 }
 
-pub fn open(env: &rkv::Rkv, name: &str) -> Result<Index, rkv::StoreError> {
+pub fn create(
+    env: &rkv::Rkv,
+    name: &str,
+    updates_notifier: crossbeam_channel::Sender<()>,
+) -> Result<Index, rkv::StoreError>
+{
+    open_options(env, name, rkv::StoreOptions::create(), updates_notifier)
+}
+
+pub fn open(
+    env: &rkv::Rkv,
+    name: &str,
+    updates_notifier: crossbeam_channel::Sender<()>,
+) -> Result<Index, rkv::StoreError>
+{
     let mut options = rkv::StoreOptions::default();
     options.create = false;
-    open_options(env, name, options)
+    open_options(env, name, options, updates_notifier)
 }
 
 fn open_options(
     env: &rkv::Rkv,
     name: &str,
     options: rkv::StoreOptions,
+    updates_notifier: crossbeam_channel::Sender<()>,
 ) -> Result<Index, rkv::StoreError>
 {
     // create all the store names
@@ -99,5 +132,6 @@ fn open_options(
         docs_words: DocsWords { docs_words },
         updates: Updates { updates },
         updates_results: UpdatesResults { updates_results },
+        updates_notifier,
     })
 }

meilidb-core/src/store/updates.rs

@@ -10,7 +10,7 @@ pub struct Updates {
 impl Updates {
     // TODO we should use the MDB_LAST op but
     //      it is not exposed by the rkv library
-    fn last_update_id<'a>(
+    pub fn last_update_id<'a>(
         &self,
         reader: &'a impl rkv::Readable,
     ) -> Result<Option<(u64, Option<Value<'a>>)>, rkv::StoreError>
@@ -60,21 +60,18 @@ impl Updates {
         self.updates.get(reader, update_id_bytes).map(|v| v.is_some())
     }
 
-    pub fn push_back(
+    pub fn put_update(
         &self,
         writer: &mut rkv::Writer,
+        update_id: u64,
         update: &Update,
-    ) -> MResult<u64>
+    ) -> MResult<()>
     {
-        let last_update_id = self.last_update_id(writer)?;
-        let last_update_id = last_update_id.map_or(0, |(n, _)| n + 1);
-        let last_update_id_bytes = last_update_id.to_be_bytes();
-
+        let update_id_bytes = update_id.to_be_bytes();
         let update = rmp_serde::to_vec_named(&update)?;
         let blob = Value::Blob(&update);
-        self.updates.put(writer, last_update_id_bytes, &blob)?;
-
-        Ok(last_update_id)
+        self.updates.put(writer, update_id_bytes, &blob)?;
+        Ok(())
     }
 
     pub fn pop_front(
@@ -82,20 +79,20 @@ impl Updates {
         writer: &mut rkv::Writer,
     ) -> MResult<Option<(u64, Update)>>
     {
-        let (last_id, last_data) = match self.first_update_id(writer)? {
+        let (first_id, first_data) = match self.first_update_id(writer)? {
             Some(entry) => entry,
             None => return Ok(None),
         };
 
-        match last_data {
+        match first_data {
             Some(Value::Blob(bytes)) => {
                 let update = rmp_serde::from_read_ref(&bytes)?;
 
                 // remove it from the database now
-                let last_id_bytes = last_id.to_be_bytes();
-                self.updates.delete(writer, last_id_bytes)?;
+                let first_id_bytes = first_id.to_be_bytes();
+                self.updates.delete(writer, first_id_bytes)?;
 
-                Ok(Some((last_id, update)))
+                Ok(Some((first_id, update)))
             },
             Some(value) => panic!("invalid type {:?}", value),
             None => Ok(None),

meilidb-core/src/store/updates_results.rs

@@ -1,3 +1,4 @@
+use std::convert::TryInto;
 use rkv::Value;
 use crate::{update::UpdateResult, MResult};
 
@@ -7,6 +8,31 @@ pub struct UpdatesResults {
 }
 
 impl UpdatesResults {
+    // TODO we should use the MDB_LAST op but
+    //      it is not exposed by the rkv library
+    pub fn last_update_id<'a>(
+        &self,
+        reader: &'a impl rkv::Readable,
+    ) -> Result<Option<(u64, Option<Value<'a>>)>, rkv::StoreError>
+    {
+        let mut last = None;
+        let iter = self.updates_results.iter_start(reader)?;
+        for result in iter {
+            let (key, data) = result?;
+            last = Some((key, data));
+        }
+
+        let (last_key, last_data) = match last {
+            Some(entry) => entry,
+            None => return Ok(None),
+        };
+
+        let array = last_key.try_into().unwrap();
+        let number = u64::from_be_bytes(array);
+
+        Ok(Some((number, last_data)))
+    }
+
     pub fn put_update_result(
         &self,
         writer: &mut rkv::Writer,