Merge #144 #173

144: Concurrent update run loop (refactor) r=MarinPostma a=MarinPostma

This PR allows multiple request to the update store to be performed concurently (i.e, one can list updates while an updates in being written to the update store).


173: Convert UpdateStatus to legacy meilisearch format r=MarinPostma a=MarinPostma

Returns the update statuses with the same format as legacy meilisearch.

The number of documents in a document addition/deletion is not known before processing, so it is only returned when the update is `processed`.

close #78 

associated milli PR: https://github.com/meilisearch/milli/pull/178


Co-authored-by: marin postma <postma.marin@protonmail.com>
Co-authored-by: Marin Postma <postma.marin@protonmail.com>

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -4,6 +4,8 @@ use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
+use async_stream::stream;
+use futures::StreamExt;
 use log::info;
 use oxidized_json_checker::JsonChecker;
 use tokio::fs;
@@ -18,7 +20,7 @@ use crate::index_controller::{UpdateMeta, UpdateStatus};
 pub struct UpdateActor<D, I> {
     path: PathBuf,
     store: Arc<UpdateStore>,
-    inbox: mpsc::Receiver<UpdateMsg<D>>,
+    inbox: Option<mpsc::Receiver<UpdateMsg<D>>>,
     index_handle: I,
     must_exit: Arc<AtomicBool>,
 }
@@ -45,7 +47,7 @@ where
 
         let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?;
         std::fs::create_dir_all(path.join("update_files"))?;
-
+        let inbox = Some(inbox);
         Ok(Self {
             path,
             store,
@@ -60,43 +62,59 @@ where
 
         info!("Started update actor.");
 
-        loop {
-            let msg = self.inbox.recv().await;
+        let mut inbox = self
+            .inbox
+            .take()
+            .expect("A receiver should be present by now.");
 
-            if self.must_exit.load(std::sync::atomic::Ordering::Relaxed) {
-                break;
-            }
+        let must_exit = self.must_exit.clone();
+        let stream = stream! {
+            loop {
+                let msg = inbox.recv().await;
 
-            match msg {
-                Some(Update {
-                    uuid,
-                    meta,
-                    data,
-                    ret,
-                }) => {
-                    let _ = ret.send(self.handle_update(uuid, meta, data).await);
+                if must_exit.load(std::sync::atomic::Ordering::Relaxed) {
+                    break;
                 }
-                Some(ListUpdates { uuid, ret }) => {
-                    let _ = ret.send(self.handle_list_updates(uuid).await);
+
+                match msg {
+                    Some(msg) => yield msg,
+                    None => break,
                 }
-                Some(GetUpdate { uuid, ret, id }) => {
-                    let _ = ret.send(self.handle_get_update(uuid, id).await);
-                }
-                Some(Delete { uuid, ret }) => {
-                    let _ = ret.send(self.handle_delete(uuid).await);
-                }
-                Some(Snapshot { uuids, path, ret }) => {
-                    let _ = ret.send(self.handle_snapshot(uuids, path).await);
-                }
-                Some(Dump { uuids, path, ret }) => {
-                    let _ = ret.send(self.handle_dump(uuids, path).await);
-                }
-                Some(GetInfo { ret }) => {
-                    let _ = ret.send(self.handle_get_info().await);
-                }
-                None => break,
             }
-        }
+        };
+
+        stream
+            .for_each_concurrent(Some(10), |msg| async {
+                match msg {
+                    Update {
+                        uuid,
+                        meta,
+                        data,
+                        ret,
+                    } => {
+                        let _ = ret.send(self.handle_update(uuid, meta, data).await);
+                    }
+                    ListUpdates { uuid, ret } => {
+                        let _ = ret.send(self.handle_list_updates(uuid).await);
+                    }
+                    GetUpdate { uuid, ret, id } => {
+                        let _ = ret.send(self.handle_get_update(uuid, id).await);
+                    }
+                    Delete { uuid, ret } => {
+                        let _ = ret.send(self.handle_delete(uuid).await);
+                    }
+                    Snapshot { uuids, path, ret } => {
+                        let _ = ret.send(self.handle_snapshot(uuids, path).await);
+                    }
+                    GetInfo { ret } => {
+                        let _ = ret.send(self.handle_get_info().await);
+                    }
+                    Dump { uuids, path, ret } => {
+                        let _ = ret.send(self.handle_dump(uuids, path).await);
+                    }
+                }
+            })
+            .await;
     }
 
     async fn handle_update(

meilisearch-http/src/index_controller/updates.rs

@@ -84,6 +84,10 @@ impl Processed {
     pub fn id(&self) -> u64 {
         self.from.id()
     }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -132,21 +136,29 @@ impl Aborted {
     pub fn id(&self) -> u64 {
         self.from.id()
     }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct Failed {
     #[serde(flatten)]
-    from: Processing,
-    error: UpdateError,
-    failed_at: DateTime<Utc>,
+    pub from: Processing,
+    pub error: UpdateError,
+    pub failed_at: DateTime<Utc>,
 }
 
 impl Failed {
     pub fn id(&self) -> u64 {
         self.from.id()
     }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -170,6 +182,16 @@ impl UpdateStatus {
         }
     }
 
+    pub fn meta(&self) -> &UpdateMeta {
+        match self {
+            UpdateStatus::Processing(u) => u.meta(),
+            UpdateStatus::Enqueued(u) => u.meta(),
+            UpdateStatus::Processed(u) => u.meta(),
+            UpdateStatus::Aborted(u) => u.meta(),
+            UpdateStatus::Failed(u) => u.meta(),
+        }
+    }
+
     pub fn processed(&self) -> Option<&Processed> {
         match self {
             UpdateStatus::Processed(p) => Some(p),