handle dump

meilisearch-lib/src/tasks/batch_handlers/dump_handler.rs

@@ -1,101 +1,34 @@
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use log::{error, trace};
-use time::{macros::format_description, OffsetDateTime};
-
-use crate::dump::DumpJob;
+use crate::dump::DumpHandler;
 use crate::index_resolver::index_store::IndexStore;
 use crate::index_resolver::meta_store::IndexMetaStore;
-use crate::index_resolver::IndexResolver;
 use crate::tasks::batch::{Batch, BatchContent};
+use crate::tasks::task::{Task, TaskContent, TaskEvent, TaskResult};
 use crate::tasks::BatchHandler;
-use crate::update_file_store::UpdateFileStore;
-
-pub struct DumpHandler<U, I> {
-    update_file_store: UpdateFileStore,
-    index_resolver: Arc<IndexResolver<U, I>>,
-    dump_path: PathBuf,
-    db_path: PathBuf,
-    update_db_size: usize,
-    index_db_size: usize,
-}
-
-/// Generate uid from creation date
-fn generate_uid() -> String {
-    OffsetDateTime::now_utc()
-        .format(format_description!(
-            "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
-        ))
-        .unwrap()
-}
-
-impl<U, I> DumpHandler<U, I>
-where
-    U: IndexMetaStore + Send + Sync + 'static,
-    I: IndexStore + Send + Sync + 'static,
-{
-    pub fn new(
-        update_file_store: UpdateFileStore,
-        dump_path: impl AsRef<Path>,
-        db_path: impl AsRef<Path>,
-        index_db_size: usize,
-        update_db_size: usize,
-        index_resolver: Arc<IndexResolver<U, I>>,
-    ) -> Self {
-        Self {
-            update_file_store,
-            dump_path: dump_path.as_ref().into(),
-            db_path: db_path.as_ref().into(),
-            index_db_size,
-            update_db_size,
-            index_resolver,
-        }
-    }
-
-    async fn create_dump(&self) {
-        let uid = generate_uid();
-
-        let task = DumpJob {
-            dump_path: self.dump_path.clone(),
-            db_path: self.db_path.clone(),
-            update_file_store: self.update_file_store.clone(),
-            uid: uid.clone(),
-            update_db_size: self.update_db_size,
-            index_db_size: self.index_db_size,
-            index_resolver: self.index_resolver.clone(),
-        };
-
-        let task_result = tokio::task::spawn_local(task.run()).await;
-
-        match task_result {
-            Ok(Ok(())) => {
-                trace!("Dump succeed");
-            }
-            Ok(Err(e)) => {
-                error!("Dump failed: {}", e);
-            }
-            Err(_) => {
-                error!("Dump panicked. Dump status set to failed");
-            }
-        };
-    }
-}
 
 #[async_trait::async_trait]
 impl<U, I> BatchHandler for DumpHandler<U, I>
 where
-    U: IndexMetaStore + Send + Sync + 'static,
-    I: IndexStore + Send + Sync + 'static,
+    U: IndexMetaStore + Sync + Send + 'static,
+    I: IndexStore + Sync + Send + 'static,
 {
     fn accept(&self, batch: &Batch) -> bool {
         matches!(batch.content, BatchContent::Dump { .. })
     }
 
-    async fn process_batch(&self, batch: Batch) -> Batch {
-        match batch.content {
-            BatchContent::Dump { .. } => {
-                self.create_dump().await;
+    async fn process_batch(&self, mut batch: Batch) -> Batch {
+        match &batch.content {
+            BatchContent::Dump(Task {
+                content: TaskContent::Dump { uid },
+                ..
+            }) => {
+                match self.run(uid.clone()).await {
+                    Ok(_) => {
+                        batch
+                            .content
+                            .push_event(TaskEvent::succeeded(TaskResult::Other));
+                    }
+                    Err(e) => batch.content.push_event(TaskEvent::failed(e.into())),
+                }
                 batch
             }
             _ => unreachable!("invalid batch content for dump"),

meilisearch-lib/src/tasks/mod.rs

@@ -1,6 +1,6 @@
 use async_trait::async_trait;
 
-pub use batch_handlers::{dump_handler::DumpHandler, empty_handler::EmptyBatchHandler};
+pub use batch_handlers::empty_handler::EmptyBatchHandler;
 pub use scheduler::Scheduler;
 pub use task_store::TaskFilter;
 

meilisearch-lib/src/tasks/scheduler.rs

@@ -1,7 +1,6 @@
 use std::cmp::Ordering;
 use std::collections::{hash_map::Entry, BinaryHeap, HashMap, VecDeque};
 use std::ops::{Deref, DerefMut};
-use std::path::Path;
 use std::slice;
 use std::sync::Arc;
 use std::time::Duration;
@@ -13,7 +12,6 @@ use tokio::sync::{watch, RwLock};
 
 use crate::options::SchedulerConfig;
 use crate::snapshot::SnapshotJob;
-use crate::update_file_store::UpdateFileStore;
 
 use super::batch::{Batch, BatchContent};
 use super::error::Result;
@@ -276,10 +274,6 @@ impl Scheduler {
         Ok(this)
     }
 
-    pub async fn dump(&self, path: &Path, file_store: UpdateFileStore) -> Result<()> {
-        self.store.dump(path, file_store).await
-    }
-
     fn register_task(&mut self, task: Task) {
         assert!(!task.is_finished());
         self.tasks.insert(task);

meilisearch-lib/src/tasks/task.rs

@@ -62,6 +62,22 @@ pub enum TaskEvent {
     },
 }
 
+impl TaskEvent {
+    pub fn succeeded(result: TaskResult) -> Self {
+        Self::Succeded {
+            result,
+            timestamp: OffsetDateTime::now_utc(),
+        }
+    }
+
+    pub fn failed(error: ResponseError) -> Self {
+        Self::Failed {
+            error,
+            timestamp: OffsetDateTime::now_utc(),
+        }
+    }
+}
+
 /// A task represents an operation that Meilisearch must do.
 /// It's stored on disk and executed from the lowest to highest Task id.
 /// Everytime a new task is created it has a higher Task id than the previous one.
@@ -140,7 +156,9 @@ pub enum TaskContent {
     IndexUpdate {
         primary_key: Option<String>,
     },
-    Dump,
+    Dump {
+        uid: String,
+    },
 }
 
 #[cfg(test)]

meilisearch-lib/src/tasks/task_store/mod.rs

@@ -204,13 +204,14 @@ impl TaskStore {
     }
 
     pub async fn dump(
-        &self,
+        env: Arc<Env>,
         dir_path: impl AsRef<Path>,
         update_file_store: UpdateFileStore,
     ) -> Result<()> {
+        let store = Self::new(env)?;
         let update_dir = dir_path.as_ref().join("updates");
         let updates_file = update_dir.join("data.jsonl");
-        let tasks = self.list_tasks(None, None, None).await?;
+        let tasks = store.list_tasks(None, None, None).await?;
 
         let dir_path = dir_path.as_ref().to_path_buf();
         tokio::task::spawn_blocking(move || -> Result<()> {
@@ -287,6 +288,14 @@ pub mod test {
             Ok(Self::Real(TaskStore::new(env)?))
         }
 
+        pub async fn dump(
+            env: Arc<milli::heed::Env>,
+            path: impl AsRef<Path>,
+            update_file_store: UpdateFileStore,
+        ) -> Result<()> {
+            TaskStore::dump(env, path, update_file_store).await
+        }
+
         pub fn mock(mocker: Mocker) -> Self {
             Self::Mock(Arc::new(mocker))
         }
@@ -329,17 +338,6 @@ pub mod test {
             }
         }
 
-        pub async fn dump(
-            &self,
-            path: impl AsRef<Path>,
-            update_file_store: UpdateFileStore,
-        ) -> Result<()> {
-            match self {
-                Self::Real(s) => s.dump(path, update_file_store).await,
-                Self::Mock(m) => unsafe { m.get("dump").call((path, update_file_store)) },
-            }
-        }
-
         pub async fn register(
             &self,
             index_uid: Option<IndexUid>,