synchronize most of the operations

cluster/Cargo.toml

@@ -16,7 +16,9 @@ serde = { version = "1.0.155", features = ["derive"] }
 serde_json = "1.0.94"
 thiserror = "1.0.39"
 meilisearch-types = { path = "../meilisearch-types" }
-roaring = "0.10.1"
+roaring = { version = "0.10.1", features = ["serde"] }
 log = "0.4.17"
 crossbeam = "0.8.2"
 bus = "2.3.0"
+time = "0.3.20"
+uuid = { version = "1.3.0", features = ["v4"] }

cluster/src/batch.rs

@@ -0,0 +1,105 @@
+use meilisearch_types::milli::update::IndexDocumentsMethod;
+use meilisearch_types::settings::{Settings, Unchecked};
+use meilisearch_types::tasks::TaskId;
+use roaring::RoaringBitmap;
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+/// Represents a combination of tasks that can all be processed at the same time.
+///
+/// A batch contains the set of tasks that it represents (accessible through
+/// [`self.ids()`](Batch::ids)), as well as additional information on how to
+/// be processed.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Batch {
+    TaskCancelation {
+        /// The task cancelation itself.
+        task: TaskId,
+        /// The date and time at which the previously processing tasks started.
+        previous_started_at: OffsetDateTime,
+        /// The list of tasks that were processing when this task cancelation appeared.
+        previous_processing_tasks: RoaringBitmap,
+    },
+    TaskDeletion(TaskId),
+    SnapshotCreation(Vec<TaskId>),
+    Dump(TaskId),
+    IndexOperation {
+        op: IndexOperation,
+        must_create_index: bool,
+    },
+    IndexCreation {
+        index_uid: String,
+        primary_key: Option<String>,
+        task: TaskId,
+    },
+    IndexUpdate {
+        index_uid: String,
+        primary_key: Option<String>,
+        task: TaskId,
+    },
+    IndexDeletion {
+        index_uid: String,
+        tasks: Vec<TaskId>,
+        index_has_been_created: bool,
+    },
+    IndexSwap {
+        task: TaskId,
+    },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum DocumentOperation {
+    Add(Uuid),
+    Delete(Vec<String>),
+}
+
+/// A [batch](Batch) that combines multiple tasks operating on an index.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum IndexOperation {
+    DocumentOperation {
+        index_uid: String,
+        primary_key: Option<String>,
+        method: IndexDocumentsMethod,
+        documents_counts: Vec<u64>,
+        operations: Vec<DocumentOperation>,
+        tasks: Vec<TaskId>,
+    },
+    DocumentDeletion {
+        index_uid: String,
+        // The vec associated with each document deletion tasks.
+        documents: Vec<Vec<String>>,
+        tasks: Vec<TaskId>,
+    },
+    DocumentClear {
+        index_uid: String,
+        tasks: Vec<TaskId>,
+    },
+    Settings {
+        index_uid: String,
+        // The boolean indicates if it's a settings deletion or creation.
+        settings: Vec<(bool, Settings<Unchecked>)>,
+        tasks: Vec<TaskId>,
+    },
+    DocumentClearAndSetting {
+        index_uid: String,
+        cleared_tasks: Vec<TaskId>,
+
+        // The boolean indicates if it's a settings deletion or creation.
+        settings: Vec<(bool, Settings<Unchecked>)>,
+        settings_tasks: Vec<TaskId>,
+    },
+    SettingsAndDocumentOperation {
+        index_uid: String,
+
+        primary_key: Option<String>,
+        method: IndexDocumentsMethod,
+        documents_counts: Vec<u64>,
+        operations: Vec<DocumentOperation>,
+        document_import_tasks: Vec<TaskId>,
+
+        // The boolean indicates if it's a settings deletion or creation.
+        settings: Vec<(bool, Settings<Unchecked>)>,
+        settings_tasks: Vec<TaskId>,
+    },
+}

cluster/src/leader.rs

@@ -7,6 +7,7 @@ use crossbeam::channel::{unbounded, Receiver, Sender};
 use ductile::{ChannelReceiver, ChannelSender, ChannelServer};
 use log::info;
 
+use crate::batch::Batch;
 use crate::{Consistency, FollowerMsg, LeaderMsg};
 
 pub struct Leader {
@@ -110,7 +111,7 @@ impl Leader {
         info!("A follower left the cluster. {} members.", size);
     }
 
-    pub fn starts_batch(&mut self, batch: Vec<u32>) {
+    pub fn starts_batch(&mut self, batch: Batch) {
         assert!(
             self.batch_id % 2 == 0,
             "Tried to start processing a batch before commiting the previous one"

cluster/src/lib.rs

@@ -1,9 +1,11 @@
 use std::net::ToSocketAddrs;
 
+use batch::Batch;
 use ductile::{connect_channel, ChannelReceiver, ChannelSender};
 use meilisearch_types::tasks::KindWithContent;
 use serde::{Deserialize, Serialize};
 
+pub mod batch;
 mod leader;
 
 pub use leader::Leader;
@@ -19,7 +21,7 @@ pub enum Error {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum LeaderMsg {
     // Starts a new batch
-    StartBatch { id: u32, batch: Vec<u32> },
+    StartBatch { id: u32, batch: Batch },
     // Tell the follower to commit the update asap
     Commit(u32),
 }
@@ -52,7 +54,7 @@ impl Follower {
         Follower { sender, receiver, batch_id: 0 }
     }
 
-    pub fn get_new_batch(&mut self) -> Vec<u32> {
+    pub fn get_new_batch(&mut self) -> Batch {
         loop {
             match self.receiver.recv() {
                 Ok(LeaderMsg::StartBatch { id, batch }) if id == self.batch_id => {