Merge #3659

3659: stops receiving tasks once the task queue is full r=Kerollmops a=irevoire

Give 20GiB to the task queue + once 50% of the task queue is used, it blocks itself and only receives task deletion requests to ensure we never get in a state where we can’t do anything.

Also, create a new error message when we reach this case:
```
Meilisearch cannot receive write operations because the size limit of the tasks database has been reached. Please delete tasks to continue performing write operations.
```

Co-authored-by: Tamo <tamo@meilisearch.com>

index-scheduler/src/error.rs

@@ -61,6 +61,8 @@ pub enum Error {
     SwapDuplicateIndexesFound(Vec<String>),
     #[error("Index `{0}` not found.")]
     SwapIndexNotFound(String),
+    #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")]
+    NoSpaceLeftInTaskQueue,
     #[error(
         "Indexes {} not found.",
         .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
@@ -152,6 +154,8 @@ impl ErrorCode for Error {
             Error::TaskNotFound(_) => Code::TaskNotFound,
             Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
             Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
+            // TODO: not sure of the Code to use
+            Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
             Error::Dump(e) => e.error_code(),
             Error::Milli(e) => e.error_code(),
             Error::ProcessBatchPanicked => Code::Internal,

index-scheduler/src/lib.rs

@@ -820,6 +820,13 @@ impl IndexScheduler {
     pub fn register(&self, kind: KindWithContent) -> Result<Task> {
         let mut wtxn = self.env.write_txn()?;
 
+        // if the task doesn't delete anything and 50% of the task queue is full, we must refuse to enqueue the incomming task
+        if !matches!(&kind, KindWithContent::TaskDeletion { tasks, .. } if !tasks.is_empty())
+            && (self.env.real_disk_size()? * 100) / self.env.map_size()? as u64 > 50
+        {
+            return Err(Error::NoSpaceLeftInTaskQueue);
+        }
+
         let mut task = Task {
             uid: self.next_task_id(&wtxn)?,
             enqueued_at: OffsetDateTime::now_utc(),