Implement task date filters

before/after enqueued/started/finished at

Cargo.lock

@@ -2287,6 +2287,7 @@ dependencies = [
  "log",
  "manifest-dir-macros",
  "maplit",
+ "meili-snap",
  "meilisearch-auth",
  "meilisearch-types",
  "mimalloc",

index-scheduler/src/batch.rs

@@ -2,6 +2,7 @@ use std::collections::HashSet;
 use std::fs::File;
 use std::io::BufWriter;
 
+use crate::utils;
 use crate::{autobatcher::BatchKind, Error, IndexScheduler, Result, TaskId};
 
 use dump::IndexMetadata;
@@ -1015,6 +1016,13 @@ impl IndexScheduler {
             // we can only delete succeeded, failed, and canceled tasks.
             // In each of those cases, the persisted data is supposed to
             // have been deleted already.
+            utils::remove_task_datetime(wtxn, self.enqueued_at, task.enqueued_at, task.uid)?;
+            if let Some(started_at) = task.started_at {
+                utils::remove_task_datetime(wtxn, self.started_at, started_at, task.uid)?;
+            }
+            if let Some(finished_at) = task.finished_at {
+                utils::remove_task_datetime(wtxn, self.finished_at, finished_at, task.uid)?;
+            }
         }
 
         for index in affected_indexes {

index-scheduler/src/lib.rs

@@ -11,6 +11,10 @@ pub type TaskId = u32;
 
 use dump::{KindDump, TaskDump, UpdateFile};
 pub use error::Error;
+use meilisearch_types::milli::documents::DocumentsBatchBuilder;
+use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
+use serde::Serialize;
+use utils::keep_tasks_within_datetimes;
 
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering::Relaxed};
@@ -20,21 +24,20 @@ use file_store::FileStore;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::milli;
 use roaring::RoaringBitmap;
-use serde::{Deserialize, Serialize};
 use synchronoise::SignalEvent;
 use time::OffsetDateTime;
 use uuid::Uuid;
 
 use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{self, Database, Env};
-use meilisearch_types::milli::documents::DocumentsBatchBuilder;
 use meilisearch_types::milli::update::IndexerConfig;
-use meilisearch_types::milli::{Index, RoaringBitmapCodec, BEU32};
+use meilisearch_types::milli::{CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
-use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
 
 use crate::index_mapper::IndexMapper;
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+type BEI128 = meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
+
+#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Query {
     pub limit: Option<u32>,
@@ -44,19 +47,19 @@ pub struct Query {
     pub kind: Option<Vec<Kind>>,
     pub index_uid: Option<Vec<String>>,
     pub uid: Option<Vec<TaskId>>,
-}
 
-impl Default for Query {
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
-    fn default() -> Self {
+    pub before_enqueued_at: Option<OffsetDateTime>,
-        Self {
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
-            limit: None,
+    pub after_enqueued_at: Option<OffsetDateTime>,
-            from: None,
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
-            status: None,
+    pub before_started_at: Option<OffsetDateTime>,
-            kind: None,
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
-            index_uid: None,
+    pub after_started_at: Option<OffsetDateTime>,
-            uid: None,
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
-        }
+    pub before_finished_at: Option<OffsetDateTime>,
-    }
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
+    pub after_finished_at: Option<OffsetDateTime>,
 }
 
 impl Query {
@@ -71,7 +74,13 @@ impl Query {
                 status: None,
                 kind: None,
                 index_uid: None,
-                uid: None
+                uid: None,
+                before_enqueued_at: None,
+                after_enqueued_at: None,
+                before_started_at: None,
+                after_started_at: None,
+                before_finished_at: None,
+                after_finished_at: None,
             }
         )
     }
@@ -177,6 +186,9 @@ mod db_name {
     pub const STATUS: &str = "status";
     pub const KIND: &str = "kind";
     pub const INDEX_TASKS: &str = "index-tasks";
+    pub const ENQUEUED_AT: &str = "enqueued-at";
+    pub const STARTED_AT: &str = "started-at";
+    pub const FINISHED_AT: &str = "finished-at";
 }
 
 /// This module is responsible for two things;
@@ -202,6 +214,20 @@ pub struct IndexScheduler {
     /// Store the tasks associated to an index.
     pub(crate) index_tasks: Database<Str, RoaringBitmapCodec>,
 
+    /// Store the task ids of tasks which were enqueued at a specific date
+    ///
+    /// Note that since we store the date with nanosecond-level precision, it would be
+    /// reasonable to assume that there is only one task per key. However, it is not a
+    /// theoretical certainty, and we might want to make it possible to enqueue multiple
+    /// tasks at a time in the future.
+    pub(crate) enqueued_at: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+
+    /// Store the task ids of finished tasks which started being processed at a specific date
+    pub(crate) started_at: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+
+    /// Store the task ids of tasks which finished at a specific date
+    pub(crate) finished_at: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+
     /// In charge of creating, opening, storing and returning indexes.
     pub(crate) index_mapper: IndexMapper,
 
@@ -247,7 +273,7 @@ impl IndexScheduler {
         std::fs::create_dir_all(&dumps_path)?;
 
         let mut options = heed::EnvOpenOptions::new();
-        options.max_dbs(6);
+        options.max_dbs(9);
 
         let env = options.open(tasks_path)?;
         let file_store = FileStore::new(&update_file_path)?;
@@ -261,6 +287,9 @@ impl IndexScheduler {
             status: env.create_database(Some(db_name::STATUS))?,
             kind: env.create_database(Some(db_name::KIND))?,
             index_tasks: env.create_database(Some(db_name::INDEX_TASKS))?,
+            enqueued_at: env.create_database(Some(db_name::ENQUEUED_AT))?,
+            started_at: env.create_database(Some(db_name::STARTED_AT))?,
+            finished_at: env.create_database(Some(db_name::FINISHED_AT))?,
             index_mapper: IndexMapper::new(&env, indexes_path, index_size, indexer_config)?,
             env,
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
@@ -287,6 +316,9 @@ impl IndexScheduler {
             status: self.status,
             kind: self.kind,
             index_tasks: self.index_tasks,
+            enqueued_at: self.enqueued_at,
+            started_at: self.started_at,
+            finished_at: self.finished_at,
             index_mapper: self.index_mapper.clone(),
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
@@ -359,6 +391,30 @@ impl IndexScheduler {
             }
             tasks &= index_tasks;
         }
+        keep_tasks_within_datetimes(
+            &rtxn,
+            &mut tasks,
+            self.enqueued_at,
+            query.after_enqueued_at,
+            query.before_enqueued_at,
+        )?;
+
+        keep_tasks_within_datetimes(
+            &rtxn,
+            &mut tasks,
+            self.started_at,
+            query.after_started_at,
+            query.before_started_at,
+        )?;
+
+        keep_tasks_within_datetimes(
+            &rtxn,
+            &mut tasks,
+            self.finished_at,
+            query.after_finished_at,
+            query.before_finished_at,
+        )?;
+
         rtxn.commit().unwrap();
         Ok(tasks)
     }
@@ -438,6 +494,8 @@ impl IndexScheduler {
             (bitmap.insert(task.uid));
         })?;
 
+        utils::insert_task_datetime(&mut wtxn, self.enqueued_at, task.enqueued_at, task.uid)?;
+
         if let Err(e) = wtxn.commit() {
             self.delete_persisted_task_data(&task)?;
             return Err(e.into());

index-scheduler/src/snapshot.rs

@@ -1,4 +1,4 @@
-use meilisearch_types::milli::{RoaringBitmapCodec, BEU32};
+use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::Details;
 use meilisearch_types::{
     heed::{
@@ -9,12 +9,13 @@ use meilisearch_types::{
 };
 use roaring::RoaringBitmap;
 
+use crate::BEI128;
 use crate::{index_mapper::IndexMapper, IndexScheduler, Kind, Status};
 
 pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
     let IndexScheduler {
         autobatching_enabled,
-        must_stop_processing,
+        must_stop_processing: _,
         processing_tasks,
         file_store,
         env,
@@ -22,6 +23,9 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         status,
         kind,
         index_tasks,
+        enqueued_at,
+        started_at,
+        finished_at,
         index_mapper,
         wake_up: _,
         dumps_path: _,
@@ -60,6 +64,18 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
     snap.push_str(&snapshot_index_mapper(&rtxn, index_mapper));
     snap.push_str("\n----------------------------------------------------------------------\n");
 
+    snap.push_str("### Enqueued At:\n");
+    snap.push_str(&snapshot_date_db(&rtxn, *enqueued_at));
+    snap.push_str("----------------------------------------------------------------------\n");
+
+    snap.push_str("### Started At:\n");
+    snap.push_str(&snapshot_date_db(&rtxn, *started_at));
+    snap.push_str("----------------------------------------------------------------------\n");
+
+    snap.push_str("### Finished At:\n");
+    snap.push_str(&snapshot_date_db(&rtxn, *finished_at));
+    snap.push_str("----------------------------------------------------------------------\n");
+
     snap.push_str("### File Store:\n");
     snap.push_str(&snapshot_file_store(file_store));
     snap.push_str("\n----------------------------------------------------------------------\n");
@@ -97,6 +113,19 @@ fn snapshot_all_tasks(rtxn: &RoTxn, db: Database<OwnedType<BEU32>, SerdeJson<Tas
     snap
 }
 
+fn snapshot_date_db(
+    rtxn: &RoTxn,
+    db: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+) -> String {
+    let mut snap = String::new();
+    let mut iter = db.iter(rtxn).unwrap();
+    while let Some(next) = iter.next() {
+        let (_timestamp, task_ids) = next.unwrap();
+        snap.push_str(&format!("[timestamp] {}\n", snapshot_bitmap(&task_ids)));
+    }
+    snap
+}
+
 fn snapshot_task(task: &Task) -> String {
     let mut snap = String::new();
     let Task {

index-scheduler/src/snapshots/lib.rs/document_addition/1.snap

@@ -20,6 +20,13 @@ doggos [0,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000000
 

index-scheduler/src/snapshots/lib.rs/document_addition/2.snap

@@ -20,6 +20,13 @@ doggos [0,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000000
 

index-scheduler/src/snapshots/lib.rs/document_addition/3.snap

@@ -21,6 +21,15 @@ doggos [0,]
 ### Index Mapper:
 ["doggos"]
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/1.snap

@@ -23,6 +23,15 @@ doggos [2,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/register/1.snap

@@ -27,6 +27,17 @@ doggo [4,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+[timestamp] [4,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_enqueued.snap

@@ -22,6 +22,14 @@ doggo [1,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000000
 00000000-0000-0000-0000-000000000001

index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,]
@@ -23,6 +23,16 @@ doggo [1,]
 ### Index Mapper:
 ["catto"]
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000001
 

index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap

@@ -25,6 +25,19 @@ doggo [1,]
 ### Index Mapper:
 ["catto"]
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000001
 

index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap

@@ -22,6 +22,14 @@ doggo [1,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000000
 00000000-0000-0000-0000-000000000001

index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap

@@ -7,7 +7,7 @@ source: index-scheduler/src/lib.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentImport { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentImport { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [1,]
@@ -23,6 +23,16 @@ doggo [1,]
 ### Index Mapper:
 ["catto"]
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000001
 

index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap

@@ -24,6 +24,16 @@ doggo [1,]
 ### Index Mapper:
 ["catto"]
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [2,]
+----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000001
 

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap

@@ -24,6 +24,15 @@ doggo [2,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap

@@ -27,6 +27,18 @@ doggo [2,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [3,]
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap

@@ -26,6 +26,16 @@ doggo [2,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap

@@ -26,6 +26,16 @@ doggo [2,]
 ### Index Mapper:
 []
 ----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
 ### File Store:
 
 ----------------------------------------------------------------------

index-scheduler/src/utils.rs

@@ -1,10 +1,15 @@
 //! Utility functions on the DBs. Mainly getter and setters.
 
-use meilisearch_types::heed::{types::DecodeIgnore, RoTxn, RwTxn};
+use std::ops::Bound;
-use meilisearch_types::milli::BEU32;
-use roaring::{MultiOps, RoaringBitmap};
 
-use crate::{Error, IndexScheduler, Result, Task, TaskId};
+use meilisearch_types::heed::types::OwnedType;
+use meilisearch_types::heed::Database;
+use meilisearch_types::heed::{types::DecodeIgnore, RoTxn, RwTxn};
+use meilisearch_types::milli::{CboRoaringBitmapCodec, BEU32};
+use roaring::{MultiOps, RoaringBitmap};
+use time::OffsetDateTime;
+
+use crate::{Error, IndexScheduler, Result, Task, TaskId, BEI128};
 use meilisearch_types::tasks::{Kind, Status};
 
 impl IndexScheduler {
@@ -75,6 +80,26 @@ impl IndexScheduler {
             })?;
         }
 
+        if old_task.enqueued_at != task.enqueued_at {
+            unreachable!("Cannot update a task's enqueued_at time");
+        }
+        if old_task.started_at != task.started_at {
+            if old_task.started_at.is_some() {
+                unreachable!("Cannot update a task's started_at time");
+            }
+            if let Some(started_at) = task.started_at {
+                insert_task_datetime(wtxn, self.started_at, started_at, task.uid)?;
+            }
+        }
+        if old_task.finished_at != task.finished_at {
+            if old_task.finished_at.is_some() {
+                unreachable!("Cannot update a task's finished_at time");
+            }
+            if let Some(finished_at) = task.finished_at {
+                insert_task_datetime(wtxn, self.finished_at, finished_at, task.uid)?;
+            }
+        }
+
         self.all_tasks.put(wtxn, &BEU32::new(task.uid), task)?;
         Ok(())
     }
@@ -158,3 +183,73 @@ impl IndexScheduler {
         Ok(())
     }
 }
+
+pub(crate) fn insert_task_datetime(
+    wtxn: &mut RwTxn,
+    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+
+    time: OffsetDateTime,
+    task_id: TaskId,
+) -> Result<()> {
+    let timestamp = BEI128::new(time.unix_timestamp_nanos());
+    let mut task_ids = if let Some(existing) = database.get(&wtxn, &timestamp)? {
+        existing
+    } else {
+        RoaringBitmap::new()
+    };
+    task_ids.insert(task_id);
+    database.put(wtxn, &timestamp, &RoaringBitmap::from_iter([task_id]))?;
+    Ok(())
+}
+pub(crate) fn remove_task_datetime(
+    wtxn: &mut RwTxn,
+    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+
+    time: OffsetDateTime,
+    task_id: TaskId,
+) -> Result<()> {
+    let timestamp = BEI128::new(time.unix_timestamp_nanos());
+    if let Some(mut existing) = database.get(&wtxn, &timestamp)? {
+        existing.remove(task_id);
+        if existing.is_empty() {
+            database.delete(wtxn, &timestamp)?;
+        } else {
+            database.put(wtxn, &timestamp, &RoaringBitmap::from_iter([task_id]))?;
+        }
+    }
+
+    Ok(())
+}
+pub(crate) fn keep_tasks_within_datetimes(
+    rtxn: &RoTxn,
+    tasks: &mut RoaringBitmap,
+    database: Database<OwnedType<BEI128>, CboRoaringBitmapCodec>,
+    after: Option<OffsetDateTime>,
+    before: Option<OffsetDateTime>,
+) -> Result<()> {
+    let (start, end) = match (&after, &before) {
+        (None, None) => return Ok(()),
+        (None, Some(before)) => (Bound::Unbounded, Bound::Excluded(*before)),
+        (Some(after), None) => (Bound::Excluded(*after), Bound::Unbounded),
+        (Some(after), Some(before)) => (Bound::Excluded(*after), Bound::Excluded(*before)),
+    };
+    let mut collected_task_ids = RoaringBitmap::new();
+    let start = map_bound(start, |b| BEI128::new(b.unix_timestamp_nanos()));
+    let end = map_bound(end, |b| BEI128::new(b.unix_timestamp_nanos()));
+    let iter = database.range(&rtxn, &(start, end))?;
+    for r in iter {
+        let (_timestamp, task_ids) = r?;
+        collected_task_ids |= task_ids;
+    }
+    *tasks &= collected_task_ids;
+    Ok(())
+}
+
+// TODO: remove when Bound::map ( https://github.com/rust-lang/rust/issues/86026 ) is available on stable
+fn map_bound<T, U>(bound: Bound<T>, map: impl FnOnce(T) -> U) -> Bound<U> {
+    match bound {
+        Bound::Included(x) => Bound::Included(map(x)),
+        Bound::Excluded(x) => Bound::Excluded(map(x)),
+        Bound::Unbounded => Bound::Unbounded,
+    }
+}

meili-snap/src/lib.rs

@@ -4,6 +4,8 @@ use std::path::PathBuf;
 use std::sync::Mutex;
 use std::{collections::HashMap, path::Path};
 
+pub use insta;
+
 static SNAPSHOT_NAMES: Lazy<Mutex<HashMap<PathBuf, usize>>> = Lazy::new(|| Mutex::default());
 
 /// Return the md5 hash of the given string
@@ -81,8 +83,8 @@ macro_rules! snapshot_hash {
         settings.bind(|| {
             let snap = format!("{}", $value);
             let hash_snap = $crate::hash_snapshot(&snap);
-            insta::assert_snapshot!(hash_snap, @$inline);
+            meili_snap::insta::assert_snapshot!(hash_snap, @$inline);
-            insta::assert_snapshot!(format!("{}.full", snap_name), snap);
+            meili_snap::insta::assert_snapshot!(format!("{}.full", snap_name), snap);
         });
     };
     ($value:expr, name: $name:expr, @$inline:literal) => {
@@ -91,8 +93,8 @@ macro_rules! snapshot_hash {
         settings.bind(|| {
             let snap = format!("{}", $value);
             let hash_snap = $crate::hash_snapshot(&snap);
-            insta::assert_snapshot!(hash_snap, @$inline);
+            meili_snap::insta::assert_snapshot!(hash_snap, @$inline);
-            insta::assert_snapshot!(format!("{}.full", snap_name), snap);
+            meili_snap::insta::assert_snapshot!(format!("{}.full", snap_name), snap);
         });
     };
 }
@@ -132,7 +134,7 @@ macro_rules! snapshot {
         let (settings, snap_name) = $crate::default_snapshot_settings_for_test(Some(&snap_name));
         settings.bind(|| {
             let snap = format!("{}", $value);
-            insta::assert_snapshot!(format!("{}", snap_name), snap);
+            meili_snap::insta::assert_snapshot!(format!("{}", snap_name), snap);
         });
     };
     ($value:expr, @$inline:literal) => {
@@ -141,21 +143,21 @@ macro_rules! snapshot {
         let (settings, _) = $crate::default_snapshot_settings_for_test(Some("_dummy_argument"));
         settings.bind(|| {
             let snap = format!("{}", $value);
-            insta::assert_snapshot!(snap, @$inline);
+            meili_snap::insta::assert_snapshot!(snap, @$inline);
         });
     };
     ($value:expr) => {
         let (settings, snap_name) = $crate::default_snapshot_settings_for_test(None);
         settings.bind(|| {
             let snap = format!("{}", $value);
-            insta::assert_snapshot!(format!("{}", snap_name), snap);
+            meili_snap::insta::assert_snapshot!(format!("{}", snap_name), snap);
         });
     };
 }
 
 #[cfg(test)]
 mod tests {
-
+    use crate as meili_snap;
     #[test]
     fn snap() {
         snapshot_hash!(10, @"d3d9446802a44259755d38e6d163e820");
@@ -180,6 +182,7 @@ mod tests {
     // Currently the name of this module is not part of the snapshot path
     // It does not bother me, but maybe it is worth changing later on.
     mod snap {
+        use crate as meili_snap;
         #[test]
         fn some_test() {
             snapshot_hash!(10, @"d3d9446802a44259755d38e6d163e820");
@@ -214,15 +217,15 @@ mod tests {
 macro_rules! json_string {
     ($value:expr, {$($k:expr => $v:expr),*$(,)?}) => {
         {
-            let (_, snap) = insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File);
+            let (_, snap) = meili_snap::insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File);
             snap
         }
     };
     ($value:expr) => {{
-        let value = insta::_macro_support::serialize_value(
+        let value = meili_snap::insta::_macro_support::serialize_value(
             &$value,
-            insta::_macro_support::SerializationFormat::Json,
+            meili_snap::insta::_macro_support::SerializationFormat::Json,
-            insta::_macro_support::SnapshotLocation::File
+            meili_snap::insta::_macro_support::SnapshotLocation::File
         );
         value
     }};

meilisearch-http/Cargo.toml

@@ -94,6 +94,7 @@ brotli = "3.3.4"
 manifest-dir-macros = "0.1.16"
 maplit = "1.0.2"
 urlencoding = "2.1.2"
+meili-snap = {path = "../meili-snap"}
 yaup = "0.2.1"
 temp-env = "0.3.1"
 

meilisearch-http/src/routes/tasks.rs

@@ -180,26 +180,79 @@ impl From<Details> for DetailsView {
     }
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct TaskDateQuery {
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    after_enqueued_at: Option<OffsetDateTime>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    before_enqueued_at: Option<OffsetDateTime>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    after_started_at: Option<OffsetDateTime>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    before_started_at: Option<OffsetDateTime>,
+
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    after_finished_at: Option<OffsetDateTime>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "time::serde::rfc3339::option::serialize",
+        deserialize_with = "rfc3339_date_or_datetime::deserialize"
+    )]
+    before_finished_at: Option<OffsetDateTime>,
+}
+
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct TasksFilterQuery {
     #[serde(rename = "type")]
-    type_: Option<CS<StarOr<Kind>>>,
+    kind: Option<CS<StarOr<Kind>>>,
+    uid: Option<CS<u32>>,
     status: Option<CS<StarOr<Status>>>,
-    index_uid: Option<CS<StarOr<IndexUid>>>,
+    index_uid: Option<CS<StarOr<String>>>,
     #[serde(default = "DEFAULT_LIMIT")]
     limit: u32,
     from: Option<TaskId>,
+    #[serde(flatten)]
+    dates: TaskDateQuery,
 }
 
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct TaskDeletionQuery {
     #[serde(rename = "type")]
-    type_: Option<CS<Kind>>,
+    kind: Option<CS<Kind>>,
     uid: Option<CS<u32>>,
     status: Option<CS<Status>>,
     index_uid: Option<CS<IndexUid>>,
+    #[serde(flatten)]
+    dates: TaskDateQuery,
 }
 
 #[derive(Deserialize, Debug)]
@@ -210,6 +263,8 @@ pub struct TaskCancelationQuery {
     uid: Option<CS<u32>>,
     status: Option<CS<Status>>,
     index_uid: Option<CS<IndexUid>>,
+    #[serde(flatten)]
+    dates: TaskDateQuery,
 }
 
 async fn cancel_tasks(
@@ -222,6 +277,15 @@ async fn cancel_tasks(
         uid,
         status,
         index_uid,
+        dates:
+            TaskDateQuery {
+                after_enqueued_at,
+                before_enqueued_at,
+                after_started_at,
+                before_started_at,
+                after_finished_at,
+                before_finished_at,
+            },
     } = params.into_inner();
 
     let kind: Option<Vec<_>> = type_.map(|x| x.into_iter().collect());
@@ -237,6 +301,12 @@ async fn cancel_tasks(
         kind,
         index_uid,
         uid,
+        before_enqueued_at,
+        after_enqueued_at,
+        before_started_at,
+        after_started_at,
+        before_finished_at,
+        after_finished_at,
     };
 
     if query.is_empty() {
@@ -262,10 +332,19 @@ async fn delete_tasks(
     params: web::Query<TaskDeletionQuery>,
 ) -> Result<HttpResponse, ResponseError> {
     let TaskDeletionQuery {
-        type_,
+        kind: type_,
         uid,
         status,
         index_uid,
+        dates:
+            TaskDateQuery {
+                after_enqueued_at,
+                before_enqueued_at,
+                after_started_at,
+                before_started_at,
+                after_finished_at,
+                before_finished_at,
+            },
     } = params.into_inner();
 
     let kind: Option<Vec<_>> = type_.map(|x| x.into_iter().collect());
@@ -281,6 +360,12 @@ async fn delete_tasks(
         kind,
         index_uid,
         uid,
+        after_enqueued_at,
+        before_enqueued_at,
+        after_started_at,
+        before_started_at,
+        after_finished_at,
+        before_finished_at,
     };
 
     if query.is_empty() {
@@ -307,18 +392,27 @@ async fn get_tasks(
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let TasksFilterQuery {
-        type_,
+        kind,
+        uid,
         status,
         index_uid,
         limit,
         from,
+        dates:
+            TaskDateQuery {
+                after_enqueued_at,
+                before_enqueued_at,
+                after_started_at,
+                before_started_at,
+                after_finished_at,
+                before_finished_at,
+            },
     } = params.into_inner();
 
-    let search_rules = &index_scheduler.filters().search_rules;
-
     // We first transform a potential indexUid=* into a "not specified indexUid filter"
     // for every one of the filters: type, status, and indexUid.
-    let type_: Option<Vec<_>> = type_.and_then(fold_star_or);
+    let type_: Option<Vec<_>> = kind.and_then(fold_star_or);
+    let uid: Option<Vec<_>> = uid.map(|x| x.into_iter().collect());
     let status: Option<Vec<_>> = status.and_then(fold_star_or);
     let index_uid: Option<Vec<_>> = index_uid.and_then(fold_star_or);
 
@@ -332,47 +426,27 @@ async fn get_tasks(
         Some(&req),
     );
 
-    // TODO: Lo: use `filter_out_inaccessible_indexes_from_query` here
-    let mut filters = index_scheduler::Query::default();
-
-    // Then we filter on potential indexes and make sure that the search filter
-    // restrictions are also applied.
-    match index_uid {
-        Some(indexes) => {
-            for name in indexes {
-                if search_rules.is_index_authorized(&name) {
-                    filters = filters.with_index(name.to_string());
-                }
-            }
-        }
-        None => {
-            if !search_rules.is_index_authorized("*") {
-                for (index, _policy) in search_rules.clone() {
-                    filters = filters.with_index(index.to_string());
-                }
-            }
-        }
-    };
-
-    if let Some(kinds) = type_ {
-        for kind in kinds {
-            filters = filters.with_kind(kind);
-        }
-    }
-
-    if let Some(statuses) = status {
-        for status in statuses {
-            filters = filters.with_status(status);
-        }
-    }
-
-    filters.from = from;
     // We +1 just to know if there is more after this "page" or not.
     let limit = limit.saturating_add(1);
-    filters.limit = Some(limit);
+
+    let query = index_scheduler::Query {
+        limit: Some(limit),
+        from,
+        status,
+        kind: type_,
+        index_uid,
+        uid,
+        before_enqueued_at,
+        after_enqueued_at,
+        before_started_at,
+        after_started_at,
+        before_finished_at,
+        after_finished_at,
+    };
+    let query = filter_out_inaccessible_indexes_from_query(&index_scheduler, &query);
 
     let mut tasks_results: Vec<TaskView> = index_scheduler
-        .get_tasks(filters)?
+        .get_tasks(query)?
         .into_iter()
         .map(|t| TaskView::from_task(&t))
         .collect();
@@ -462,3 +536,126 @@ fn filter_out_inaccessible_indexes_from_query<const ACTION: u8>(
 
     query
 }
+
+/// Deserialize a datetime optional string using rfc3339, assuming midnight and UTC+0 if not specified
+pub mod rfc3339_date_or_datetime {
+    #[allow(clippy::wildcard_imports)]
+    use super::*;
+    use serde::Deserializer;
+    use time::format_description::well_known::iso8601::{Config, EncodedConfig};
+    use time::format_description::well_known::{Iso8601, Rfc3339};
+    use time::{Date, PrimitiveDateTime, Time};
+    const SERDE_CONFIG: EncodedConfig = Config::DEFAULT.set_year_is_six_digits(true).encode();
+
+    /// Deserialize an [`Option<OffsetDateTime>`] from its ISO 8601 representation.
+    pub fn deserialize<'a, D: Deserializer<'a>>(
+        deserializer: D,
+    ) -> Result<Option<OffsetDateTime>, D::Error> {
+        deserializer.deserialize_option(Visitor)
+    }
+    struct Visitor;
+
+    #[derive(Debug)]
+    struct DeserializeError;
+
+    impl<'a> serde::de::Visitor<'a> for Visitor {
+        type Value = Option<OffsetDateTime>;
+
+        fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            formatter.write_str("an rfc3339- or iso8601-formatted datetime")
+        }
+        fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Option<OffsetDateTime>, E> {
+            let datetime = OffsetDateTime::parse(value, &Rfc3339)
+                .or_else(|_e| OffsetDateTime::parse(value, &Iso8601::<SERDE_CONFIG>))
+                .or_else(|_e| {
+                    PrimitiveDateTime::parse(value, &Iso8601::<SERDE_CONFIG>)
+                        .map(|x| x.assume_utc())
+                })
+                .or_else(|_e| {
+                    Date::parse(value, &Iso8601::<SERDE_CONFIG>)
+                        .map(|date| date.with_time(Time::MIDNIGHT).assume_utc())
+                })
+                .map_err(|_e| {
+                    serde::de::Error::custom(
+                        "could not parse an rfc3339- or iso8601-formatted date",
+                    )
+                })?;
+
+            Ok(Some(datetime))
+        }
+        fn visit_some<D: Deserializer<'a>>(
+            self,
+            deserializer: D,
+        ) -> Result<Option<OffsetDateTime>, D::Error> {
+            deserializer.deserialize_str(Visitor)
+        }
+
+        fn visit_none<E: serde::de::Error>(self) -> Result<Option<OffsetDateTime>, E> {
+            Ok(None)
+        }
+
+        fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
+            Ok(None)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::routes::tasks::TaskDeletionQuery;
+    use meili_snap::snapshot;
+
+    #[test]
+    fn deserialize_task_deletion_query_datetime() {
+        {
+            let json = r#" { "afterEnqueuedAt": "2021" } "#;
+            let err = serde_json::from_str::<TaskDeletionQuery>(json).unwrap_err();
+            snapshot!(format!("{err}"), @"could not parse an rfc3339- or iso8601-formatted date at line 1 column 30");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12" } "#;
+            let err = serde_json::from_str::<TaskDeletionQuery>(json).unwrap_err();
+            snapshot!(format!("{err}"), @"could not parse an rfc3339- or iso8601-formatted date at line 1 column 33");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"2021-12-03 0:00:00.0 +00:00:00");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03T23" } "#;
+            let err = serde_json::from_str::<TaskDeletionQuery>(json).unwrap_err();
+            snapshot!(format!("{err}"), @"could not parse an rfc3339- or iso8601-formatted date at line 1 column 39");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03T23:45" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"2021-12-03 23:45:00.0 +00:00:00");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03T23:45:23" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"2021-12-03 23:45:23.0 +00:00:00");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03T23:45:23 +01:00" } "#;
+            let err = serde_json::from_str::<TaskDeletionQuery>(json).unwrap_err();
+            snapshot!(format!("{err}"), @"could not parse an rfc3339- or iso8601-formatted date at line 1 column 52");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "2021-12-03T23:45:23+01:00" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"2021-12-03 23:45:23.0 +01:00:00");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "1997-11-12T09:55:06.000000000-06:00" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"1997-11-12 9:55:06.0 -06:00:00");
+        }
+        {
+            let json = r#" { "afterEnqueuedAt": "1997-11-12T09:55:06.000000000Z" } "#;
+            let query = serde_json::from_str::<TaskDeletionQuery>(json).unwrap();
+            snapshot!(format!("{:?}", query.dates.after_enqueued_at.unwrap()), @"1997-11-12 9:55:06.0 +00:00:00");
+        }
+    }
+}