stop leaking the update files of the canceled tasks

crates/index-scheduler/src/lib.rs

@@ -1556,7 +1556,7 @@ impl IndexScheduler {
         drop(rtxn);
 
         // 1. store the starting date with the bitmap of processing tasks.
-        let ids = batch.ids();
+        let mut ids = batch.ids();
         let processed_tasks = ids.len();
 
         // We reset the must_stop flag to be sure that we don't stop processing tasks
@@ -1593,6 +1593,7 @@ impl IndexScheduler {
 
         processing_batch.finished();
         let mut wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?;
+        let mut canceled = RoaringBitmap::new();
 
         match res {
             Ok(tasks) => {
@@ -1602,7 +1603,6 @@ impl IndexScheduler {
                 let mut success = 0;
                 let mut failure = 0;
                 let mut canceled_by = None;
-                let mut canceled = RoaringBitmap::new();
 
                 #[allow(unused_variables)]
                 for (i, mut task) in tasks.into_iter().enumerate() {
@@ -1693,9 +1693,12 @@ impl IndexScheduler {
             }
         }
 
-        let processed = self.processing_tasks.write().unwrap().stop_processing();
+        self.processing_tasks.write().unwrap().stop_processing();
+        // We must re-add the canceled task so they're part of the same batch.
+        // processed.processing |= canceled;
+        ids |= canceled;
 
-        self.write_batch(&mut wtxn, processing_batch, &processed.processing)?;
+        self.write_batch(&mut wtxn, processing_batch, &ids)?;
 
         #[cfg(test)]
         self.maybe_fail(tests::FailureLocation::CommittingWtxn)?;
@@ -1725,7 +1728,7 @@ impl IndexScheduler {
         })?;
 
         // We shouldn't crash the tick function if we can't send data to the webhook.
-        let _ = self.notify_webhook(&processed.processing);
+        let _ = self.notify_webhook(&ids);
 
         #[cfg(test)]
         self.breakpoint(Breakpoint::AfterProcessing);

crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap

@@ -43,7 +43,7 @@ catto [0,]
 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0,"matchedTasks":1,"canceledTasks":1,"originalFilter":"test_query"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"documentAdditionOrUpdate":1,"taskCancelation":1},"indexUids":{"catto":1}}, }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
-0 [1,]
+0 [0,1,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,]
@@ -67,6 +67,5 @@ catto [0,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### File Store:
-00000000-0000-0000-0000-000000000000
 
 ----------------------------------------------------------------------

crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap

@@ -55,7 +55,7 @@ catto: { number_of_documents: 1, field_distribution: {"id": 1} }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
-1 [3,]
+1 [1,2,3,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,1,]
@@ -84,7 +84,5 @@ wolfo [1,]
 [timestamp] [1,]
 ----------------------------------------------------------------------
 ### File Store:
-00000000-0000-0000-0000-000000000001
-00000000-0000-0000-0000-000000000002
 
 ----------------------------------------------------------------------

crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap

@@ -42,7 +42,7 @@ canceled [0,]
 0 {uid: 0, details: {"matchedTasks":1,"canceledTasks":1,"originalFilter":"cancel dump"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"taskCancelation":1,"dumpCreation":1},"indexUids":{}}, }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
-0 [1,]
+0 [0,1,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,]

crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap

@@ -44,7 +44,7 @@ catto: { number_of_documents: 0, field_distribution: {} }
 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0,"matchedTasks":1,"canceledTasks":1,"originalFilter":"test_query"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"documentAdditionOrUpdate":1,"taskCancelation":1},"indexUids":{"catto":1}}, }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
-0 [1,]
+0 [0,1,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,]
@@ -68,6 +68,5 @@ catto [0,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### File Store:
-00000000-0000-0000-0000-000000000000
 
 ----------------------------------------------------------------------

crates/index-scheduler/src/snapshots/lib.rs/query_batches_canceled_by/start.snap

@@ -54,7 +54,7 @@ catto: { number_of_documents: 0, field_distribution: {} }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
-1 [3,]
+1 [1,2,3,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,1,]

crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap

@@ -54,7 +54,7 @@ catto: { number_of_documents: 0, field_distribution: {} }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
-1 [3,]
+1 [1,2,3,]
 ----------------------------------------------------------------------
 ### Batches Status:
 succeeded [0,1,]