From 8ad68dd7088c71df821a63aab998c80992496515 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 20 Nov 2024 13:17:54 +0100 Subject: [PATCH] stop leaking the update files of the canceled tasks --- crates/index-scheduler/src/lib.rs | 13 ++++++++----- .../cancel_enqueued_task/cancel_processed.snap | 3 +-- .../cancel_mix_of_tasks/cancel_processed.snap | 4 +--- .../cancel_processing_dump/cancel_processed.snap | 2 +- .../cancel_processing_task/cancel_processed.snap | 3 +-- .../lib.rs/query_batches_canceled_by/start.snap | 2 +- .../lib.rs/query_tasks_canceled_by/start.snap | 2 +- 7 files changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 36ab93eb0..b976d7342 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -1556,7 +1556,7 @@ impl IndexScheduler { drop(rtxn); // 1. store the starting date with the bitmap of processing tasks. - let ids = batch.ids(); + let mut ids = batch.ids(); let processed_tasks = ids.len(); // We reset the must_stop flag to be sure that we don't stop processing tasks @@ -1593,6 +1593,7 @@ impl IndexScheduler { processing_batch.finished(); let mut wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?; + let mut canceled = RoaringBitmap::new(); match res { Ok(tasks) => { @@ -1602,7 +1603,6 @@ impl IndexScheduler { let mut success = 0; let mut failure = 0; let mut canceled_by = None; - let mut canceled = RoaringBitmap::new(); #[allow(unused_variables)] for (i, mut task) in tasks.into_iter().enumerate() { @@ -1693,9 +1693,12 @@ impl IndexScheduler { } } - let processed = self.processing_tasks.write().unwrap().stop_processing(); + self.processing_tasks.write().unwrap().stop_processing(); + // We must re-add the canceled task so they're part of the same batch. + // processed.processing |= canceled; + ids |= canceled; - self.write_batch(&mut wtxn, processing_batch, &processed.processing)?; + self.write_batch(&mut wtxn, processing_batch, &ids)?; #[cfg(test)] self.maybe_fail(tests::FailureLocation::CommittingWtxn)?; @@ -1725,7 +1728,7 @@ impl IndexScheduler { })?; // We shouldn't crash the tick function if we can't send data to the webhook. - let _ = self.notify_webhook(&processed.processing); + let _ = self.notify_webhook(&ids); #[cfg(test)] self.breakpoint(Breakpoint::AfterProcessing); diff --git a/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap index fa67aa779..f0c382d86 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap @@ -43,7 +43,7 @@ catto [0,] 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0,"matchedTasks":1,"canceledTasks":1,"originalFilter":"test_query"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"documentAdditionOrUpdate":1,"taskCancelation":1},"indexUids":{"catto":1}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,1,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,] @@ -67,6 +67,5 @@ catto [0,] [timestamp] [0,] ---------------------------------------------------------------------- ### File Store: -00000000-0000-0000-0000-000000000000 ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap index d94494a26..444b171dd 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap @@ -55,7 +55,7 @@ catto: { number_of_documents: 1, field_distribution: {"id": 1} } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] -1 [3,] +1 [1,2,3,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,1,] @@ -84,7 +84,5 @@ wolfo [1,] [timestamp] [1,] ---------------------------------------------------------------------- ### File Store: -00000000-0000-0000-0000-000000000001 -00000000-0000-0000-0000-000000000002 ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap index 670116fe1..dbae3a082 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap @@ -42,7 +42,7 @@ canceled [0,] 0 {uid: 0, details: {"matchedTasks":1,"canceledTasks":1,"originalFilter":"cancel dump"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"taskCancelation":1,"dumpCreation":1},"indexUids":{}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,1,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,] diff --git a/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap index ece94e06a..ef6845b05 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap @@ -44,7 +44,7 @@ catto: { number_of_documents: 0, field_distribution: {} } 0 {uid: 0, details: {"receivedDocuments":1,"indexedDocuments":0,"matchedTasks":1,"canceledTasks":1,"originalFilter":"test_query"}, stats: {"totalNbTasks":2,"status":{"succeeded":1,"canceled":1},"types":{"documentAdditionOrUpdate":1,"taskCancelation":1},"indexUids":{"catto":1}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,1,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,] @@ -68,6 +68,5 @@ catto [0,] [timestamp] [0,] ---------------------------------------------------------------------- ### File Store: -00000000-0000-0000-0000-000000000000 ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/snapshots/lib.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_batches_canceled_by/start.snap index 79c5e6499..ea3a75e8f 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/query_batches_canceled_by/start.snap @@ -54,7 +54,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] -1 [3,] +1 [1,2,3,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,1,] diff --git a/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap index 79c5e6499..ea3a75e8f 100644 --- a/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap @@ -54,7 +54,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] -1 [3,] +1 [1,2,3,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,1,]