mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-11 07:28:56 +01:00
Delete a task's persisted data when appropriate
This commit is contained in:
parent
f7e546eea3
commit
ea60d35c71
@ -1,6 +1,8 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
use std::fs::File as StdFile;
|
use std::fs::File as StdFile;
|
||||||
use std::ops::{Deref, DerefMut};
|
use std::ops::{Deref, DerefMut};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::str::FromStr;
|
||||||
|
|
||||||
use tempfile::NamedTempFile;
|
use tempfile::NamedTempFile;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@ -96,6 +98,20 @@ impl FileStore {
|
|||||||
std::fs::remove_file(path)?;
|
std::fs::remove_file(path)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// List the Uuids of the files in the FileStore
|
||||||
|
///
|
||||||
|
/// This function is meant to be used by tests only.
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub fn __all_uuids(&self) -> BTreeSet<Uuid> {
|
||||||
|
let mut uuids = BTreeSet::new();
|
||||||
|
for entry in self.path.read_dir().unwrap() {
|
||||||
|
let entry = entry.unwrap();
|
||||||
|
let uuid = Uuid::from_str(entry.file_name().to_str().unwrap()).unwrap();
|
||||||
|
uuids.insert(uuid);
|
||||||
|
}
|
||||||
|
uuids
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct File {
|
pub struct File {
|
||||||
|
@ -953,15 +953,18 @@ impl IndexScheduler {
|
|||||||
let mut affected_kinds = HashSet::new();
|
let mut affected_kinds = HashSet::new();
|
||||||
|
|
||||||
for task_id in to_delete_tasks.iter() {
|
for task_id in to_delete_tasks.iter() {
|
||||||
// This should never fail, but there is no harm done if it does. The function
|
let task = self
|
||||||
// will still be 99% correct (the number of deleted tasks will be slightly incorrect).
|
.get_task(wtxn, task_id)?
|
||||||
if let Some(task) = self.get_task(wtxn, task_id)? {
|
.ok_or(Error::CorruptedTaskQueue)?;
|
||||||
if let Some(task_indexes) = task.indexes() {
|
if let Some(task_indexes) = task.indexes() {
|
||||||
affected_indexes.extend(task_indexes.into_iter().map(|x| x.to_owned()));
|
affected_indexes.extend(task_indexes.into_iter().map(|x| x.to_owned()));
|
||||||
}
|
|
||||||
affected_statuses.insert(task.status);
|
|
||||||
affected_kinds.insert(task.kind.as_kind());
|
|
||||||
}
|
}
|
||||||
|
affected_statuses.insert(task.status);
|
||||||
|
affected_kinds.insert(task.kind.as_kind());
|
||||||
|
// Note: don't delete the persisted task data since
|
||||||
|
// we can only delete succeeded, failed, and canceled tasks.
|
||||||
|
// In each of those cases, the persisted data is supposed to
|
||||||
|
// have been deleted already.
|
||||||
}
|
}
|
||||||
for index in affected_indexes {
|
for index in affected_indexes {
|
||||||
self.update_index(wtxn, &index, |bitmap| {
|
self.update_index(wtxn, &index, |bitmap| {
|
||||||
|
@ -381,7 +381,7 @@ impl IndexScheduler {
|
|||||||
match wtxn.commit() {
|
match wtxn.commit() {
|
||||||
Ok(()) => (),
|
Ok(()) => (),
|
||||||
_e @ Err(_) => {
|
_e @ Err(_) => {
|
||||||
todo!("remove the data associated with the task");
|
self.delete_persisted_task_data(&task)?;
|
||||||
// _e?;
|
// _e?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -577,8 +577,8 @@ impl IndexScheduler {
|
|||||||
for mut task in tasks {
|
for mut task in tasks {
|
||||||
task.started_at = Some(started_at);
|
task.started_at = Some(started_at);
|
||||||
task.finished_at = Some(finished_at);
|
task.finished_at = Some(finished_at);
|
||||||
// TODO the info field should've been set by the process_batch function
|
|
||||||
self.update_task(&mut wtxn, &task)?;
|
self.update_task(&mut wtxn, &task)?;
|
||||||
|
self.delete_persisted_task_data(&task)?;
|
||||||
}
|
}
|
||||||
log::info!("A batch of tasks was successfully completed.");
|
log::info!("A batch of tasks was successfully completed.");
|
||||||
}
|
}
|
||||||
@ -606,6 +606,25 @@ impl IndexScheduler {
|
|||||||
|
|
||||||
Ok(processed_tasks)
|
Ok(processed_tasks)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn delete_persisted_task_data(&self, task: &Task) -> Result<()> {
|
||||||
|
match &task.kind {
|
||||||
|
KindWithContent::DocumentImport { content_file, .. } => {
|
||||||
|
self.delete_update_file(*content_file)
|
||||||
|
}
|
||||||
|
KindWithContent::DocumentDeletion { .. }
|
||||||
|
| KindWithContent::DocumentClear { .. }
|
||||||
|
| KindWithContent::Settings { .. }
|
||||||
|
| KindWithContent::IndexDeletion { .. }
|
||||||
|
| KindWithContent::IndexCreation { .. }
|
||||||
|
| KindWithContent::IndexUpdate { .. }
|
||||||
|
| KindWithContent::IndexSwap { .. }
|
||||||
|
| KindWithContent::CancelTask { .. }
|
||||||
|
| KindWithContent::TaskDeletion { .. }
|
||||||
|
| KindWithContent::DumpExport { .. }
|
||||||
|
| KindWithContent::Snapshot => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -15,7 +15,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
|
|||||||
let IndexScheduler {
|
let IndexScheduler {
|
||||||
autobatching_enabled,
|
autobatching_enabled,
|
||||||
processing_tasks,
|
processing_tasks,
|
||||||
file_store: _,
|
file_store,
|
||||||
env,
|
env,
|
||||||
all_tasks,
|
all_tasks,
|
||||||
status,
|
status,
|
||||||
@ -59,11 +59,23 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
|
|||||||
snap.push_str(&snapshot_index_mapper(&rtxn, index_mapper));
|
snap.push_str(&snapshot_index_mapper(&rtxn, index_mapper));
|
||||||
snap.push_str("\n----------------------------------------------------------------------\n");
|
snap.push_str("\n----------------------------------------------------------------------\n");
|
||||||
|
|
||||||
|
snap.push_str("### File Store:\n");
|
||||||
|
snap.push_str(&snapshot_file_store(file_store));
|
||||||
|
snap.push_str("\n----------------------------------------------------------------------\n");
|
||||||
|
|
||||||
rtxn.commit().unwrap();
|
rtxn.commit().unwrap();
|
||||||
|
|
||||||
snap
|
snap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn snapshot_file_store(file_store: &file_store::FileStore) -> String {
|
||||||
|
let mut snap = String::new();
|
||||||
|
for uuid in file_store.__all_uuids() {
|
||||||
|
snap.push_str(&format!("{uuid}\n"));
|
||||||
|
}
|
||||||
|
snap
|
||||||
|
}
|
||||||
|
|
||||||
fn snapshot_bitmap(r: &RoaringBitmap) -> String {
|
fn snapshot_bitmap(r: &RoaringBitmap) -> String {
|
||||||
let mut snap = String::new();
|
let mut snap = String::new();
|
||||||
snap.push('[');
|
snap.push('[');
|
||||||
|
@ -20,4 +20,8 @@ doggos [0,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000000
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -20,4 +20,8 @@ doggos [0,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000000
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -21,4 +21,7 @@ doggos [0,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
["doggos"]
|
["doggos"]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -23,4 +23,7 @@ doggos [2,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -27,4 +27,7 @@ doggo [4,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -22,4 +22,9 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000000
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -23,4 +23,8 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
["catto"]
|
["catto"]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -25,4 +25,8 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
["catto"]
|
["catto"]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -22,4 +22,9 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000000
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -23,4 +23,8 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
["catto"]
|
["catto"]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -24,4 +24,8 @@ doggo [1,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
["catto"]
|
["catto"]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
00000000-0000-0000-0000-000000000001
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -24,4 +24,7 @@ doggo [2,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -27,4 +27,7 @@ doggo [2,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -26,4 +26,7 @@ doggo [2,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -26,4 +26,7 @@ doggo [2,]
|
|||||||
### Index Mapper:
|
### Index Mapper:
|
||||||
[]
|
[]
|
||||||
----------------------------------------------------------------------
|
----------------------------------------------------------------------
|
||||||
|
### File Store:
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user