From c317af58bcacf71f901279e2beeb734a14b9b308 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 12 Feb 2021 17:39:14 +0100 Subject: [PATCH] implement delete document batches --- src/data/updates.rs | 11 ++++++++++ .../local_index_controller/mod.rs | 15 ++++++++++--- .../local_index_controller/update_handler.rs | 22 +++++++++++++++++++ src/index_controller/mod.rs | 5 +++++ src/routes/document.rs | 22 +++++++++++++++---- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index d3566bb6f..1f3290e47 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -72,6 +72,17 @@ impl Data { Ok(update.into()) } + pub async fn delete_documents( + &self, + index: impl AsRef, + document_ids: Vec, + ) -> anyhow::Result { + let index_controller = self.index_controller.clone(); + let index = index.as_ref().to_string(); + let update = tokio::task::spawn_blocking(move || index_controller.delete_documents(index, document_ids)).await??; + Ok(update.into()) + } + #[inline] pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index a0c7e0805..c0903a5e1 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -43,7 +43,7 @@ impl IndexController for LocalIndexController { ) -> anyhow::Result> { let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; let meta = UpdateMeta::DocumentsAddition { method, format }; - let pending = update_store.register_update(meta, data).unwrap(); + let pending = update_store.register_update(meta, data)?; Ok(pending.into()) } @@ -54,7 +54,7 @@ impl IndexController for LocalIndexController { ) -> anyhow::Result> { let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; let meta = UpdateMeta::Settings(settings); - let pending = update_store.register_update(meta, &[]).unwrap(); + let pending = update_store.register_update(meta, &[])?; Ok(pending.into()) } @@ -187,7 +187,16 @@ impl IndexController for LocalIndexController { let (_, update_store) = self.indexes.index(&index)? .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; let meta = UpdateMeta::ClearDocuments; - let pending = update_store.register_update(meta, &[]).unwrap(); + let pending = update_store.register_update(meta, &[])?; + Ok(pending.into()) + } + + fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result { + let (_, update_store) = self.indexes.index(&index)? + .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; + let meta = UpdateMeta::DeleteDocuments; + let content = serde_json::to_vec(&document_ids)?; + let pending = update_store.register_update(meta, &content)?; Ok(pending.into()) } } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index c89b9f686..505e9aff0 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -177,6 +177,27 @@ impl UpdateHandler { Err(e) => Err(e.into()) } } + + fn delete_documents( + &self, + document_ids: &[u8], + update_builder: UpdateBuilder, + ) -> anyhow::Result { + let ids: Vec = serde_json::from_slice(document_ids)?; + let mut txn = self.index.write_txn()?; + let mut builder = update_builder.delete_documents(&mut txn, &self.index)?; + + // we ignore unexisting document ids + ids.iter().for_each(|id| { builder.delete_external_id(id); }); + + match builder.execute() { + Ok(deleted) => txn + .commit() + .and(Ok(UpdateResult::DocumentDeletion { deleted })) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } } impl HandleUpdate for UpdateHandler { @@ -194,6 +215,7 @@ impl HandleUpdate for UpdateHandler { let result = match meta.meta() { DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), ClearDocuments => self.clear_documents(update_builder), + DeleteDocuments => self.delete_documents(content, update_builder,), Settings(settings) => self.update_settings(settings, update_builder), Facets(levels) => self.update_facets(levels, update_builder), }; diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0ea71a72a..e811b6478 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -33,6 +33,7 @@ pub struct IndexMetadata { pub enum UpdateMeta { DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, ClearDocuments, + DeleteDocuments, Settings(Settings), Facets(Facets), } @@ -94,6 +95,7 @@ impl Settings { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { DocumentsAddition(DocumentAdditionResult), + DocumentDeletion { deleted: usize }, Other, } @@ -131,6 +133,9 @@ pub trait IndexController { /// Clear all documents in the given index. fn clear_documents(&self, index: impl AsRef) -> anyhow::Result; + /// Clear all documents in the given index. + fn delete_documents(&self, index: impl AsRef, document_ids: Vec) -> anyhow::Result; + /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; diff --git a/src/routes/document.rs b/src/routes/document.rs index 744a2e7f1..67ca2a02f 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -160,11 +160,25 @@ async fn update_documents( wrap = "Authentication::Private" )] async fn delete_documents( - _data: web::Data, - _path: web::Path, - _body: web::Json>, + data: web::Data, + path: web::Path, + body: web::Json>, ) -> Result { - todo!() + let ids = body + .iter() + .map(ToString::to_string) + .collect(); + + match data.delete_documents(&path.index_uid, ids).await { + Ok(result) => { + let json = serde_json::to_string(&result).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + unimplemented!() + } + } } #[delete("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]