From c6b3c18c85e234929f517a2dbfd7dfcd01e71c36 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 24 Oct 2023 17:04:02 +0200 Subject: [PATCH] WIP: Comment out document deletion in other pipelines than update TODO: fix calls to DELETE route --- milli/src/update/delete_documents.rs | 10 ++++++---- milli/src/update/index_documents/mod.rs | 8 ++++---- milli/src/update/index_documents/transform.rs | 12 +++--------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index 9044f03be..0299e1e4f 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -255,12 +255,14 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { } // We acquire the current external documents ids map... // Note that its soft-deleted document ids field will be equal to the `to_delete_docids` - let mut new_external_documents_ids = self.index.external_documents_ids(self.wtxn)?; + //let mut new_external_documents_ids = self.index.external_documents_ids(self.wtxn)?; // We then remove the soft-deleted docids from it - new_external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; + //new_external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; // and write it back to the main database. - let new_external_documents_ids = new_external_documents_ids.into_static(); - self.index.put_external_documents_ids(self.wtxn, &new_external_documents_ids)?; + //let new_external_documents_ids = new_external_documents_ids.into_static(); + //self.index.put_external_documents_ids(self.wtxn, &new_external_documents_ids)?; + + todo!("please autobatch deletions for now"); let mut words_to_keep = BTreeSet::default(); let mut words_to_delete = BTreeSet::default(); diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index d1fa28826..8d187a89d 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -457,10 +457,10 @@ where self.index.put_primary_key(self.wtxn, &primary_key)?; // We write the external documents ids into the main database. - let mut external_documents_ids = self.index.external_documents_ids(self.wtxn)?; - external_documents_ids.insert_ids(&new_external_documents_ids)?; - let external_documents_ids = external_documents_ids.into_static(); - self.index.put_external_documents_ids(self.wtxn, &external_documents_ids)?; + //let mut external_documents_ids = self.index.external_documents_ids(self.wtxn)?; + //external_documents_ids.insert_ids(&new_external_documents_ids)?; + //let external_documents_ids = external_documents_ids.into_static(); + //self.index.put_external_documents_ids(self.wtxn, &external_documents_ids)?; // FIXME: remove `new_documents_ids` entirely and `replaced_documents_ids` let all_documents_ids = index_documents_ids | new_documents_ids; diff --git a/milli/src/update/index_documents/transform.rs b/milli/src/update/index_documents/transform.rs index 2b77768cb..e02da8cb5 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/milli/src/update/index_documents/transform.rs @@ -763,14 +763,6 @@ impl<'a, 'i> Transform<'a, 'i> { .to_string(); let field_distribution = self.index.field_distribution(wtxn)?; - // Delete the soft deleted document ids from the maps inside the external_document_ids structure - let new_external_documents_ids = { - let mut external_documents_ids = self.index.external_documents_ids(wtxn)?; - external_documents_ids.delete_soft_deleted_documents_ids_from_fsts()?; - // This call should be free and can't fail since the previous method merged both fsts. - external_documents_ids.into_static().to_fst()?.into_owned() - }; - let documents_ids = self.index.documents_ids(wtxn)?; let documents_count = documents_ids.len() as usize; @@ -858,8 +850,10 @@ impl<'a, 'i> Transform<'a, 'i> { primary_key, fields_ids_map: new_fields_ids_map, field_distribution, - new_external_documents_ids, + // FIXME: remove this now unused field + new_external_documents_ids: fst::Map::default().map_data(Cow::Owned).unwrap(), new_documents_ids: documents_ids, + // FIXME: remove this now unused field replaced_documents_ids: RoaringBitmap::default(), documents_count, original_documents: original_documents.into_inner().map_err(|err| err.into_error())?,