From 047eba3ff3c9e36915216520cba4ba6ac75c53a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 23 Oct 2019 16:32:11 +0200 Subject: [PATCH] Introduce a clear all documents update --- meilidb-core/src/store/docs_words.rs | 4 +++ meilidb-core/src/store/documents_fields.rs | 4 +++ .../src/store/documents_fields_counts.rs | 4 +++ meilidb-core/src/store/mod.rs | 4 +++ meilidb-core/src/update/clear_all.rs | 36 +++++++++++++++++++ meilidb-core/src/update/documents_addition.rs | 1 + meilidb-core/src/update/mod.rs | 19 ++++++++++ 7 files changed, 72 insertions(+) create mode 100644 meilidb-core/src/update/clear_all.rs diff --git a/meilidb-core/src/store/docs_words.rs b/meilidb-core/src/store/docs_words.rs index b7ef6e542..2dbd805fe 100644 --- a/meilidb-core/src/store/docs_words.rs +++ b/meilidb-core/src/store/docs_words.rs @@ -26,6 +26,10 @@ impl DocsWords { self.docs_words.delete(writer, &document_id) } + pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.docs_words.clear(writer) + } + pub fn doc_words( self, reader: &heed::RoTxn, diff --git a/meilidb-core/src/store/documents_fields.rs b/meilidb-core/src/store/documents_fields.rs index 17fa20380..f4c218501 100644 --- a/meilidb-core/src/store/documents_fields.rs +++ b/meilidb-core/src/store/documents_fields.rs @@ -32,6 +32,10 @@ impl DocumentsFields { self.documents_fields.delete_range(writer, start..=end) } + pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.documents_fields.clear(writer) + } + pub fn document_attribute<'txn>( self, reader: &'txn heed::RoTxn, diff --git a/meilidb-core/src/store/documents_fields_counts.rs b/meilidb-core/src/store/documents_fields_counts.rs index 9ba9c4478..2070cff6e 100644 --- a/meilidb-core/src/store/documents_fields_counts.rs +++ b/meilidb-core/src/store/documents_fields_counts.rs @@ -32,6 +32,10 @@ impl DocumentsFieldsCounts { .delete_range(writer, start..=end) } + pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { + self.documents_fields_counts.clear(writer) + } + pub fn document_field_count( self, reader: &heed::RoTxn, diff --git a/meilidb-core/src/store/mod.rs b/meilidb-core/src/store/mod.rs index ad14d46c5..db3051ddd 100644 --- a/meilidb-core/src/store/mod.rs +++ b/meilidb-core/src/store/mod.rs @@ -166,6 +166,10 @@ impl Index { ) } + pub fn clear_all(&self, writer: &mut heed::RwTxn) -> MResult { + update::push_clear_all(writer, self.updates, self.updates_results) + } + pub fn synonyms_addition(&self) -> update::SynonymsAddition { update::SynonymsAddition::new( self.updates, diff --git a/meilidb-core/src/update/clear_all.rs b/meilidb-core/src/update/clear_all.rs new file mode 100644 index 000000000..b4b1ab9bd --- /dev/null +++ b/meilidb-core/src/update/clear_all.rs @@ -0,0 +1,36 @@ +use meilidb_schema::{Diff, Schema}; + +use crate::update::documents_addition::reindex_all_documents; +use crate::update::{next_update_id, Update}; +use crate::{error::UnsupportedOperation, store, MResult, RankedMap}; + +pub fn apply_clear_all( + writer: &mut heed::RwTxn, + main_store: store::Main, + documents_fields_store: store::DocumentsFields, + documents_fields_counts_store: store::DocumentsFieldsCounts, + postings_lists_store: store::PostingsLists, + docs_words_store: store::DocsWords, +) -> MResult<()> { + main_store.put_words_fst(writer, &fst::Set::default())?; + main_store.put_ranked_map(writer, &RankedMap::default())?; + main_store.put_number_of_documents(writer, |_| 0)?; + documents_fields_store.clear(writer)?; + documents_fields_counts_store.clear(writer)?; + postings_lists_store.clear(writer)?; + docs_words_store.clear(writer)?; + + Ok(()) +} + +pub fn push_clear_all( + writer: &mut heed::RwTxn, + updates_store: store::Updates, + updates_results_store: store::UpdatesResults, +) -> MResult { + let last_update_id = next_update_id(writer, updates_store, updates_results_store)?; + let update = Update::ClearAll; + updates_store.put_update(writer, last_update_id, &update)?; + + Ok(last_update_id) +} diff --git a/meilidb-core/src/update/documents_addition.rs b/meilidb-core/src/update/documents_addition.rs index 1b02b5598..1973ad903 100644 --- a/meilidb-core/src/update/documents_addition.rs +++ b/meilidb-core/src/update/documents_addition.rs @@ -184,6 +184,7 @@ pub fn reindex_all_documents( main_store.put_ranked_map(writer, &ranked_map)?; main_store.put_number_of_documents(writer, |_| 0)?; postings_lists_store.clear(writer)?; + docs_words_store.clear(writer)?; // 3. re-index one document by one document (otherwise we make the borrow checker unhappy) let mut indexer = RawIndexer::new(); diff --git a/meilidb-core/src/update/mod.rs b/meilidb-core/src/update/mod.rs index 1e8a95621..abb19995f 100644 --- a/meilidb-core/src/update/mod.rs +++ b/meilidb-core/src/update/mod.rs @@ -1,3 +1,4 @@ +mod clear_all; mod customs_update; mod documents_addition; mod documents_deletion; @@ -5,6 +6,7 @@ mod schema_update; mod synonyms_addition; mod synonyms_deletion; +pub use self::clear_all::{apply_clear_all, push_clear_all}; pub use self::customs_update::{apply_customs_update, push_customs_update}; pub use self::documents_addition::{apply_documents_addition, DocumentsAddition}; pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion}; @@ -25,6 +27,7 @@ use meilidb_schema::Schema; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Update { + ClearAll, Schema(Schema), Customs(Vec), DocumentsAddition(Vec), @@ -35,6 +38,7 @@ pub enum Update { #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateType { + ClearAll, Schema { schema: Schema }, Customs, DocumentsAddition { number: usize }, @@ -107,6 +111,21 @@ pub fn update_task(writer: &mut heed::RwTxn, index: store::Index) -> MResult { + let start = Instant::now(); + + let update_type = UpdateType::ClearAll; + let result = apply_clear_all( + writer, + index.main, + index.documents_fields, + index.documents_fields_counts, + index.postings_lists, + index.docs_words, + ); + + (update_type, result, start.elapsed()) + } Update::Schema(schema) => { let start = Instant::now();