From 92ec908303a5dc6f0aa7347f76a7e5fd141a7a9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Wed, 11 Nov 2020 16:04:04 +0100
Subject: [PATCH] Introduce the facet field id values engine database

---
 src/index.rs                   | 15 +++++++++++++--
 src/update/clear_documents.rs  | 24 ++++++++----------------
 src/update/delete_documents.rs | 19 ++++++++++++++++++-
 3 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/src/index.rs b/src/index.rs
index 1a0c5d4d6..49f877d81 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -34,22 +34,33 @@ pub struct Index {
     pub docid_word_positions: Database<BEU32StrCodec, BoRoaringBitmapCodec>,
     /// Maps the proximity between a pair of words with all the docids where this relation appears.
     pub word_pair_proximity_docids: Database<StrStrU8Codec, CboRoaringBitmapCodec>,
+    /// Maps the facet field id and the globally ordered value with the docids that corresponds to it.
+    pub facet_field_id_value_docids: Database<ByteSlice, CboRoaringBitmapCodec>,
     /// Maps the document id to the document as an obkv store.
     pub documents: Database<OwnedType<BEU32>, ObkvCodec>,
 }
 
 impl Index {
     pub fn new<P: AsRef<Path>>(mut options: heed::EnvOpenOptions, path: P) -> anyhow::Result<Index> {
-        options.max_dbs(5);
+        options.max_dbs(6);
 
         let env = options.open(path)?;
         let main = env.create_poly_database(Some("main"))?;
         let word_docids = env.create_database(Some("word-docids"))?;
         let docid_word_positions = env.create_database(Some("docid-word-positions"))?;
         let word_pair_proximity_docids = env.create_database(Some("word-pair-proximity-docids"))?;
+        let facet_field_id_value_docids = env.create_database(Some("facet-field-id-value-docids"))?;
         let documents = env.create_database(Some("documents"))?;
 
-        Ok(Index { env, main, word_docids, docid_word_positions, word_pair_proximity_docids, documents })
+        Ok(Index {
+            env,
+            main,
+            word_docids,
+            docid_word_positions,
+            word_pair_proximity_docids,
+            facet_field_id_value_docids,
+            documents,
+        })
     }
 
     /// Create a write transaction to be able to write into the index.
diff --git a/src/update/clear_documents.rs b/src/update/clear_documents.rs
index ae739bd0b..c49ae9104 100644
--- a/src/update/clear_documents.rs
+++ b/src/update/clear_documents.rs
@@ -18,33 +18,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
             word_docids,
             docid_word_positions,
             word_pair_proximity_docids,
+            facet_field_id_value_docids,
             documents,
         } = self.index;
 
-        // We clear the word fst.
+        // We retrieve the number of documents ids that we are deleting.
+        let number_of_documents = self.index.number_of_documents(self.wtxn)?;
+
+        // We clean some of the main engine datastructures.
         self.index.put_words_fst(self.wtxn, &fst::Set::default())?;
-
-        // We clear the users ids documents ids.
         self.index.put_users_ids_documents_ids(self.wtxn, &fst::Map::default())?;
-
-        // We retrieve the documents ids.
-        let documents_ids = self.index.documents_ids(self.wtxn)?;
-
-        // We clear the internal documents ids.
         self.index.put_documents_ids(self.wtxn, &RoaringBitmap::default())?;
 
-        // We clear the word docids.
+        // Clear the other databases.
         word_docids.clear(self.wtxn)?;
-
-        // We clear the docid word positions.
         docid_word_positions.clear(self.wtxn)?;
-
-        // We clear the word pair proximity docids.
         word_pair_proximity_docids.clear(self.wtxn)?;
-
-        // We clear the documents themselves.
+        facet_field_id_value_docids.clear(self.wtxn)?;
         documents.clear(self.wtxn)?;
 
-        Ok(documents_ids.len() as usize)
+        Ok(number_of_documents)
     }
 }
diff --git a/src/update/delete_documents.rs b/src/update/delete_documents.rs
index e3a3be15a..d68bca81c 100644
--- a/src/update/delete_documents.rs
+++ b/src/update/delete_documents.rs
@@ -76,6 +76,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             word_docids,
             docid_word_positions,
             word_pair_proximity_docids,
+            facet_field_id_value_docids,
             documents,
         } = self.index;
 
@@ -158,7 +159,9 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
         }
 
         // We construct an FST set that contains the words to delete from the words FST.
-        let words_to_delete = words.iter().filter_map(|(w, d)| if *d { Some(w.as_ref()) } else { None });
+        let words_to_delete = words.iter().filter_map(|(word, must_remove)| {
+            if *must_remove { Some(word.as_ref()) } else { None }
+        });
         let words_to_delete = fst::Set::from_iter(words_to_delete)?;
 
         let new_words_fst = {
@@ -191,6 +194,20 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             }
         }
 
+        drop(iter);
+
+        // We delete the documents ids that are under the facet field id values.
+        let mut iter = facet_field_id_value_docids.iter_mut(self.wtxn)?;
+        while let Some(result) = iter.next() {
+            let (bytes, mut docids) = result?;
+            docids.difference_with(&self.documents_ids);
+            if docids.is_empty() {
+                iter.del_current()?;
+            } else {
+                iter.put_current(bytes, &docids)?;
+            }
+        }
+
         Ok(self.documents_ids.len() as usize)
     }
 }