Merge #556

556: Add EXISTS filter r=loiclec a=loiclec

## What does this PR do?

Fixes issue [#2484](https://github.com/meilisearch/meilisearch/issues/2484) in the meilisearch repo.

It creates a `field EXISTS` filter which selects all documents containing the `field` key. 
For example, with the following documents:
```json
[{
	"id": 0,
	"colour": []
},
{
	"id": 1,
	"colour": ["blue", "green"]
},
{
	"id": 2,
	"colour": 145238
},
{
	"id": 3,
	"colour": null
},
{
	"id": 4,
	"colour": {
		"green": []
	}
},
{
	"id": 5,
	"colour": {}
},
{
	"id": 6
}]
```
Then the filter `colour EXISTS` selects the ids `[0, 1, 2, 3, 4, 5]`. The filter `colour NOT EXISTS` selects `[6]`.

## Details
There is a new database named `facet-id-exists-docids`. Its keys are field ids and its values are bitmaps of all the document ids where the corresponding field exists.

To create this database, the indexing part of milli had to be adapted. The implementation there is basically copy/pasted from the code handling the `facet-id-f64-docids` database, with appropriate modifications in place.

There was an issue involving the flattening of documents during (re)indexing. Previously, the following JSON:
```json
{
    "id": 0,
    "colour": [],
    "size": {}
}
```
would be flattened to:
```json
{
    "id": 0
}
```
prior to being given to the extraction pipeline.

This transformation would lose the information that is needed to populate the `facet-id-exists-docids` database. Therefore, I have also changed the implementation of the `flatten-serde-json` crate. Now, as it traverses the Json, it keeps track of which key was encountered. Then, at the end, if a previously encountered key is not present in the flattened object, it adds that key to the object with an empty array as value. For example:
```json
{
    "id": 0,
    "colour": {
        "green": [],
        "blue": 1
    },
    "size": {}
} 
```
becomes
```json
{
    "id": 0,
    "colour": [],
    "colour.green": [],
    "colour.blue": 1,
    "size": []
} 
```


Co-authored-by: Kerollmops <clement@meilisearch.com>

milli/src/update/index_documents/mod.rs

@@ -613,6 +613,7 @@ mod tests {
     use super::*;
     use crate::documents::DocumentsBatchBuilder;
     use crate::update::DeleteDocuments;
+    use crate::BEU16;
 
     #[test]
     fn simple_document_replacement() {
@@ -2040,6 +2041,109 @@ mod tests {
         assert_eq!(ids.len(), map.len());
     }
 
+    #[test]
+    fn index_documents_check_exists_database() {
+        let config = IndexerConfig::default();
+        let indexing_config = IndexDocumentsConfig::default();
+
+        let faceted_fields = hashset!(S("colour"));
+        let content = || {
+            documents!([
+                {
+                    "id": 0,
+                    "colour": 0,
+                },
+                {
+                    "id": 1,
+                    "colour": []
+                },
+                {
+                    "id": 2,
+                    "colour": {}
+                },
+                {
+                    "id": 3,
+                    "colour": null
+                },
+                {
+                    "id": 4,
+                    "colour": [1]
+                },
+                {
+                    "id": 5
+                },
+                {
+                    "id": 6,
+                    "colour": {
+                        "green": 1
+                    }
+                },
+                {
+                    "id": 7,
+                    "colour": {
+                        "green": {
+                            "blue": []
+                        }
+                    }
+                }
+            ])
+        };
+        let make_index = || {
+            let path = tempfile::tempdir().unwrap();
+            let mut options = EnvOpenOptions::new();
+            options.map_size(10 * 1024 * 1024); // 10 MB
+            Index::new(options, &path).unwrap()
+        };
+
+        let set_filterable_fields = |index: &Index| {
+            let mut wtxn = index.write_txn().unwrap();
+            let mut builder = update::Settings::new(&mut wtxn, &index, &config);
+            builder.set_filterable_fields(faceted_fields.clone());
+            builder.execute(|_| ()).unwrap();
+            wtxn.commit().unwrap();
+        };
+        let add_documents = |index: &Index| {
+            let mut wtxn = index.write_txn().unwrap();
+            let builder =
+                IndexDocuments::new(&mut wtxn, index, &config, indexing_config.clone(), |_| ())
+                    .unwrap();
+            let (builder, user_error) = builder.add_documents(content()).unwrap();
+            user_error.unwrap();
+            builder.execute().unwrap();
+            wtxn.commit().unwrap();
+        };
+
+        let check_ok = |index: &Index| {
+            let rtxn = index.read_txn().unwrap();
+            let facets = index.faceted_fields(&rtxn).unwrap();
+            assert_eq!(facets, hashset!(S("colour"), S("colour.green"), S("colour.green.blue")));
+
+            let colour_id = index.fields_ids_map(&rtxn).unwrap().id("colour").unwrap();
+            let colour_green_id = index.fields_ids_map(&rtxn).unwrap().id("colour.green").unwrap();
+
+            let bitmap_colour =
+                index.facet_id_exists_docids.get(&rtxn, &BEU16::new(colour_id)).unwrap().unwrap();
+            assert_eq!(bitmap_colour.into_iter().collect::<Vec<_>>(), vec![0, 1, 2, 3, 4, 6, 7]);
+
+            let bitmap_colour_green = index
+                .facet_id_exists_docids
+                .get(&rtxn, &BEU16::new(colour_green_id))
+                .unwrap()
+                .unwrap();
+            assert_eq!(bitmap_colour_green.into_iter().collect::<Vec<_>>(), vec![6, 7]);
+        };
+
+        let index = make_index();
+        add_documents(&index);
+        set_filterable_fields(&index);
+        check_ok(&index);
+
+        let index = make_index();
+        set_filterable_fields(&index);
+        add_documents(&index);
+        check_ok(&index);
+    }
+
     #[test]
     fn primary_key_must_not_contain_floats() {
         let tmp = tempfile::tempdir().unwrap();