Merge #5325

5325: Documents database stats r=irevoire a=ManyTheFish

# Pull Request

## Related issue
Fixes #5319

## List

- Create a DatabaseStats struct
- Compute and store the documents database stats in the IndexStats
- Force dumpless upgrade to update the index stats
- when a document addition/modification/deletion is made, we only recompute the database stats on the added/modified/deleted documents

Co-authored-by: ManyTheFish <many@meilisearch.com>
Co-authored-by: Many the fish <many@meilisearch.com>

crates/meilisearch/src/routes/indexes/mod.rs

@@ -494,6 +494,10 @@ pub async fn delete_index(
 pub struct IndexStats {
     /// Number of documents in the index
     pub number_of_documents: u64,
+    /// Size of the documents database, in bytes.
+    pub raw_document_db_size: u64,
+    /// Average size of a document in the documents database.
+    pub avg_document_size: u64,
     /// Whether or not the index is currently ingesting document
     pub is_indexing: bool,
     /// Number of embeddings in the index
@@ -510,7 +514,9 @@ pub struct IndexStats {
 impl From<index_scheduler::IndexStats> for IndexStats {
     fn from(stats: index_scheduler::IndexStats) -> Self {
         IndexStats {
-            number_of_documents: stats.inner_stats.number_of_documents,
+            number_of_documents: stats.inner_stats.documents_database_stats.number_of_entries(),
+            raw_document_db_size: stats.inner_stats.documents_database_stats.total_value_size(),
+            avg_document_size: stats.inner_stats.documents_database_stats.average_value_size(),
             is_indexing: stats.is_indexing,
             number_of_embeddings: stats.inner_stats.number_of_embeddings,
             number_of_embedded_documents: stats.inner_stats.number_of_embedded_documents,
@@ -532,6 +538,8 @@ impl From<index_scheduler::IndexStats> for IndexStats {
         (status = OK, description = "The stats of the index", body = IndexStats, content_type = "application/json", example = json!(
             {
                 "numberOfDocuments": 10,
+                "rawDocumentDbSize": 10,
+                "avgDocumentSize": 10,
                 "numberOfEmbeddings": 10,
                 "numberOfEmbeddedDocuments": 10,
                 "isIndexing": true,

crates/meilisearch/src/routes/mod.rs

@@ -392,6 +392,9 @@ pub struct Stats {
                 "indexes": {
                     "movies": {
                         "numberOfDocuments": 10,
+                        "rawDocumentDbSize": 100,
+                        "maxDocumentSize": 16,
+                        "avgDocumentSize": 10,
                         "isIndexing": true,
                         "fieldDistribution": {
                             "genre": 10,

crates/meilisearch/tests/documents/delete_documents.rs

@@ -160,6 +160,8 @@ async fn delete_document_by_filter() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 4,
+      "rawDocumentDbSize": 42,
+      "avgDocumentSize": 10,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -209,6 +211,8 @@ async fn delete_document_by_filter() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 16,
+      "avgDocumentSize": 8,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -277,6 +281,8 @@ async fn delete_document_by_filter() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 1,
+      "rawDocumentDbSize": 12,
+      "avgDocumentSize": 12,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,

crates/meilisearch/tests/dumps/mod.rs

@@ -32,6 +32,8 @@ async fn import_dump_v1_movie_raw() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -187,6 +189,8 @@ async fn import_dump_v1_movie_with_settings() {
         @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -355,6 +359,8 @@ async fn import_dump_v1_rubygems_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 8606,
+      "avgDocumentSize": 162,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -520,6 +526,8 @@ async fn import_dump_v2_movie_raw() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -675,6 +683,8 @@ async fn import_dump_v2_movie_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -840,6 +850,8 @@ async fn import_dump_v2_rubygems_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 8606,
+      "avgDocumentSize": 162,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1002,6 +1014,8 @@ async fn import_dump_v3_movie_raw() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1157,6 +1171,8 @@ async fn import_dump_v3_movie_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1322,6 +1338,8 @@ async fn import_dump_v3_rubygems_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 8606,
+      "avgDocumentSize": 162,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1484,6 +1502,8 @@ async fn import_dump_v4_movie_raw() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1639,6 +1659,8 @@ async fn import_dump_v4_movie_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 21965,
+      "avgDocumentSize": 414,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1804,6 +1826,8 @@ async fn import_dump_v4_rubygems_with_settings() {
       @r###"
     {
       "numberOfDocuments": 53,
+      "rawDocumentDbSize": 8606,
+      "avgDocumentSize": 162,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -1973,6 +1997,8 @@ async fn import_dump_v5() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 10,
+      "rawDocumentDbSize": 6782,
+      "avgDocumentSize": 678,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -2009,6 +2035,8 @@ async fn import_dump_v5() {
       @r###"
     {
       "numberOfDocuments": 10,
+      "rawDocumentDbSize": 6782,
+      "avgDocumentSize": 678,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,

crates/meilisearch/tests/stats/mod.rs

@@ -113,6 +113,8 @@ async fn add_remove_embeddings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 27,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 5,
       "numberOfEmbeddedDocuments": 2,
@@ -136,6 +138,8 @@ async fn add_remove_embeddings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 27,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 3,
       "numberOfEmbeddedDocuments": 2,
@@ -159,6 +163,8 @@ async fn add_remove_embeddings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 27,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 2,
       "numberOfEmbeddedDocuments": 2,
@@ -183,6 +189,8 @@ async fn add_remove_embeddings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 27,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 2,
       "numberOfEmbeddedDocuments": 1,
@@ -231,6 +239,8 @@ async fn add_remove_embedded_documents() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 27,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 5,
       "numberOfEmbeddedDocuments": 2,
@@ -250,6 +260,8 @@ async fn add_remove_embedded_documents() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 1,
+      "rawDocumentDbSize": 13,
+      "avgDocumentSize": 13,
       "isIndexing": false,
       "numberOfEmbeddings": 3,
       "numberOfEmbeddedDocuments": 1,
@@ -281,6 +293,8 @@ async fn update_embedder_settings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 108,
+      "avgDocumentSize": 54,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,
@@ -315,6 +329,8 @@ async fn update_embedder_settings() {
     snapshot!(json_string!(stats), @r###"
     {
       "numberOfDocuments": 2,
+      "rawDocumentDbSize": 108,
+      "avgDocumentSize": 54,
       "isIndexing": false,
       "numberOfEmbeddings": 3,
       "numberOfEmbeddedDocuments": 2,

crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs

@@ -134,6 +134,8 @@ async fn check_the_index_scheduler(server: &Server) {
       "indexes": {
         "kefir": {
           "numberOfDocuments": 1,
+          "rawDocumentDbSize": 109,
+          "avgDocumentSize": 109,
           "isIndexing": false,
           "numberOfEmbeddings": 0,
           "numberOfEmbeddedDocuments": 0,
@@ -216,6 +218,8 @@ async fn check_the_index_scheduler(server: &Server) {
       "indexes": {
         "kefir": {
           "numberOfDocuments": 1,
+          "rawDocumentDbSize": 109,
+          "avgDocumentSize": 109,
           "isIndexing": false,
           "numberOfEmbeddings": 0,
           "numberOfEmbeddedDocuments": 0,
@@ -235,6 +239,8 @@ async fn check_the_index_scheduler(server: &Server) {
     snapshot!(stats, @r###"
     {
       "numberOfDocuments": 1,
+      "rawDocumentDbSize": 109,
+      "avgDocumentSize": 109,
       "isIndexing": false,
       "numberOfEmbeddings": 0,
       "numberOfEmbeddedDocuments": 0,