From a71a96894dbfdef57114ec9b3f4baa82f4ab9b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Wed, 11 Nov 2020 13:14:16 +0100
Subject: [PATCH] Use the new indexing progress events in the http server

---
 http-ui/public/updates-script.js | 15 +++++++++++---
 http-ui/src/main.rs              | 35 ++++++++++++++++++++++++--------
 2 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/http-ui/public/updates-script.js b/http-ui/public/updates-script.js
index c034c45b8..5d439a7f5 100644
--- a/http-ui/public/updates-script.js
+++ b/http-ui/public/updates-script.js
@@ -54,13 +54,22 @@ $(window).on('load', function () {
       const content = $(`#${id} .updateStatus.content`);
 
       let html;
-      let { type, processed_number_of_documents, total_number_of_documents } = status.meta;
-      if (type === 'DocumentsAddition' && processed_number_of_documents && total_number_of_documents) {
-        let progress = Math.round(processed_number_of_documents / total_number_of_documents * 100);
+
+      let { type, step, total_steps, current, total } = status.meta;
+
+      if (type === 'DocumentsAddition') {
+        // If the total is null or undefined then the progress results is infinity.
+        let progress = Math.round(current / total * 100);
+        // We must divide the progress by the total number of indexing steps.
+        progress = progress / total_steps;
+        // And mark the previous steps as processed.
+        progress = progress + (step * 100 / total_steps);
+        // Generate the appropriate html bulma progress bar.
         html = `<progress class="progress" title="${progress}%" value="${progress}" max="100"></progress>`;
       } else {
         html = `<progress class="progress" max="100"></progress>`;
       }
+
       content.html(html);
     }
 
diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs
index 281eb0c8e..2af2c9d90 100644
--- a/http-ui/src/main.rs
+++ b/http-ui/src/main.rs
@@ -26,6 +26,7 @@ use warp::filters::ws::Message;
 use warp::{Filter, http::Response};
 
 use milli::tokenizer::{simple_tokenizer, TokenType};
+use milli::update::UpdateIndexingStep::*;
 use milli::update::{UpdateBuilder, IndexDocumentsMethod, UpdateFormat};
 use milli::{obkv_to_json, Index, UpdateStore, SearchResult};
 
@@ -201,8 +202,10 @@ enum UpdateMeta {
 #[serde(tag = "type")]
 enum UpdateMetaProgress {
     DocumentsAddition {
-        processed_number_of_documents: usize,
-        total_number_of_documents: Option<usize>,
+        step: usize,
+        total_steps: usize,
+        current: usize,
+        total: Option<usize>,
     },
 }
 
@@ -310,12 +313,20 @@ async fn main() -> anyhow::Result<()> {
                         Box::new(content) as Box<dyn io::Read>
                     };
 
-                    let result = builder.execute(reader, |count, total| {
+                    let result = builder.execute(reader, |indexing_step| {
+                        let (current, total) = match indexing_step {
+                            TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None),
+                            ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)),
+                            IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)),
+                            MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)),
+                        };
                         let _ = update_status_sender_cloned.send(UpdateStatus::Progressing {
                             update_id,
                             meta: UpdateMetaProgress::DocumentsAddition {
-                                processed_number_of_documents: count,
-                                total_number_of_documents: Some(total),
+                                step: indexing_step.step(),
+                                total_steps: indexing_step.number_of_steps(),
+                                current,
+                                total,
                             }
                         });
                     });
@@ -356,12 +367,20 @@ async fn main() -> anyhow::Result<()> {
                         }
                     }
 
-                    let result = builder.execute(|count, total| {
+                    let result = builder.execute(|indexing_step| {
+                        let (current, total) = match indexing_step {
+                            TransformFromUserIntoGenericFormat { documents_seen } => (documents_seen, None),
+                            ComputeIdsAndMergeDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)),
+                            IndexDocuments { documents_seen, total_documents } => (documents_seen, Some(total_documents)),
+                            MergeDataIntoFinalDatabase { databases_seen, total_databases } => (databases_seen, Some(total_databases)),
+                        };
                         let _ = update_status_sender_cloned.send(UpdateStatus::Progressing {
                             update_id,
                             meta: UpdateMetaProgress::DocumentsAddition {
-                                processed_number_of_documents: count,
-                                total_number_of_documents: Some(total),
+                                step: indexing_step.step(),
+                                total_steps: indexing_step.number_of_steps(),
+                                current,
+                                total,
                             }
                         });
                     });