From 47827ca5c188a138ad0580e6b55485dc25f5c361 Mon Sep 17 00:00:00 2001
From: Takahiro Ebato <takahiro.ebato@gmail.com>
Date: Fri, 20 Dec 2024 22:46:16 +0900
Subject: [PATCH] Add Prometheus metrics to measure task queue latency

---
 crates/meilisearch/src/metrics.rs        |  9 ++++--
 crates/meilisearch/src/routes/metrics.rs | 35 ++++++++++++++++++------
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs
index be9fbfc49..a48554e6c 100644
--- a/crates/meilisearch/src/metrics.rs
+++ b/crates/meilisearch/src/metrics.rs
@@ -1,7 +1,7 @@
 use lazy_static::lazy_static;
 use prometheus::{
-    opts, register_histogram_vec, register_int_counter_vec, register_int_gauge,
-    register_int_gauge_vec, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
+    opts, register_gauge, register_histogram_vec, register_int_counter_vec, register_int_gauge,
+    register_int_gauge_vec, Gauge, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
 };
 
 lazy_static! {
@@ -63,4 +63,9 @@ lazy_static! {
             "Meilisearch Searches Being Processed"
         ))
         .expect("Can't create a metric");
+    pub static ref MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS: Gauge = register_gauge!(
+        "meilisearch_task_queue_latency_seconds",
+        "Meilisearch Task Queue Latency in Seconds",
+    )
+    .expect("Can't create a metric");
 }
diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs
index 48b5d09f5..7dd9ee3bb 100644
--- a/crates/meilisearch/src/routes/metrics.rs
+++ b/crates/meilisearch/src/routes/metrics.rs
@@ -1,16 +1,17 @@
-use actix_web::http::header;
-use actix_web::web::{self, Data};
-use actix_web::HttpResponse;
-use index_scheduler::IndexScheduler;
-use meilisearch_auth::AuthController;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::keys::actions;
-use prometheus::{Encoder, TextEncoder};
-
 use crate::extractors::authentication::policies::ActionPolicy;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::routes::create_all_stats;
 use crate::search_queue::SearchQueue;
+use actix_web::http::header;
+use actix_web::web::{self, Data};
+use actix_web::HttpResponse;
+use index_scheduler::{IndexScheduler, Query};
+use meilisearch_auth::AuthController;
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::keys::actions;
+use meilisearch_types::tasks::Status;
+use prometheus::{Encoder, TextEncoder};
+use time::OffsetDateTime;
 
 pub fn configure(config: &mut web::ServiceConfig) {
     config.service(web::resource("").route(web::get().to(get_metrics)));
@@ -61,6 +62,22 @@ pub async fn get_metrics(
     }
     crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing()? as i64);
 
+    let task_queue_latency_seconds = index_scheduler
+        .get_tasks_from_authorized_indexes(
+            Query {
+                limit: Some(1),
+                reverse: Some(true),
+                statuses: Some(vec![Status::Enqueued, Status::Processing]),
+                ..Query::default()
+            },
+            auth_filters,
+        )?
+        .0
+        .first()
+        .map(|task| (OffsetDateTime::now_utc() - task.enqueued_at).as_seconds_f64())
+        .unwrap_or(0.0);
+    crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds);
+
     let encoder = TextEncoder::new();
     let mut buffer = vec![];
     encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics");