From 47827ca5c188a138ad0580e6b55485dc25f5c361 Mon Sep 17 00:00:00 2001 From: Takahiro Ebato Date: Fri, 20 Dec 2024 22:46:16 +0900 Subject: [PATCH 1/2] Add Prometheus metrics to measure task queue latency --- crates/meilisearch/src/metrics.rs | 9 ++++-- crates/meilisearch/src/routes/metrics.rs | 35 ++++++++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index be9fbfc49..a48554e6c 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -1,7 +1,7 @@ use lazy_static::lazy_static; use prometheus::{ - opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, - register_int_gauge_vec, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, + opts, register_gauge, register_histogram_vec, register_int_counter_vec, register_int_gauge, + register_int_gauge_vec, Gauge, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, }; lazy_static! { @@ -63,4 +63,9 @@ lazy_static! { "Meilisearch Searches Being Processed" )) .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS: Gauge = register_gauge!( + "meilisearch_task_queue_latency_seconds", + "Meilisearch Task Queue Latency in Seconds", + ) + .expect("Can't create a metric"); } diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 48b5d09f5..7dd9ee3bb 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -1,16 +1,17 @@ -use actix_web::http::header; -use actix_web::web::{self, Data}; -use actix_web::HttpResponse; -use index_scheduler::IndexScheduler; -use meilisearch_auth::AuthController; -use meilisearch_types::error::ResponseError; -use meilisearch_types::keys::actions; -use prometheus::{Encoder, TextEncoder}; - use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::routes::create_all_stats; use crate::search_queue::SearchQueue; +use actix_web::http::header; +use actix_web::web::{self, Data}; +use actix_web::HttpResponse; +use index_scheduler::{IndexScheduler, Query}; +use meilisearch_auth::AuthController; +use meilisearch_types::error::ResponseError; +use meilisearch_types::keys::actions; +use meilisearch_types::tasks::Status; +use prometheus::{Encoder, TextEncoder}; +use time::OffsetDateTime; pub fn configure(config: &mut web::ServiceConfig) { config.service(web::resource("").route(web::get().to(get_metrics))); @@ -61,6 +62,22 @@ pub async fn get_metrics( } crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing()? as i64); + let task_queue_latency_seconds = index_scheduler + .get_tasks_from_authorized_indexes( + Query { + limit: Some(1), + reverse: Some(true), + statuses: Some(vec![Status::Enqueued, Status::Processing]), + ..Query::default() + }, + auth_filters, + )? + .0 + .first() + .map(|task| (OffsetDateTime::now_utc() - task.enqueued_at).as_seconds_f64()) + .unwrap_or(0.0); + crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds); + let encoder = TextEncoder::new(); let mut buffer = vec![]; encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); From 195785c47f1f528443f92674814e7ed5063ad6a8 Mon Sep 17 00:00:00 2001 From: Takahiro Ebato Date: Fri, 27 Dec 2024 23:26:20 +0900 Subject: [PATCH 2/2] Add a task queue latency panel to the grafana dashboard --- assets/grafana-dashboard.json | 98 +++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/assets/grafana-dashboard.json b/assets/grafana-dashboard.json index 2cfa85a46..027afcbba 100644 --- a/assets/grafana-dashboard.json +++ b/assets/grafana-dashboard.json @@ -1403,6 +1403,104 @@ "title": "Number of tasks by indexes", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_latency_seconds{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue latency", + "type": "timeseries" + }, { "collapsed": true, "datasource": {