Merge #2657

2657: prometheus and grafana dashboards implemented r=irevoire a=pavo-tusker

Implemented Basic Prometheus Metrics and Grafana Dashboard using this [Prometheus Crate](https://crates.io/crates/prometheus) [#496](https://github.com/meilisearch/product/issues/496)
![Screenshot from 2022-08-04 19-59-06](https://user-images.githubusercontent.com/43550760/182880420-71ec8591-a2cb-4fd5-b1c5-911a6dcbdaf9.png)
![Screenshot from 2022-08-04 19-58-56](https://user-images.githubusercontent.com/43550760/182880433-11727814-e230-44dd-89c9-fec3baa47b11.png)
![Screenshot from 2022-08-04 19-58-40](https://user-images.githubusercontent.com/43550760/182880436-73312a68-4f20-49f0-80e9-5e344f96db6f.png)


Co-authored-by: mohandasspat <mohan.s@pavo-tusker.com>
Co-authored-by: Pavo-Tusker <43550760+pavo-tusker@users.noreply.github.com>
Co-authored-by: Tamo <tamo@meilisearch.com>

Cargo.lock

@@ -2048,6 +2048,7 @@ dependencies = [
  "indexmap",
  "itertools",
  "jsonwebtoken",
+ "lazy_static",
  "log",
  "manifest-dir-macros",
  "maplit",
@@ -2061,6 +2062,7 @@ dependencies = [
  "parking_lot",
  "pin-project-lite",
  "platform-dirs",
+ "prometheus",
  "rand",
  "rayon",
  "regex",
@@ -2667,6 +2669,36 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "procfs"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "hex",
+ "lazy_static",
+ "libc",
+]
+
+[[package]]
+name = "prometheus"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39"
+dependencies = [
+ "cfg-if 1.0.0",
+ "fnv",
+ "lazy_static",
+ "libc",
+ "memchr",
+ "parking_lot",
+ "procfs",
+ "protobuf",
+ "thiserror",
+]
+
 [[package]]
 name = "proptest"
 version = "1.0.0"
@@ -2698,6 +2730,12 @@ dependencies = [
  "syn 0.15.44",
 ]
 
+[[package]]
+name = "protobuf"
+version = "2.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"

meilisearch-http/Cargo.toml

@@ -77,6 +77,8 @@ tokio = { version = "1.17.0", features = ["full"] }
 tokio-stream = "0.1.8"
 uuid = { version = "1.1.2", features = ["serde", "v4"] }
 walkdir = "2.3.2"
+prometheus = { version = "0.13.0", features = ["process"] }
+lazy_static = "1.4.0"
 
 [dev-dependencies]
 actix-rt = "2.7.0"

meilisearch-http/src/lib.rs

@@ -5,7 +5,9 @@ pub mod analytics;
 pub mod task;
 #[macro_use]
 pub mod extractors;
+pub mod metrics;
 pub mod option;
+pub mod route_metrics;
 pub mod routes;
 
 use std::sync::{atomic::AtomicBool, Arc};
@@ -140,22 +142,33 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
     config.service(web::resource("/").route(web::get().to(routes::running)));
 }
 
+pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) {
+    if enable_metrics_route {
+        config.service(web::resource("/metrics").route(web::get().to(routes::get_metrics)));
+    }
+}
+
 #[macro_export]
 macro_rules! create_app {
     ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{
         use actix_cors::Cors;
+        use actix_web::dev::Service;
+        use actix_web::middleware::Condition;
         use actix_web::middleware::TrailingSlash;
         use actix_web::App;
         use actix_web::{middleware, web};
         use meilisearch_http::error::MeilisearchHttpError;
+        use meilisearch_http::metrics;
+        use meilisearch_http::route_metrics;
         use meilisearch_http::routes;
-        use meilisearch_http::{configure_data, dashboard};
+        use meilisearch_http::{configure_data, configure_metrics_route, dashboard};
         use meilisearch_types::error::ResponseError;
 
         App::new()
             .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics))
             .configure(routes::configure)
             .configure(|s| dashboard(s, $enable_frontend))
+            .configure(|s| configure_metrics_route(s, $opt.enable_metrics_route))
             .wrap(
                 Cors::default()
                     .send_wildcard()
@@ -169,5 +182,9 @@ macro_rules! create_app {
             .wrap(middleware::NormalizePath::new(
                 middleware::TrailingSlash::Trim,
             ))
+            .wrap(Condition::new(
+                $opt.enable_metrics_route,
+                route_metrics::RouteMetrics,
+            ))
     }};
 }

meilisearch-http/src/metrics.rs

@@ -0,0 +1,41 @@
+use lazy_static::lazy_static;
+use prometheus::{
+    opts, register_histogram_vec, register_int_counter_vec, register_int_gauge,
+    register_int_gauge_vec,
+};
+use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec};
+
+const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[
+    0.0005, 0.0008, 0.00085, 0.0009, 0.00095, 0.001, 0.00105, 0.0011, 0.00115, 0.0012, 0.0015,
+    0.002, 0.003, 1.0,
+];
+
+lazy_static! {
+    pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
+        opts!("http_requests_total", "HTTP requests total"),
+        &["method", "path"]
+    )
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!(opts!(
+        "meilisearch_database_size",
+        "MeiliSearch Stats DbSize"
+    ))
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!(opts!(
+        "meilisearch_total_index",
+        "MeiliSearch Stats Index Count"
+    ))
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!(
+        opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"),
+        &["index"]
+    )
+    .expect("Can't create a metric");
+    pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!(
+        "http_response_time_seconds",
+        "HTTP response times",
+        &["method", "path"],
+        HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec()
+    )
+    .expect("Can't create a metric");
+}

meilisearch-http/src/option.rs

@@ -146,6 +146,10 @@ pub struct Opt {
     #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
     pub log_level: String,
 
+    /// Enables Prometheus metrics and /metrics route.
+    #[clap(long)]
+    pub enable_metrics_route: bool,
+
     #[serde(flatten)]
     #[clap(flatten)]
     pub indexer_options: IndexerOpts,

meilisearch-http/src/route_metrics.rs

@@ -0,0 +1,75 @@
+use std::future::{ready, Ready};
+
+use actix_web::{
+    dev::{self, Service, ServiceRequest, ServiceResponse, Transform},
+    Error,
+};
+use futures_util::future::LocalBoxFuture;
+use prometheus::HistogramTimer;
+
+pub struct RouteMetrics;
+
+// Middleware factory is `Transform` trait from actix-service crate
+// `S` - type of the next service
+// `B` - type of response's body
+impl<S, B> Transform<S, ServiceRequest> for RouteMetrics
+where
+    S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+    B: 'static,
+{
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type InitError = ();
+    type Transform = RouteMetricsMiddleware<S>;
+    type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ready(Ok(RouteMetricsMiddleware { service }))
+    }
+}
+
+pub struct RouteMetricsMiddleware<S> {
+    service: S,
+}
+
+impl<S, B> Service<ServiceRequest> for RouteMetricsMiddleware<S>
+where
+    S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error>,
+    S::Future: 'static,
+    B: 'static,
+{
+    type Response = ServiceResponse<B>;
+    type Error = Error;
+    type Future = LocalBoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+    dev::forward_ready!(service);
+
+    fn call(&self, req: ServiceRequest) -> Self::Future {
+        let mut histogram_timer: Option<HistogramTimer> = None;
+        let request_path = req.path();
+        let is_registered_resource = req.resource_map().has_resource(request_path);
+        if is_registered_resource {
+            let request_method = req.method().to_string();
+            histogram_timer = Some(
+                crate::metrics::HTTP_RESPONSE_TIME_SECONDS
+                    .with_label_values(&[&request_method, request_path])
+                    .start_timer(),
+            );
+            crate::metrics::HTTP_REQUESTS_TOTAL
+                .with_label_values(&[&request_method, request_path])
+                .inc();
+        }
+
+        let fut = self.service.call(req);
+
+        Box::pin(async move {
+            let res = fut.await?;
+
+            if let Some(histogram_timer) = histogram_timer {
+                histogram_timer.observe_duration();
+            };
+            Ok(res)
+        })
+    }
+}

meilisearch-http/src/routes/mod.rs

@@ -1,3 +1,4 @@
+use actix_web::http::header::{self};
 use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
 use serde::{Deserialize, Serialize};
@@ -12,6 +13,7 @@ use meilisearch_types::star_or::StarOr;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::{policies::*, GuardedData};
+use prometheus::{Encoder, TextEncoder};
 
 mod api_key;
 mod dump;
@@ -278,3 +280,31 @@ struct KeysResponse {
 pub async fn get_health() -> Result<HttpResponse, ResponseError> {
     Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
 }
+
+pub async fn get_metrics(
+    meilisearch: GuardedData<ActionPolicy<{ actions::STATS_GET }>, MeiliSearch>,
+) -> Result<HttpResponse, ResponseError> {
+    let search_rules = &meilisearch.filters().search_rules;
+    let response = meilisearch.get_all_stats(search_rules).await?;
+
+    crate::metrics::MEILISEARCH_DB_SIZE.set(response.database_size as i64);
+    crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64);
+
+    for (index, value) in response.indexes.iter() {
+        crate::metrics::MEILISEARCH_DOCS_COUNT
+            .with_label_values(&[index])
+            .set(value.number_of_documents as i64);
+    }
+
+    let encoder = TextEncoder::new();
+    let mut buffer = vec![];
+    encoder
+        .encode(&prometheus::gather(), &mut buffer)
+        .expect("Failed to encode metrics");
+
+    let response = String::from_utf8(buffer).expect("Failed to convert bytes to string");
+
+    Ok(HttpResponse::Ok()
+        .insert_header(header::ContentType(mime::TEXT_PLAIN))
+        .body(response))
+}

meilisearch-http/tests/auth/payload.rs

@@ -19,7 +19,7 @@ async fn error_api_key_bad_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -91,7 +91,7 @@ async fn error_api_key_empty_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -163,7 +163,7 @@ async fn error_api_key_missing_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -227,7 +227,7 @@ async fn error_api_key_empty_payload() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -283,7 +283,7 @@ async fn error_api_key_malformed_payload() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;

meilisearch-http/tests/common/service.rs

@@ -18,7 +18,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;
@@ -46,7 +46,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;
@@ -72,7 +72,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;
@@ -95,7 +95,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;
@@ -118,7 +118,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;
@@ -141,7 +141,7 @@ impl Service {
             &self.meilisearch,
             &self.auth,
             true,
-            &self.options,
+            self.options,
             analytics::MockAnalytics::new(&self.options).0
         ))
         .await;

meilisearch-http/tests/content_type.rs

@@ -63,7 +63,7 @@ async fn error_json_bad_content_type() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -146,7 +146,7 @@ async fn extract_actual_content_type() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;

meilisearch-http/tests/documents/add_documents.rs

@@ -21,7 +21,7 @@ async fn add_documents_test_json_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -66,7 +66,7 @@ async fn add_single_document_test_json_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -112,7 +112,7 @@ async fn error_add_documents_test_bad_content_types() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -180,7 +180,7 @@ async fn error_add_documents_test_no_content_type() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -240,7 +240,7 @@ async fn error_add_malformed_csv_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -302,7 +302,7 @@ async fn error_add_malformed_json_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -414,7 +414,7 @@ async fn error_add_malformed_ndjson_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -474,7 +474,7 @@ async fn error_add_missing_payload_csv_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -526,7 +526,7 @@ async fn error_add_missing_payload_json_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;
@@ -578,7 +578,7 @@ async fn error_add_missing_payload_ndjson_documents() {
         &server.service.meilisearch,
         &server.service.auth,
         true,
-        &server.service.options,
+        server.service.options,
         analytics::MockAnalytics::new(&server.service.options).0
     ))
     .await;