Merge #3524

3524: Update the metrics route r=irevoire a=irevoire

Fixes #3523

Make the metrics available by default without a feature flag.
+ Rename the cli-flag to `experimental-enable-metrics`.

Co-authored-by: Tamo <tamo@meilisearch.com>

config.toml

@@ -118,3 +118,13 @@ ssl_resumption = false
 ssl_tickets = false
 # Activates SSL tickets.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-tickets
+
+#############################
+### Experimental features ###
+#############################
+
+experimental-enable-metrics = false
+# Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
+# Enables the Prometheus metrics on the `GET /metrics` endpoint.
+
+

meilisearch/Cargo.toml

@@ -52,7 +52,7 @@ parking_lot = "0.12.1"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
 pin-project-lite = "0.2.9"
 platform-dirs = "0.3.0"
-prometheus = { version = "0.13.2", features = ["process"], optional = true }
+prometheus = { version = "0.13.2", features = ["process"] }
 rand = "0.8.5"
 rayon = "1.5.3"
 regex = "1.6.0"
@@ -107,7 +107,6 @@ zip = { version = "0.6.2", optional = true }
 
 [features]
 default = ["analytics", "meilisearch-types/default", "mini-dashboard"]
-metrics = ["prometheus"]
 analytics = ["segment"]
 mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"]
 chinese = ["meilisearch-types/chinese"]

meilisearch/src/analytics/segment_analytics.rs

@@ -224,6 +224,7 @@ impl super::Analytics for SegmentAnalytics {
 #[derive(Debug, Clone, Serialize)]
 struct Infos {
     env: String,
+    experimental_enable_metrics: bool,
     db_path: bool,
     import_dump: bool,
     dump_dir: bool,
@@ -255,9 +256,8 @@ impl From<Opt> for Infos {
         // to add analytics when we add a field in the Opt.
         // Thus we must not insert `..` at the end.
         let Opt {
-            #[cfg(features = "metrics")]
-                enable_metrics_route: _,
             db_path,
+            experimental_enable_metrics,
             http_addr,
             master_key: _,
             env,
@@ -299,6 +299,7 @@ impl From<Opt> for Infos {
         // We consider information sensible if it contains a path, an address, or a key.
         Self {
             env,
+            experimental_enable_metrics,
             db_path: db_path != PathBuf::from("./data.ms"),
             import_dump: import_dump.is_some(),
             dump_dir: dump_dir != PathBuf::from("dumps/"),

meilisearch/src/lib.rs

@@ -4,15 +4,12 @@ pub mod error;
 pub mod analytics;
 #[macro_use]
 pub mod extractors;
+pub mod metrics;
+pub mod middleware;
 pub mod option;
 pub mod routes;
 pub mod search;
 
-#[cfg(feature = "metrics")]
-pub mod metrics;
-#[cfg(feature = "metrics")]
-pub mod route_metrics;
-
 use std::fs::File;
 use std::io::{BufReader, BufWriter};
 use std::path::Path;
@@ -25,7 +22,7 @@ use actix_http::body::MessageBody;
 use actix_web::dev::{ServiceFactory, ServiceResponse};
 use actix_web::error::JsonPayloadError;
 use actix_web::web::Data;
-use actix_web::{middleware, web, HttpRequest};
+use actix_web::{web, HttpRequest};
 use analytics::Analytics;
 use anyhow::bail;
 use error::PayloadError;
@@ -114,14 +111,13 @@ pub fn create_app(
                 analytics.clone(),
             )
         })
-        .configure(routes::configure)
+        .configure(|cfg| routes::configure(cfg, opt.experimental_enable_metrics))
         .configure(|s| dashboard(s, enable_dashboard));
-    #[cfg(feature = "metrics")]
-    let app = app.configure(|s| configure_metrics_route(s, opt.enable_metrics_route));
 
-    #[cfg(feature = "metrics")]
-    let app =
-        app.wrap(middleware::Condition::new(opt.enable_metrics_route, route_metrics::RouteMetrics));
+    let app = app.wrap(actix_web::middleware::Condition::new(
+        opt.experimental_enable_metrics,
+        middleware::RouteMetrics,
+    ));
     app.wrap(
         Cors::default()
             .send_wildcard()
@@ -130,9 +126,9 @@ pub fn create_app(
             .allow_any_method()
             .max_age(86_400), // 24h
     )
-    .wrap(middleware::Logger::default())
-    .wrap(middleware::Compress::default())
-    .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim))
+    .wrap(actix_web::middleware::Logger::default())
+    .wrap(actix_web::middleware::Compress::default())
+    .wrap(actix_web::middleware::NormalizePath::new(actix_web::middleware::TrailingSlash::Trim))
 }
 
 enum OnFailure {
@@ -450,15 +446,6 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
     config.service(web::resource("/").route(web::get().to(routes::running)));
 }
 
-#[cfg(feature = "metrics")]
-pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) {
-    if enable_metrics_route {
-        config.service(
-            web::resource("/metrics").route(web::get().to(crate::route_metrics::get_metrics)),
-        );
-    }
-}
-
 /// Parses the output of
 /// [`VERGEN_GIT_SEMVER_LIGHTWEIGHT`](https://docs.rs/vergen/latest/vergen/struct.Git.html#instructions)
 ///  as a prototype name.

meilisearch/src/middleware.rs

@@ -1,45 +1,11 @@
+//! Contains all the custom middleware used in meilisearch
+
 use std::future::{ready, Ready};
 
 use actix_web::dev::{self, Service, ServiceRequest, ServiceResponse, Transform};
-use actix_web::http::header;
-use actix_web::web::Data;
-use actix_web::{Error, HttpResponse};
+use actix_web::Error;
 use futures_util::future::LocalBoxFuture;
-use index_scheduler::IndexScheduler;
-use meilisearch_auth::AuthController;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::keys::actions;
-use prometheus::{Encoder, HistogramTimer, TextEncoder};
-
-use crate::extractors::authentication::policies::ActionPolicy;
-use crate::extractors::authentication::GuardedData;
-use crate::routes::create_all_stats;
-
-pub async fn get_metrics(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
-    auth_controller: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, AuthController>,
-) -> Result<HttpResponse, ResponseError> {
-    let search_rules = &index_scheduler.filters().search_rules;
-    let response =
-        create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), search_rules)?;
-
-    crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64);
-    crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64);
-
-    for (index, value) in response.indexes.iter() {
-        crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT
-            .with_label_values(&[index])
-            .set(value.number_of_documents as i64);
-    }
-
-    let encoder = TextEncoder::new();
-    let mut buffer = vec![];
-    encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics");
-
-    let response = String::from_utf8(buffer).expect("Failed to convert bytes to string");
-
-    Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response))
-}
+use prometheus::HistogramTimer;
 
 pub struct RouteMetrics;
 

meilisearch/src/option.rs

@@ -47,8 +47,7 @@ const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
 const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
 const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR";
 const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
-#[cfg(feature = "metrics")]
-const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE";
+const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
 
 const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
 const DEFAULT_DB_PATH: &str = "./data.ms";
@@ -287,11 +286,12 @@ pub struct Opt {
     #[serde(default)]
     pub log_level: LogLevel,
 
-    /// Enables Prometheus metrics and /metrics route.
-    #[cfg(feature = "metrics")]
-    #[clap(long, env = MEILI_ENABLE_METRICS_ROUTE)]
+    /// Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
+    ///
+    /// Enables the Prometheus metrics on the `GET /metrics` endpoint.
+    #[clap(long, env = MEILI_EXPERIMENTAL_ENABLE_METRICS)]
     #[serde(default)]
-    pub enable_metrics_route: bool,
+    pub experimental_enable_metrics: bool,
 
     #[serde(flatten)]
     #[clap(flatten)]
@@ -384,8 +384,7 @@ impl Opt {
             config_file_path: _,
             #[cfg(all(not(debug_assertions), feature = "analytics"))]
             no_analytics,
-            #[cfg(feature = "metrics")]
-            enable_metrics_route,
+            experimental_enable_metrics: enable_metrics_route,
         } = self;
         export_to_env_if_not_present(MEILI_DB_PATH, db_path);
         export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr);
@@ -423,13 +422,10 @@ impl Opt {
 
         export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
         export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string());
-        #[cfg(feature = "metrics")]
-        {
-            export_to_env_if_not_present(
-                MEILI_ENABLE_METRICS_ROUTE,
-                enable_metrics_route.to_string(),
-            );
-        }
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_ENABLE_METRICS,
+            enable_metrics_route.to_string(),
+        );
         indexer_options.export_to_env();
     }
 

meilisearch/src/routes/metrics.rs

@@ -0,0 +1,45 @@
+use actix_web::http::header;
+use actix_web::web::{self, Data};
+use actix_web::HttpResponse;
+use index_scheduler::IndexScheduler;
+use meilisearch_auth::{AuthController, AuthFilter};
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::keys::actions;
+use prometheus::{Encoder, TextEncoder};
+
+use crate::extractors::authentication::policies::ActionPolicy;
+use crate::extractors::authentication::GuardedData;
+use crate::routes::create_all_stats;
+
+pub fn configure(config: &mut web::ServiceConfig) {
+    config.service(web::resource("").route(web::get().to(get_metrics)));
+}
+
+pub async fn get_metrics(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
+    auth_controller: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, AuthController>,
+) -> Result<HttpResponse, ResponseError> {
+    let response = create_all_stats(
+        (*index_scheduler).clone(),
+        (*auth_controller).clone(),
+        // we don't use the filters contained in the `ActionPolicy` because the metrics must have the right to access all the indexes.
+        &AuthFilter::default(),
+    )?;
+
+    crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64);
+    crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64);
+
+    for (index, value) in response.indexes.iter() {
+        crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT
+            .with_label_values(&[index])
+            .set(value.number_of_documents as i64);
+    }
+
+    let encoder = TextEncoder::new();
+    let mut buffer = vec![];
+    encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics");
+
+    let response = String::from_utf8(buffer).expect("Failed to convert bytes to string");
+
+    Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response))
+}

meilisearch/src/routes/mod.rs

@@ -22,11 +22,12 @@ const PAGINATION_DEFAULT_LIMIT: usize = 20;
 mod api_key;
 mod dump;
 pub mod indexes;
+mod metrics;
 mod multi_search;
 mod swap_indexes;
 pub mod tasks;
 
-pub fn configure(cfg: &mut web::ServiceConfig) {
+pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) {
     cfg.service(web::scope("/tasks").configure(tasks::configure))
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
@@ -36,6 +37,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::scope("/indexes").configure(indexes::configure))
         .service(web::scope("/multi-search").configure(multi_search::configure))
         .service(web::scope("/swap-indexes").configure(swap_indexes::configure));
+
+    if enable_metrics {
+        cfg.service(web::scope("/metrics").configure(metrics::configure));
+    }
 }
 
 #[derive(Debug, Serialize)]

meilisearch/tests/auth/authorization.rs

@@ -10,7 +10,7 @@ use crate::common::Server;
 
 pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
     Lazy::new(|| {
-        let mut authorizations = hashmap! {
+        let authorizations = hashmap! {
             ("POST",    "/multi-search") =>                                    hashset!{"search", "*"},
             ("POST",    "/indexes/products/search") =>                         hashset!{"search", "*"},
             ("GET",     "/indexes/products/search") =>                         hashset!{"search", "*"},
@@ -52,6 +52,7 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/stats") =>                                           hashset!{"stats.get", "stats.*", "*"},
             ("POST",    "/dumps") =>                                           hashset!{"dumps.create", "dumps.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
+            ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},
             ("GET",     "/keys/mykey/") =>                                     hashset!{"keys.get", "*"},
             ("DELETE",  "/keys/mykey/") =>                                     hashset!{"keys.delete", "*"},
@@ -59,10 +60,6 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/keys") =>                                            hashset!{"keys.get", "*"},
         };
 
-        if cfg!(feature = "metrics") {
-            authorizations.insert(("GET", "/metrics"), hashset! {"metrics.get", "metrics.*", "*"});
-        }
-
         authorizations
     });
 

meilisearch/tests/common/server.rs

@@ -208,8 +208,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
             skip_index_budget: true,
             ..Parser::parse_from(None as Option<&str>)
         },
-        #[cfg(feature = "metrics")]
-        enable_metrics_route: true,
+        experimental_enable_metrics: true,
         ..Parser::parse_from(None as Option<&str>)
     }
 }