diff --git a/config.toml b/config.toml index 903ce1446..1da619688 100644 --- a/config.toml +++ b/config.toml @@ -118,3 +118,13 @@ ssl_resumption = false ssl_tickets = false # Activates SSL tickets. # https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-tickets + +############################# +### Experimental features ### +############################# + +experimental-enable-metrics = false +# Experimental metrics feature. For more information, see: +# Enables the Prometheus metrics on the `GET /metrics` endpoint. + + diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 324081880..809975ec7 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -52,7 +52,7 @@ parking_lot = "0.12.1" permissive-json-pointer = { path = "../permissive-json-pointer" } pin-project-lite = "0.2.9" platform-dirs = "0.3.0" -prometheus = { version = "0.13.2", features = ["process"], optional = true } +prometheus = { version = "0.13.2", features = ["process"] } rand = "0.8.5" rayon = "1.5.3" regex = "1.6.0" @@ -107,7 +107,6 @@ zip = { version = "0.6.2", optional = true } [features] default = ["analytics", "meilisearch-types/default", "mini-dashboard"] -metrics = ["prometheus"] analytics = ["segment"] mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"] chinese = ["meilisearch-types/chinese"] diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 472001004..57afb9c87 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -224,6 +224,7 @@ impl super::Analytics for SegmentAnalytics { #[derive(Debug, Clone, Serialize)] struct Infos { env: String, + experimental_enable_metrics: bool, db_path: bool, import_dump: bool, dump_dir: bool, @@ -255,9 +256,8 @@ impl From for Infos { // to add analytics when we add a field in the Opt. // Thus we must not insert `..` at the end. let Opt { - #[cfg(features = "metrics")] - enable_metrics_route: _, db_path, + experimental_enable_metrics, http_addr, master_key: _, env, @@ -299,6 +299,7 @@ impl From for Infos { // We consider information sensible if it contains a path, an address, or a key. Self { env, + experimental_enable_metrics, db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), dump_dir: dump_dir != PathBuf::from("dumps/"), diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 3352aa4f5..13c236983 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -4,15 +4,12 @@ pub mod error; pub mod analytics; #[macro_use] pub mod extractors; +pub mod metrics; +pub mod middleware; pub mod option; pub mod routes; pub mod search; -#[cfg(feature = "metrics")] -pub mod metrics; -#[cfg(feature = "metrics")] -pub mod route_metrics; - use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -25,7 +22,7 @@ use actix_http::body::MessageBody; use actix_web::dev::{ServiceFactory, ServiceResponse}; use actix_web::error::JsonPayloadError; use actix_web::web::Data; -use actix_web::{middleware, web, HttpRequest}; +use actix_web::{web, HttpRequest}; use analytics::Analytics; use anyhow::bail; use error::PayloadError; @@ -114,14 +111,13 @@ pub fn create_app( analytics.clone(), ) }) - .configure(routes::configure) + .configure(|cfg| routes::configure(cfg, opt.experimental_enable_metrics)) .configure(|s| dashboard(s, enable_dashboard)); - #[cfg(feature = "metrics")] - let app = app.configure(|s| configure_metrics_route(s, opt.enable_metrics_route)); - #[cfg(feature = "metrics")] - let app = - app.wrap(middleware::Condition::new(opt.enable_metrics_route, route_metrics::RouteMetrics)); + let app = app.wrap(actix_web::middleware::Condition::new( + opt.experimental_enable_metrics, + middleware::RouteMetrics, + )); app.wrap( Cors::default() .send_wildcard() @@ -130,9 +126,9 @@ pub fn create_app( .allow_any_method() .max_age(86_400), // 24h ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim)) + .wrap(actix_web::middleware::Logger::default()) + .wrap(actix_web::middleware::Compress::default()) + .wrap(actix_web::middleware::NormalizePath::new(actix_web::middleware::TrailingSlash::Trim)) } enum OnFailure { @@ -450,15 +446,6 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { config.service(web::resource("/").route(web::get().to(routes::running))); } -#[cfg(feature = "metrics")] -pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) { - if enable_metrics_route { - config.service( - web::resource("/metrics").route(web::get().to(crate::route_metrics::get_metrics)), - ); - } -} - /// Parses the output of /// [`VERGEN_GIT_SEMVER_LIGHTWEIGHT`](https://docs.rs/vergen/latest/vergen/struct.Git.html#instructions) /// as a prototype name. diff --git a/meilisearch/src/route_metrics.rs b/meilisearch/src/middleware.rs similarity index 58% rename from meilisearch/src/route_metrics.rs rename to meilisearch/src/middleware.rs index a23c2b93d..080a52634 100644 --- a/meilisearch/src/route_metrics.rs +++ b/meilisearch/src/middleware.rs @@ -1,45 +1,11 @@ +//! Contains all the custom middleware used in meilisearch + use std::future::{ready, Ready}; use actix_web::dev::{self, Service, ServiceRequest, ServiceResponse, Transform}; -use actix_web::http::header; -use actix_web::web::Data; -use actix_web::{Error, HttpResponse}; +use actix_web::Error; use futures_util::future::LocalBoxFuture; -use index_scheduler::IndexScheduler; -use meilisearch_auth::AuthController; -use meilisearch_types::error::ResponseError; -use meilisearch_types::keys::actions; -use prometheus::{Encoder, HistogramTimer, TextEncoder}; - -use crate::extractors::authentication::policies::ActionPolicy; -use crate::extractors::authentication::GuardedData; -use crate::routes::create_all_stats; - -pub async fn get_metrics( - index_scheduler: GuardedData, Data>, - auth_controller: GuardedData, AuthController>, -) -> Result { - let search_rules = &index_scheduler.filters().search_rules; - let response = - create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), search_rules)?; - - crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); - - for (index, value) in response.indexes.iter() { - crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT - .with_label_values(&[index]) - .set(value.number_of_documents as i64); - } - - let encoder = TextEncoder::new(); - let mut buffer = vec![]; - encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); - - let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); - - Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response)) -} +use prometheus::HistogramTimer; pub struct RouteMetrics; diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 544827387..0c6457e7a 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -47,8 +47,7 @@ const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; -#[cfg(feature = "metrics")] -const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; +const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_DB_PATH: &str = "./data.ms"; @@ -287,11 +286,12 @@ pub struct Opt { #[serde(default)] pub log_level: LogLevel, - /// Enables Prometheus metrics and /metrics route. - #[cfg(feature = "metrics")] - #[clap(long, env = MEILI_ENABLE_METRICS_ROUTE)] + /// Experimental metrics feature. For more information, see: + /// + /// Enables the Prometheus metrics on the `GET /metrics` endpoint. + #[clap(long, env = MEILI_EXPERIMENTAL_ENABLE_METRICS)] #[serde(default)] - pub enable_metrics_route: bool, + pub experimental_enable_metrics: bool, #[serde(flatten)] #[clap(flatten)] @@ -384,8 +384,7 @@ impl Opt { config_file_path: _, #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics, - #[cfg(feature = "metrics")] - enable_metrics_route, + experimental_enable_metrics: enable_metrics_route, } = self; export_to_env_if_not_present(MEILI_DB_PATH, db_path); export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr); @@ -423,13 +422,10 @@ impl Opt { export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string()); - #[cfg(feature = "metrics")] - { - export_to_env_if_not_present( - MEILI_ENABLE_METRICS_ROUTE, - enable_metrics_route.to_string(), - ); - } + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_ENABLE_METRICS, + enable_metrics_route.to_string(), + ); indexer_options.export_to_env(); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs new file mode 100644 index 000000000..1d3fb834e --- /dev/null +++ b/meilisearch/src/routes/metrics.rs @@ -0,0 +1,45 @@ +use actix_web::http::header; +use actix_web::web::{self, Data}; +use actix_web::HttpResponse; +use index_scheduler::IndexScheduler; +use meilisearch_auth::{AuthController, AuthFilter}; +use meilisearch_types::error::ResponseError; +use meilisearch_types::keys::actions; +use prometheus::{Encoder, TextEncoder}; + +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::GuardedData; +use crate::routes::create_all_stats; + +pub fn configure(config: &mut web::ServiceConfig) { + config.service(web::resource("").route(web::get().to(get_metrics))); +} + +pub async fn get_metrics( + index_scheduler: GuardedData, Data>, + auth_controller: GuardedData, AuthController>, +) -> Result { + let response = create_all_stats( + (*index_scheduler).clone(), + (*auth_controller).clone(), + // we don't use the filters contained in the `ActionPolicy` because the metrics must have the right to access all the indexes. + &AuthFilter::default(), + )?; + + crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64); + crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); + + for (index, value) in response.indexes.iter() { + crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT + .with_label_values(&[index]) + .set(value.number_of_documents as i64); + } + + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); + + let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); + + Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response)) +} diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index bd3dd0649..a4523e53f 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -22,11 +22,12 @@ const PAGINATION_DEFAULT_LIMIT: usize = 20; mod api_key; mod dump; pub mod indexes; +mod metrics; mod multi_search; mod swap_indexes; pub mod tasks; -pub fn configure(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) .service(web::scope("/keys").configure(api_key::configure)) @@ -36,6 +37,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/indexes").configure(indexes::configure)) .service(web::scope("/multi-search").configure(multi_search::configure)) .service(web::scope("/swap-indexes").configure(swap_indexes::configure)); + + if enable_metrics { + cfg.service(web::scope("/metrics").configure(metrics::configure)); + } } #[derive(Debug, Serialize)] diff --git a/meilisearch/tests/auth/authorization.rs b/meilisearch/tests/auth/authorization.rs index b0bc7ab03..e92f41bcf 100644 --- a/meilisearch/tests/auth/authorization.rs +++ b/meilisearch/tests/auth/authorization.rs @@ -10,7 +10,7 @@ use crate::common::Server; pub static AUTHORIZATIONS: Lazy>> = Lazy::new(|| { - let mut authorizations = hashmap! { + let authorizations = hashmap! { ("POST", "/multi-search") => hashset!{"search", "*"}, ("POST", "/indexes/products/search") => hashset!{"search", "*"}, ("GET", "/indexes/products/search") => hashset!{"search", "*"}, @@ -52,6 +52,7 @@ pub static AUTHORIZATIONS: Lazy hashset!{"stats.get", "stats.*", "*"}, ("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"}, ("GET", "/version") => hashset!{"version", "*"}, + ("GET", "/metrics") => hashset!{"metrics.get", "metrics.*", "*"}, ("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"}, ("GET", "/keys/mykey/") => hashset!{"keys.get", "*"}, ("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"}, @@ -59,10 +60,6 @@ pub static AUTHORIZATIONS: Lazy hashset!{"keys.get", "*"}, }; - if cfg!(feature = "metrics") { - authorizations.insert(("GET", "/metrics"), hashset! {"metrics.get", "metrics.*", "*"}); - } - authorizations }); diff --git a/meilisearch/tests/common/server.rs b/meilisearch/tests/common/server.rs index 268b5c4b7..d4cf6184d 100644 --- a/meilisearch/tests/common/server.rs +++ b/meilisearch/tests/common/server.rs @@ -208,8 +208,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { skip_index_budget: true, ..Parser::parse_from(None as Option<&str>) }, - #[cfg(feature = "metrics")] - enable_metrics_route: true, + experimental_enable_metrics: true, ..Parser::parse_from(None as Option<&str>) } }