mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-05 12:38:55 +01:00
prometheus and grafana dashboards implemented
This commit is contained in:
parent
22874ce300
commit
de58ccd4ba
38
Cargo.lock
generated
38
Cargo.lock
generated
@ -2045,6 +2045,7 @@ dependencies = [
|
||||
"indexmap",
|
||||
"itertools",
|
||||
"jsonwebtoken",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"manifest-dir-macros",
|
||||
"maplit",
|
||||
@ -2058,6 +2059,7 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"platform-dirs",
|
||||
"prometheus",
|
||||
"rand",
|
||||
"rayon",
|
||||
"regex",
|
||||
@ -2664,6 +2666,36 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "procfs"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"hex",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"fnv",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"memchr",
|
||||
"parking_lot",
|
||||
"procfs",
|
||||
"protobuf",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proptest"
|
||||
version = "1.0.0"
|
||||
@ -2695,6 +2727,12 @@ dependencies = [
|
||||
"syn 0.15.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "2.27.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
|
1007
grafana-dashboards/dashboard.json
Normal file
1007
grafana-dashboards/dashboard.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -77,6 +77,8 @@ tokio = { version = "1.17.0", features = ["full"] }
|
||||
tokio-stream = "0.1.8"
|
||||
uuid = { version = "1.1.2", features = ["serde", "v4"] }
|
||||
walkdir = "2.3.2"
|
||||
prometheus = { version = "0.13.0", features = ["process"] }
|
||||
lazy_static = "1.4.0"
|
||||
|
||||
[dev-dependencies]
|
||||
actix-rt = "2.7.0"
|
||||
|
@ -7,6 +7,7 @@ pub mod task;
|
||||
pub mod extractors;
|
||||
pub mod option;
|
||||
pub mod routes;
|
||||
pub mod metrics;
|
||||
|
||||
use std::sync::{atomic::AtomicBool, Arc};
|
||||
use std::time::Duration;
|
||||
@ -146,16 +147,48 @@ macro_rules! create_app {
|
||||
use actix_cors::Cors;
|
||||
use actix_web::middleware::TrailingSlash;
|
||||
use actix_web::App;
|
||||
use actix_web::dev::Service;
|
||||
use actix_web::{middleware, web};
|
||||
use meilisearch_http::error::MeilisearchHttpError;
|
||||
use meilisearch_http::routes;
|
||||
use meilisearch_http::{configure_data, dashboard};
|
||||
use meilisearch_types::error::ResponseError;
|
||||
use meilisearch_http::metrics;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge};
|
||||
use prometheus::{HistogramVec, IntCounterVec, IntGauge, HistogramTimer};
|
||||
|
||||
App::new()
|
||||
.configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics))
|
||||
.configure(routes::configure)
|
||||
.configure(|s| dashboard(s, $enable_frontend))
|
||||
.wrap_fn(|req, srv| {
|
||||
let mut histogram_timer: Option<HistogramTimer> = None;
|
||||
let request_path = req.path();
|
||||
let is_registered_resource = req.resource_map().has_resource(request_path);
|
||||
if is_registered_resource {
|
||||
let request_method = req.method().to_string();
|
||||
histogram_timer = Some(
|
||||
metrics::HTTP_RESPONSE_TIME_SECONDS
|
||||
.with_label_values(&[&request_method, request_path])
|
||||
.start_timer(),
|
||||
);
|
||||
metrics::HTTP_REQUESTS_TOTAL
|
||||
.with_label_values(&[&request_method, request_path])
|
||||
.inc();
|
||||
}
|
||||
|
||||
let fut = srv.call(req);
|
||||
|
||||
async {
|
||||
let res = fut.await?;
|
||||
if let Some(histogram_timer) = histogram_timer {
|
||||
histogram_timer.observe_duration();
|
||||
};
|
||||
Ok(res)
|
||||
}
|
||||
})
|
||||
.wrap(
|
||||
Cors::default()
|
||||
.send_wildcard()
|
||||
|
40
meilisearch-http/src/metrics.rs
Normal file
40
meilisearch-http/src/metrics.rs
Normal file
@ -0,0 +1,40 @@
|
||||
use lazy_static::lazy_static;
|
||||
use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec};
|
||||
use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec};
|
||||
|
||||
const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[
|
||||
0.0005, 0.0008, 0.00085, 0.0009, 0.00095, 0.001, 0.00105, 0.0011, 0.00115, 0.0012, 0.0015,
|
||||
0.002, 0.003, 1.0,
|
||||
];
|
||||
|
||||
lazy_static! {
|
||||
pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
|
||||
opts!("http_requests_total", "HTTP requests total"),
|
||||
&["method", "path"]
|
||||
)
|
||||
.expect("Can't create a metric");
|
||||
|
||||
pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!(
|
||||
opts!("meilisearch_database_size", "MeiliSearch Stats DbSize")
|
||||
)
|
||||
.expect("Can't create a metric");
|
||||
|
||||
pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!(
|
||||
opts!("meilisearch_total_index", "MeiliSearch Stats Index Count")
|
||||
)
|
||||
.expect("Can't create a metric");
|
||||
|
||||
pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!(
|
||||
opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"),
|
||||
&["index"]
|
||||
)
|
||||
.expect("Can't create a metric");
|
||||
|
||||
pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!(
|
||||
"http_response_time_seconds",
|
||||
"HTTP response times",
|
||||
&["method", "path"],
|
||||
HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec()
|
||||
)
|
||||
.expect("Can't create a metric");
|
||||
}
|
@ -1,4 +1,5 @@
|
||||
use actix_web::{web, HttpResponse};
|
||||
use actix_web::http::header::{self};
|
||||
use log::debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@ -10,6 +11,7 @@ use meilisearch_types::error::ResponseError;
|
||||
use meilisearch_types::star_or::StarOr;
|
||||
|
||||
use crate::extractors::authentication::{policies::*, GuardedData};
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
|
||||
mod api_key;
|
||||
mod dump;
|
||||
@ -19,6 +21,7 @@ mod tasks;
|
||||
pub fn configure(cfg: &mut web::ServiceConfig) {
|
||||
cfg.service(web::scope("/tasks").configure(tasks::configure))
|
||||
.service(web::resource("/health").route(web::get().to(get_health)))
|
||||
.service(web::resource("/metrics").route(web::get().to(get_metrics)))
|
||||
.service(web::scope("/keys").configure(api_key::configure))
|
||||
.service(web::scope("/dumps").configure(dump::configure))
|
||||
.service(web::resource("/stats").route(web::get().to(get_stats)))
|
||||
@ -269,3 +272,36 @@ struct KeysResponse {
|
||||
pub async fn get_health() -> Result<HttpResponse, ResponseError> {
|
||||
Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
|
||||
}
|
||||
|
||||
pub async fn get_metrics(
|
||||
meilisearch: GuardedData<ActionPolicy<{ actions::STATS_GET }>, MeiliSearch>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
|
||||
let search_rules = &meilisearch.filters().search_rules;
|
||||
let response = meilisearch.get_all_stats(search_rules).await?;
|
||||
|
||||
crate::metrics::MEILISEARCH_DB_SIZE
|
||||
.set(response.database_size as i64);
|
||||
|
||||
crate::metrics::MEILISEARCH_INDEX_COUNT
|
||||
.set(response.indexes.len() as i64);
|
||||
|
||||
for (index, value) in response.indexes.iter() {
|
||||
crate::metrics::MEILISEARCH_DOCS_COUNT
|
||||
.with_label_values(&[&index])
|
||||
.set(value.number_of_documents as i64);
|
||||
}
|
||||
|
||||
let encoder = TextEncoder::new();
|
||||
let mut buffer = vec![];
|
||||
encoder
|
||||
.encode(&prometheus::gather(), &mut buffer)
|
||||
.expect("Failed to encode metrics");
|
||||
|
||||
let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string");
|
||||
buffer.clear();
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.insert_header(header::ContentType(mime::TEXT_PLAIN))
|
||||
.body(response))
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user