From c9b65677bf153571df274cf959719311821bfc8c Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 25 May 2023 18:30:30 +0200 Subject: [PATCH] return the on disk size actually used by meilisearch --- grafana-dashboards/dashboard.json | 219 ++++++++++++++---------- index-scheduler/src/index_mapper/mod.rs | 6 +- index-scheduler/src/lib.rs | 6 + meilisearch-auth/src/lib.rs | 5 + meilisearch-auth/src/store.rs | 5 + meilisearch/src/metrics.rs | 7 +- meilisearch/src/routes/metrics.rs | 1 + meilisearch/src/routes/mod.rs | 12 +- 8 files changed, 169 insertions(+), 92 deletions(-) diff --git a/grafana-dashboards/dashboard.json b/grafana-dashboards/dashboard.json index ec5d8530b..b546b4969 100644 --- a/grafana-dashboards/dashboard.json +++ b/grafana-dashboards/dashboard.json @@ -52,71 +52,6 @@ "title": "Web application metrics", "type": "row" }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "decimals": 0, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "interval": "5s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.2", - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "builder", - "exemplar": true, - "expr": "meilisearch_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Database Size", - "type": "stat" - }, { "datasource": { "type": "prometheus" @@ -145,7 +80,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 4, + "x": 0, "y": 1 }, "id": 22, @@ -206,7 +141,7 @@ "gridPos": { "h": 6, "w": 4, - "x": 8, + "x": 4, "y": 1 }, "id": 18, @@ -431,10 +366,11 @@ "type": "prometheus", "uid": "c4085c47-f6d3-45dd-b761-6809055bb749" }, + "description": "", "fieldConfig": { "defaults": { "color": { - "mode": "continuous-YlBl" + "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, @@ -443,7 +379,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -456,18 +392,18 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -480,7 +416,8 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, @@ -490,13 +427,13 @@ "x": 0, "y": 7 }, - "id": 1, + "id": 2, "interval": "5s", "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -504,6 +441,7 @@ "sort": "none" } }, + "pluginVersion": "9.5.2", "targets": [ { "datasource": { @@ -511,14 +449,26 @@ }, "editorMode": "builder", "exemplar": true, - "expr": "rate(meilisearch_http_requests_total{instance=\"$instance\", job=\"meilisearch\"}[5m])", + "expr": "meilisearch_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}", "interval": "", - "legendFormat": "{{method}} {{path}}", + "legendFormat": "Database size on disk", "range": true, - "refId": "A" + "refId": "DB Size on disk" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c4085c47-f6d3-45dd-b761-6809055bb749" + }, + "editorMode": "builder", + "expr": "meilisearch_used_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}", + "hide": false, + "legendFormat": "Used bytes", + "range": true, + "refId": "Actual used bytes" } ], - "title": "HTTP requests per second (All Indexes)", + "title": "Database Size in bytes", "type": "timeseries" }, { @@ -616,6 +566,101 @@ "title": "Mean response time (All Indexes)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "c4085c47-f6d3-45dd-b761-6809055bb749" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 1, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(meilisearch_http_requests_total{instance=\"$instance\", job=\"meilisearch\"}[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP requests per second (All Indexes)", + "type": "timeseries" + }, { "cards": {}, "color": { @@ -648,7 +693,7 @@ "h": 11, "w": 24, "x": 0, - "y": 18 + "y": 29 }, "heatmap": {}, "hideZeroBuckets": false, @@ -790,7 +835,7 @@ "h": 11, "w": 12, "x": 0, - "y": 29 + "y": 40 }, "id": 23, "interval": "5s", @@ -883,7 +928,7 @@ "h": 11, "w": 12, "x": 12, - "y": 29 + "y": 40 }, "id": 24, "interval": "5s", @@ -976,7 +1021,7 @@ "h": 11, "w": 12, "x": 0, - "y": 40 + "y": 51 }, "id": 25, "interval": "5s", @@ -1020,7 +1065,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 62 }, "id": 12, "panels": [], @@ -1098,7 +1143,7 @@ "h": 11, "w": 12, "x": 0, - "y": 52 + "y": 63 }, "id": 4, "interval": "5s", @@ -1201,7 +1246,7 @@ "h": 11, "w": 12, "x": 12, - "y": 52 + "y": 63 }, "id": 5, "interval": "5s", @@ -1323,6 +1368,6 @@ "timezone": "", "title": "Meilisearch", "uid": "7wcZ94dnz", - "version": 3, + "version": 4, "weekStart": "" } \ No newline at end of file diff --git a/index-scheduler/src/index_mapper/mod.rs b/index-scheduler/src/index_mapper/mod.rs index 2bf6f46ad..86bec2927 100644 --- a/index-scheduler/src/index_mapper/mod.rs +++ b/index-scheduler/src/index_mapper/mod.rs @@ -90,6 +90,8 @@ pub struct IndexStats { pub number_of_documents: u64, /// Size of the index' DB, in bytes. pub database_size: u64, + /// Size of the index' DB, in bytes. + pub used_database_size: u64, /// Association of every field name with the number of times it occurs in the documents. pub field_distribution: FieldDistribution, /// Creation date of the index. @@ -105,10 +107,10 @@ impl IndexStats { /// /// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`. pub fn new(index: &Index, rtxn: &RoTxn) -> Result { - let database_size = index.on_disk_size()?; Ok(IndexStats { number_of_documents: index.number_of_documents(rtxn)?, - database_size, + database_size: index.on_disk_size()?, + used_database_size: index.used_size()?, field_distribution: index.field_distribution(rtxn)?, created_at: index.created_at(rtxn)?, updated_at: index.updated_at(rtxn)?, diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index a05be8afb..c27546c8e 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -554,10 +554,16 @@ impl IndexScheduler { &self.index_mapper.indexer_config } + /// Return the real database size (i.e.: The size **with** the free pages) pub fn size(&self) -> Result { Ok(self.env.real_disk_size()?) } + /// Return the used database size (i.e.: The size **without** the free pages) + pub fn used_size(&self) -> Result { + Ok(self.env.non_free_pages_size()?) + } + /// Return the index corresponding to the name. /// /// * If the index wasn't opened before, the index will be opened. diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 2a02776bd..e74f1707c 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -45,6 +45,11 @@ impl AuthController { self.store.size() } + /// Return the used size of the `AuthController` database in bytes. + pub fn used_size(&self) -> Result { + self.store.used_size() + } + pub fn create_key(&self, create_key: CreateApiKey) -> Result { match self.store.get_api_key(create_key.uid)? { Some(_) => Err(AuthControllerError::ApiKeyAlreadyExists(create_key.uid.to_string())), diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 5c2776154..eb93f5a46 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -73,6 +73,11 @@ impl HeedAuthStore { Ok(self.env.real_disk_size()?) } + /// Return the number of bytes actually used in the database + pub fn used_size(&self) -> Result { + Ok(self.env.non_free_pages_size()?) + } + pub fn set_drop_on_close(&mut self, v: bool) { self.should_close_on_drop = v; } diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index 07c7706bf..79332f360 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -23,8 +23,13 @@ lazy_static! { ) .expect("Can't create a metric"); pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge = - register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch Db Size In Bytes")) + register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch DB Size In Bytes")) .expect("Can't create a metric"); + pub static ref MEILISEARCH_USED_DB_SIZE_BYTES: IntGauge = register_int_gauge!(opts!( + "meilisearch_used_db_size_bytes", + "Meilisearch Used DB Size In Bytes" + )) + .expect("Can't create a metric"); pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!(opts!("meilisearch_index_count", "Meilisearch Index Count")) .expect("Can't create a metric"); diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index 3c9a52e26..a7d41e33e 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -31,6 +31,7 @@ pub async fn get_metrics( let response = create_all_stats((*index_scheduler).clone(), auth_controller, auth_filters)?; crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64); + crate::metrics::MEILISEARCH_USED_DB_SIZE_BYTES.set(response.used_database_size as i64); crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); for (index, value) in response.indexes.iter() { diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 51340ac1b..57d670b5f 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -231,6 +231,8 @@ pub async fn running() -> HttpResponse { #[serde(rename_all = "camelCase")] pub struct Stats { pub database_size: u64, + #[serde(skip)] + pub used_database_size: u64, #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] pub last_update: Option, pub indexes: BTreeMap, @@ -259,6 +261,7 @@ pub fn create_all_stats( let mut last_task: Option = None; let mut indexes = BTreeMap::new(); let mut database_size = 0; + let mut used_database_size = 0; for index_uid in index_scheduler.index_names()? { // Accumulate the size of all indexes, even unauthorized ones, so @@ -266,6 +269,7 @@ pub fn create_all_stats( // See for context. let stats = index_scheduler.index_stats(&index_uid)?; database_size += stats.inner_stats.database_size; + used_database_size += stats.inner_stats.used_database_size; if !filters.is_index_authorized(&index_uid) { continue; @@ -278,10 +282,14 @@ pub fn create_all_stats( } database_size += index_scheduler.size()?; + used_database_size += index_scheduler.used_size()?; database_size += auth_controller.size()?; - database_size += index_scheduler.compute_update_file_size()?; + used_database_size += auth_controller.used_size()?; + let update_file_size = index_scheduler.compute_update_file_size()?; + database_size += update_file_size; + used_database_size += update_file_size; - let stats = Stats { database_size, last_update: last_task, indexes }; + let stats = Stats { database_size, used_database_size, last_update: last_task, indexes }; Ok(stats) }