diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index 379a25030..a01cfabe0 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -8,6 +8,7 @@ use serde_qs as qs; use siphasher::sip::SipHasher; use walkdir::WalkDir; +use crate::helpers::EnvSizer; use crate::Data; use crate::Opt; @@ -33,12 +34,7 @@ impl EventProperties { } } - let database_size = WalkDir::new(&data.db_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()); + let database_size = data.env.size(); let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); @@ -116,7 +112,7 @@ pub fn analytics_sender(data: Data, opt: Opt) { time, app_version, user_properties, - event_properties + event_properties, }; let event = serde_json::to_string(&event).unwrap(); diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index 9227454a5..3aee18217 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -1,14 +1,14 @@ +use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; +use chrono::{DateTime, Utc}; use sha2::Digest; use crate::index::Settings; use crate::index_controller::{IndexController, IndexStats}; use crate::index_controller::{IndexMetadata, IndexSettings}; use crate::option::Opt; -use std::collections::HashMap; -use chrono::{DateTime, Utc}; pub mod search; mod updates; @@ -119,15 +119,22 @@ impl Data { pub async fn get_stats(&self) -> anyhow::Result { let mut stats = Stats::default(); + stats.database_size += self.index_controller.get_uuids_size().await?; for index in self.index_controller.list_indexes().await? { let index_stats = self.index_controller.get_stats(index.uid.clone()).await?; stats.database_size += index_stats.size; + stats.database_size += self + .index_controller + .get_updates_size(index.uid.clone()) + .await?; + stats.last_update = Some(match stats.last_update { Some(last_update) => last_update.max(index.meta.updated_at), None => index.meta.updated_at, }); + stats.indexes.insert(index.uid, index_stats); } diff --git a/meilisearch-http/src/helpers/env.rs b/meilisearch-http/src/helpers/env.rs new file mode 100644 index 000000000..9bc81bc69 --- /dev/null +++ b/meilisearch-http/src/helpers/env.rs @@ -0,0 +1,16 @@ +use walkdir::WalkDir; + +pub trait EnvSizer { + fn size(&self) -> u64; +} + +impl EnvSizer for heed::Env { + fn size(&self) -> u64 { + WalkDir::new(self.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()) + } +} diff --git a/meilisearch-http/src/helpers/mod.rs b/meilisearch-http/src/helpers/mod.rs index d1204908f..a5cddf29c 100644 --- a/meilisearch-http/src/helpers/mod.rs +++ b/meilisearch-http/src/helpers/mod.rs @@ -1,4 +1,6 @@ pub mod authentication; pub mod compression; +mod env; pub use authentication::Authentication; +pub use env::EnvSizer; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 9ba03ddd2..f8835eceb 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -5,10 +5,10 @@ use std::sync::Arc; use anyhow::{bail, Context}; use milli::obkv_to_json; use serde_json::{Map, Value}; -use walkdir::WalkDir; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use updates::{Facets, Settings, UpdateResult}; +use crate::helpers::EnvSizer; mod search; mod updates; @@ -55,11 +55,7 @@ impl Index { let stop_words = self .stop_words(&txn)? .map(|stop_words| -> anyhow::Result> { - Ok(stop_words - .stream() - .into_strs()? - .into_iter() - .collect()) + Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? .unwrap_or_else(BTreeSet::new); @@ -127,13 +123,8 @@ impl Index { } } - pub fn size(&self) -> anyhow::Result { - Ok(WalkDir::new(self.env.path()) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len())) + pub fn size(&self) -> u64 { + self.env.size() } fn fields_to_display>( diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 099bb97ca..c54b2edd2 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -360,7 +360,7 @@ impl IndexActor { let rtxn = index.read_txn()?; Ok(IndexStats { - size: index.size()?, + size: index.size(), number_of_documents: index.number_of_documents(&rtxn)?, is_indexing, fields_distribution: index.fields_distribution(&rtxn)?, diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index e459af10c..8361c45cc 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -356,6 +356,16 @@ impl IndexController { Ok(self.index_handle.get_index_stats(uuid).await?) } + + pub async fn get_updates_size(&self, uid: String) -> anyhow::Result { + let uuid = self.uuid_resolver.get(uid.clone()).await?; + + Ok(self.update_handle.get_size(uuid).await?) + } + + pub async fn get_uuids_size(&self) -> anyhow::Result { + Ok(self.uuid_resolver.get_size().await?) + } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index d87b910d6..f725dda84 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -8,10 +8,11 @@ use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::mpsc; use uuid::Uuid; -use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; use crate::index_controller::index_actor::IndexActorHandle; use crate::index_controller::{get_arc_ownership_blocking, UpdateMeta, UpdateStatus}; +use super::{PayloadData, Result, UpdateError, UpdateMsg, UpdateStoreStore}; + pub struct UpdateActor { path: PathBuf, store: S, @@ -72,6 +73,9 @@ where Some(Snapshot { uuid, path, ret }) => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } + Some(GetSize { uuid, ret }) => { + let _ = ret.send(self.handle_get_size(uuid).await); + } None => break, } } @@ -223,4 +227,20 @@ where Ok(()) } + + async fn handle_get_size(&self, uuid: Uuid) -> Result { + let size = match self.store.get(uuid).await? { + Some(update_store) => tokio::task::spawn_blocking(move || -> anyhow::Result { + let txn = update_store.env.read_txn()?; + + update_store.get_size(&txn) + }) + .await + .map_err(|e| UpdateError::Error(e.into()))? + .map_err(|e| UpdateError::Error(e.into()))?, + None => 0, + }; + + Ok(size) + } } diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 8778a3674..860cc2bc8 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -79,6 +79,13 @@ where receiver.await.expect("update actor killed.") } + async fn get_size(&self, uuid: Uuid) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UpdateMsg::GetSize { uuid, ret }; + let _ = self.sender.send(msg).await; + receiver.await.expect("update actor killed.") + } + async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 8e6e3c212..f8150c00a 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -34,4 +34,8 @@ pub enum UpdateMsg { path: PathBuf, ret: oneshot::Sender>, }, + GetSize { + uuid: Uuid, + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index f3c3caf04..228b47b02 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -46,6 +46,7 @@ pub trait UpdateActorHandle { async fn delete(&self, uuid: Uuid) -> Result<()>; async fn create(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>; + async fn get_size(&self, uuid: Uuid) -> Result; async fn update( &self, meta: UpdateMeta, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index a083eb186..3d6c4e396 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,3 +1,4 @@ +use std::fs::File; use std::fs::{copy, create_dir_all, remove_file}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -6,10 +7,10 @@ use heed::types::{DecodeIgnore, OwnedType, SerdeJson}; use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use parking_lot::{Mutex, RwLock}; use serde::{Deserialize, Serialize}; -use std::fs::File; use tokio::sync::mpsc; use uuid::Uuid; +use crate::helpers::EnvSizer; use crate::index_controller::updates::*; type BEU64 = heed::zerocopy::U64; @@ -409,4 +410,18 @@ where Ok(()) } + + pub fn get_size(&self, txn: &heed::RoTxn) -> anyhow::Result { + let mut size = self.env.size(); + + for path in self.pending.iter(txn)? { + let (_, path) = path?; + + if let Ok(metadata) = path.metadata() { + size += metadata.len() + } + } + + Ok(size) + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index d5cde13e7..27ffaa05e 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -41,6 +41,9 @@ impl UuidResolverActor { Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } + Some(GetSize { ret }) => { + let _ = ret.send(self.handle_get_size().await); + } // all senders have been dropped, need to quit. None => break, } @@ -86,6 +89,10 @@ impl UuidResolverActor { self.store.insert(uid, uuid).await?; Ok(()) } + + async fn handle_get_size(&self) -> Result { + self.store.get_size().await + } } fn is_index_uid_valid(uid: &str) -> bool { diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index f8625b379..c522e87e6 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -75,4 +75,13 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .await .expect("Uuid resolver actor has been killed")?) } + + async fn get_size(&self) -> Result { + let (ret, receiver) = oneshot::channel(); + let msg = UuidResolveMsg::GetSize { ret }; + let _ = self.sender.send(msg).await; + Ok(receiver + .await + .expect("Uuid resolver actor has been killed")?) + } } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 975c709b3..e7d29f05f 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -4,6 +4,7 @@ use tokio::sync::oneshot; use uuid::Uuid; use super::Result; + pub enum UuidResolveMsg { Get { uid: String, @@ -29,4 +30,7 @@ pub enum UuidResolveMsg { path: PathBuf, ret: oneshot::Sender>>, }, + GetSize { + ret: oneshot::Sender>, + }, } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index 43cd9995b..33a089ddb 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -30,6 +30,7 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; } #[derive(Debug, Error)] diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 435314911..1f057830b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -8,6 +8,7 @@ use heed::{ use uuid::Uuid; use super::{Result, UuidError, UUID_STORE_SIZE}; +use crate::helpers::EnvSizer; #[async_trait::async_trait] pub trait UuidStore { @@ -19,6 +20,7 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; + async fn get_size(&self) -> Result; } pub struct HeedUuidStore { @@ -151,4 +153,8 @@ impl UuidStore for HeedUuidStore { }) .await? } + + async fn get_size(&self) -> Result { + Ok(self.env.size()) + } }