From c92948b143dfd19b79c8a4cc61ccd3a198d168df Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 24 Jan 2023 16:17:23 +0100 Subject: [PATCH] Compute the size of the auth-controller, index-scheduler and all update files in the global stats --- file-store/src/lib.rs | 33 +++++++++++++++------------ index-scheduler/src/insta_snapshot.rs | 5 +++- index-scheduler/src/lib.rs | 9 ++++++++ index-scheduler/src/utils.rs | 19 +++++++++++---- meilisearch-auth/src/lib.rs | 5 ++++ meilisearch-auth/src/store.rs | 5 ++++ meilisearch-types/src/error.rs | 1 + meilisearch/src/routes/mod.rs | 12 +++++++++- 8 files changed, 67 insertions(+), 22 deletions(-) diff --git a/file-store/src/lib.rs b/file-store/src/lib.rs index e05694c92..ed36f3a91 100644 --- a/file-store/src/lib.rs +++ b/file-store/src/lib.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::fs::File as StdFile; use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; @@ -11,10 +10,14 @@ const UPDATE_FILES_PATH: &str = "updates/updates_files"; #[derive(Debug, thiserror::Error)] pub enum Error { + #[error("Could not parse file name as utf-8")] + CouldNotParseFileNameAsUtf8, #[error(transparent)] IoError(#[from] std::io::Error), #[error(transparent)] PersistError(#[from] tempfile::PersistError), + #[error(transparent)] + UuidError(#[from] uuid::Error), } pub type Result = std::result::Result; @@ -33,13 +36,11 @@ impl DerefMut for File { } } -#[cfg_attr(test, faux::create)] #[derive(Clone, Debug)] pub struct FileStore { path: PathBuf, } -#[cfg(not(test))] impl FileStore { pub fn new(path: impl AsRef) -> Result { let path = path.as_ref().to_path_buf(); @@ -48,7 +49,6 @@ impl FileStore { } } -#[cfg_attr(test, faux::methods)] impl FileStore { /// Creates a new temporary update file. /// A call to `persist` is needed to persist the file in the database. @@ -94,6 +94,14 @@ impl FileStore { Ok(()) } + pub fn update_total_size(&self) -> Result { + let mut total = 0; + for uuid in self.all_uuids()? { + total += self.get_size(uuid?)?; + } + Ok(total) + } + pub fn get_size(&self, uuid: Uuid) -> Result { Ok(self.get_update(uuid)?.metadata()?.len()) } @@ -105,17 +113,12 @@ impl FileStore { } /// List the Uuids of the files in the FileStore - /// - /// This function is meant to be used by tests only. - #[doc(hidden)] - pub fn __all_uuids(&self) -> BTreeSet { - let mut uuids = BTreeSet::new(); - for entry in self.path.read_dir().unwrap() { - let entry = entry.unwrap(); - let uuid = Uuid::from_str(entry.file_name().to_str().unwrap()).unwrap(); - uuids.insert(uuid); - } - uuids + pub fn all_uuids(&self) -> Result>> { + Ok(self.path.read_dir()?.map(|entry| { + Ok(Uuid::from_str( + entry?.file_name().to_str().ok_or(Error::CouldNotParseFileNameAsUtf8)?, + )?) + })) } } diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs index 0f0c9953a..e8d07ee63 100644 --- a/index-scheduler/src/insta_snapshot.rs +++ b/index-scheduler/src/insta_snapshot.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::fmt::Write; use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str}; @@ -92,7 +93,9 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { pub fn snapshot_file_store(file_store: &file_store::FileStore) -> String { let mut snap = String::new(); - for uuid in file_store.__all_uuids() { + // we store the uuid in a `BTreeSet` to keep them ordered. + let all_uuids = file_store.all_uuids().unwrap().collect::, _>>().unwrap(); + for uuid in all_uuids { snap.push_str(&format!("{uuid}\n")); } snap diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 895b97813..1cc28c260 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -452,6 +452,10 @@ impl IndexScheduler { &self.index_mapper.indexer_config } + pub fn size(&self) -> Result { + Ok(self.env.real_disk_size()?) + } + /// Return the index corresponding to the name. /// /// * If the index wasn't opened before, the index will be opened. @@ -898,6 +902,11 @@ impl IndexScheduler { Ok(self.file_store.new_update_with_uuid(uuid)?) } + /// List the update files contained in the IndexScheduler. + pub fn update_file_size(&self) -> Result { + Ok(self.file_store.update_total_size()?) + } + /// Delete a file from the index scheduler. /// /// Counterpart to the [`create_update_file`](IndexScheduler::create_update_file) method. diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index e13d0e375..8c50e00c7 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -508,14 +508,23 @@ impl IndexScheduler { if let KindWithContent::DocumentAdditionOrUpdate { content_file, .. } = kind { match status { Status::Enqueued | Status::Processing => { - assert!( - self.file_store.__all_uuids().contains(&content_file), - "Could not find uuid `{content_file}` in the file_store. Available uuids are {:?}.", - self.file_store.__all_uuids(), + assert!(self + .file_store + .all_uuids() + .unwrap() + .find(|uuid| uuid.as_ref().unwrap() == &content_file) + .is_some(), + "Could not find uuid `{content_file}` in the file_store. Available uuids are {:?}.", + self.file_store.all_uuids().unwrap().collect::>>(), ); } Status::Succeeded | Status::Failed | Status::Canceled => { - assert!(!self.file_store.__all_uuids().contains(&content_file)); + assert!(self + .file_store + .all_uuids() + .unwrap() + .find(|uuid| uuid.as_ref().unwrap() == &content_file) + .is_none()); } } } diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 609409cf7..adfd00ce5 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -33,6 +33,11 @@ impl AuthController { Ok(Self { store: Arc::new(store), master_key: master_key.clone() }) } + /// Return the size of the `AuthController` database in bytes. + pub fn size(&self) -> Result { + self.store.size() + } + pub fn create_key(&self, create_key: CreateApiKey) -> Result { match self.store.get_api_key(create_key.uid)? { Some(_) => Err(AuthControllerError::ApiKeyAlreadyExists(create_key.uid.to_string())), diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 2574572be..c1cec0ede 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -60,6 +60,11 @@ impl HeedAuthStore { Ok(Self { env, keys, action_keyid_index_expiration, should_close_on_drop: true }) } + /// Return the size in bytes of database + pub fn size(&self) -> Result { + Ok(self.env.real_disk_size()?) + } + pub fn set_drop_on_close(&mut self, v: bool) { self.should_close_on_drop = v; } diff --git a/meilisearch-types/src/error.rs b/meilisearch-types/src/error.rs index 0f551d584..39d9a1551 100644 --- a/meilisearch-types/src/error.rs +++ b/meilisearch-types/src/error.rs @@ -343,6 +343,7 @@ impl ErrorCode for file_store::Error { match self { Self::IoError(e) => e.error_code(), Self::PersistError(e) => e.error_code(), + Self::CouldNotParseFileNameAsUtf8 | Self::UuidError(_) => Code::Internal, } } } diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 9ef036554..eaf014a81 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -4,6 +4,7 @@ use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; use index_scheduler::{IndexScheduler, Query}; use log::debug; +use meilisearch_auth::AuthController; use meilisearch_types::error::ResponseError; use meilisearch_types::settings::{Settings, Unchecked}; use meilisearch_types::tasks::{Kind, Status, Task, TaskId}; @@ -230,13 +231,15 @@ pub struct Stats { async fn get_stats( index_scheduler: GuardedData, Data>, + auth_controller: GuardedData, AuthController>, req: HttpRequest, analytics: web::Data, ) -> Result { analytics.publish("Stats Seen".to_string(), json!({ "per_index_uid": false }), Some(&req)); let search_rules = &index_scheduler.filters().search_rules; - let stats = create_all_stats((*index_scheduler).clone(), search_rules)?; + let stats = + create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), search_rules)?; debug!("returns: {:?}", stats); Ok(HttpResponse::Ok().json(stats)) @@ -244,6 +247,7 @@ async fn get_stats( pub fn create_all_stats( index_scheduler: Data, + auth_controller: AuthController, search_rules: &meilisearch_auth::SearchRules, ) -> Result { let mut last_task: Option = None; @@ -253,6 +257,7 @@ pub fn create_all_stats( Query { statuses: Some(vec![Status::Processing]), limit: Some(1), ..Query::default() }, search_rules.authorized_indexes(), )?; + // accumulate the size of each indexes let processing_index = processing_task.first().and_then(|task| task.index_uid()); for (name, index) in index_scheduler.indexes()? { if !search_rules.is_index_authorized(&name) { @@ -273,6 +278,11 @@ pub fn create_all_stats( indexes.insert(name, stats); } + + database_size += index_scheduler.size()?; + database_size += auth_controller.size()?; + database_size += index_scheduler.update_file_size()?; + let stats = Stats { database_size, last_update: last_task, indexes }; Ok(stats) }