From 7e832105d7dd62505b3c2ea0908abc15c62440d4 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Tue, 22 Feb 2022 18:16:02 +0100 Subject: [PATCH] bug(snapshot): Correctly open environments in snapshots --- meilisearch-auth/src/lib.rs | 1 + meilisearch-auth/src/store.rs | 12 ++++++++---- meilisearch-http/src/routes/mod.rs | 1 - meilisearch-lib/src/index_controller/mod.rs | 13 ++++++++----- meilisearch-lib/src/snapshot.rs | 18 ++++++++---------- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index cfaa605a3..4edd73b1a 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -17,6 +17,7 @@ use sha2::{Digest, Sha256}; pub use action::{actions, Action}; use error::{AuthControllerError, Result}; pub use key::Key; +pub use store::open_auth_store_env; use store::HeedAuthStore; #[derive(Clone)] diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 757a8338b..19ff50990 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -39,14 +39,18 @@ impl Drop for HeedAuthStore { } } +pub fn open_auth_store_env(path: &Path) -> heed::Result { + let mut options = EnvOpenOptions::new(); + options.map_size(AUTH_STORE_SIZE); // 1GB + options.max_dbs(2); + options.open(path) +} + impl HeedAuthStore { pub fn new(path: impl AsRef) -> Result { let path = path.as_ref().join(AUTH_DB_PATH); create_dir_all(&path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(AUTH_STORE_SIZE); // 1GB - options.max_dbs(2); - let env = Arc::new(options.open(path)?); + let env = Arc::new(open_auth_store_env(path.as_ref())?); let keys = env.create_database(Some(KEY_DB_NAME))?; let action_keyid_index_expiration = env.create_database(Some(KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME))?; diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 63d537967..0ebc77042 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -128,7 +128,6 @@ async fn get_stats( meilisearch: GuardedData, MeiliSearch>, ) -> Result { let search_rules = &meilisearch.filters().search_rules; - let response = meilisearch.get_all_stats(search_rules).await?; debug!("returns: {:?}", response); diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 5dfaae848..34e37be82 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -48,6 +48,13 @@ pub type Payload = Box< dyn Stream> + Send + Sync + 'static + Unpin, >; +pub fn open_meta_env(path: &Path, size: usize) -> heed::Result { + let mut options = heed::EnvOpenOptions::new(); + options.map_size(size); + options.max_dbs(20); + options.open(path) +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { @@ -201,11 +208,7 @@ impl IndexControllerBuilder { std::fs::create_dir_all(db_path.as_ref())?; - let mut options = heed::EnvOpenOptions::new(); - options.map_size(task_store_size); - options.max_dbs(20); - - let meta_env = Arc::new(options.open(&db_path)?); + let meta_env = Arc::new(open_meta_env(db_path.as_ref(), task_store_size)?); let update_file_store = UpdateFileStore::new(&db_path)?; // Create or overwrite the version file for this DB diff --git a/meilisearch-lib/src/snapshot.rs b/meilisearch-lib/src/snapshot.rs index 528c0f64a..3667d1021 100644 --- a/meilisearch-lib/src/snapshot.rs +++ b/meilisearch-lib/src/snapshot.rs @@ -6,11 +6,13 @@ use std::time::Duration; use anyhow::bail; use fs_extra::dir::{self, CopyOptions}; use log::{info, trace}; +use meilisearch_auth::open_auth_store_env; use tokio::sync::RwLock; use tokio::time::sleep; use walkdir::WalkDir; use crate::compression::from_tar_gz; +use crate::index_controller::open_meta_env; use crate::index_controller::versioning::VERSION_FILE_NAME; use crate::tasks::task::Job; use crate::tasks::Scheduler; @@ -39,7 +41,6 @@ impl SnapshotService { }; let job = Job::Snapshot(snapshot_job); self.scheduler.write().await.schedule_job(job).await; - sleep(self.snapshot_period).await; } } @@ -145,9 +146,7 @@ impl SnapshotJob { } fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> { - let mut options = heed::EnvOpenOptions::new(); - options.map_size(self.meta_env_size); - let env = options.open(&self.src_path)?; + let env = open_meta_env(&self.src_path, self.meta_env_size)?; let dst = path.join("data.mdb"); env.copy_to_path(dst, heed::CompactionOption::Enabled)?; @@ -183,9 +182,10 @@ impl SnapshotJob { let mut options = heed::EnvOpenOptions::new(); options.map_size(self.index_size); - let env = options.open(entry.path())?; - - env.copy_to_path(dst, heed::CompactionOption::Enabled)?; + let index = milli::Index::new(options, entry.path())?; + index + .env + .copy_to_path(dst, heed::CompactionOption::Enabled)?; } Ok(()) @@ -197,9 +197,7 @@ impl SnapshotJob { std::fs::create_dir_all(&dst)?; let dst = dst.join("data.mdb"); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(1_073_741_824); - let env = options.open(auth_path)?; + let env = open_auth_store_env(&auth_path)?; env.copy_to_path(dst, heed::CompactionOption::Enabled)?; Ok(())