bug(snapshot): Correctly open environments in snapshots

This commit is contained in:
ad hoc 2022-02-22 18:16:02 +01:00 committed by Clémentine Urquizar
parent ff6a7b6007
commit 4fbb83a34d
No known key found for this signature in database
GPG Key ID: D8E7CC7422E77E1A
5 changed files with 25 additions and 20 deletions

View File

@ -17,6 +17,7 @@ use time::OffsetDateTime;
pub use action::{actions, Action}; pub use action::{actions, Action};
use error::{AuthControllerError, Result}; use error::{AuthControllerError, Result};
pub use key::Key; pub use key::Key;
pub use store::open_auth_store_env;
use store::HeedAuthStore; use store::HeedAuthStore;
#[derive(Clone)] #[derive(Clone)]

View File

@ -39,14 +39,18 @@ impl Drop for HeedAuthStore {
} }
} }
pub fn open_auth_store_env(path: &Path) -> heed::Result<heed::Env> {
let mut options = EnvOpenOptions::new();
options.map_size(AUTH_STORE_SIZE); // 1GB
options.max_dbs(2);
options.open(path)
}
impl HeedAuthStore { impl HeedAuthStore {
pub fn new(path: impl AsRef<Path>) -> Result<Self> { pub fn new(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref().join(AUTH_DB_PATH); let path = path.as_ref().join(AUTH_DB_PATH);
create_dir_all(&path)?; create_dir_all(&path)?;
let mut options = EnvOpenOptions::new(); let env = Arc::new(open_auth_store_env(path.as_ref())?);
options.map_size(AUTH_STORE_SIZE); // 1GB
options.max_dbs(2);
let env = Arc::new(options.open(path)?);
let keys = env.create_database(Some(KEY_DB_NAME))?; let keys = env.create_database(Some(KEY_DB_NAME))?;
let action_keyid_index_expiration = let action_keyid_index_expiration =
env.create_database(Some(KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME))?; env.create_database(Some(KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME))?;

View File

@ -136,7 +136,6 @@ async fn get_stats(
meilisearch: GuardedData<ActionPolicy<{ actions::STATS_GET }>, MeiliSearch>, meilisearch: GuardedData<ActionPolicy<{ actions::STATS_GET }>, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> { ) -> Result<HttpResponse, ResponseError> {
let search_rules = &meilisearch.filters().search_rules; let search_rules = &meilisearch.filters().search_rules;
let response = meilisearch.get_all_stats(search_rules).await?; let response = meilisearch.get_all_stats(search_rules).await?;
debug!("returns: {:?}", response); debug!("returns: {:?}", response);

View File

@ -48,6 +48,13 @@ pub type Payload = Box<
dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin, dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
>; >;
pub fn open_meta_env(path: &Path, size: usize) -> heed::Result<heed::Env> {
let mut options = heed::EnvOpenOptions::new();
options.map_size(size);
options.max_dbs(20);
options.open(path)
}
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct IndexMetadata { pub struct IndexMetadata {
@ -202,11 +209,7 @@ impl IndexControllerBuilder {
std::fs::create_dir_all(db_path.as_ref())?; std::fs::create_dir_all(db_path.as_ref())?;
let mut options = heed::EnvOpenOptions::new(); let meta_env = Arc::new(open_meta_env(db_path.as_ref(), task_store_size)?);
options.map_size(task_store_size);
options.max_dbs(20);
let meta_env = Arc::new(options.open(&db_path)?);
let update_file_store = UpdateFileStore::new(&db_path)?; let update_file_store = UpdateFileStore::new(&db_path)?;
// Create or overwrite the version file for this DB // Create or overwrite the version file for this DB

View File

@ -6,11 +6,13 @@ use std::time::Duration;
use anyhow::bail; use anyhow::bail;
use fs_extra::dir::{self, CopyOptions}; use fs_extra::dir::{self, CopyOptions};
use log::{info, trace}; use log::{info, trace};
use meilisearch_auth::open_auth_store_env;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tokio::time::sleep; use tokio::time::sleep;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::compression::from_tar_gz; use crate::compression::from_tar_gz;
use crate::index_controller::open_meta_env;
use crate::index_controller::versioning::VERSION_FILE_NAME; use crate::index_controller::versioning::VERSION_FILE_NAME;
use crate::tasks::task::Job; use crate::tasks::task::Job;
use crate::tasks::Scheduler; use crate::tasks::Scheduler;
@ -39,7 +41,6 @@ impl SnapshotService {
}; };
let job = Job::Snapshot(snapshot_job); let job = Job::Snapshot(snapshot_job);
self.scheduler.write().await.schedule_job(job).await; self.scheduler.write().await.schedule_job(job).await;
sleep(self.snapshot_period).await; sleep(self.snapshot_period).await;
} }
} }
@ -145,9 +146,7 @@ impl SnapshotJob {
} }
fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> { fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> {
let mut options = heed::EnvOpenOptions::new(); let env = open_meta_env(&self.src_path, self.meta_env_size)?;
options.map_size(self.meta_env_size);
let env = options.open(&self.src_path)?;
let dst = path.join("data.mdb"); let dst = path.join("data.mdb");
env.copy_to_path(dst, heed::CompactionOption::Enabled)?; env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
@ -183,9 +182,10 @@ impl SnapshotJob {
let mut options = heed::EnvOpenOptions::new(); let mut options = heed::EnvOpenOptions::new();
options.map_size(self.index_size); options.map_size(self.index_size);
let env = options.open(entry.path())?; let index = milli::Index::new(options, entry.path())?;
index
env.copy_to_path(dst, heed::CompactionOption::Enabled)?; .env
.copy_to_path(dst, heed::CompactionOption::Enabled)?;
} }
Ok(()) Ok(())
@ -197,9 +197,7 @@ impl SnapshotJob {
std::fs::create_dir_all(&dst)?; std::fs::create_dir_all(&dst)?;
let dst = dst.join("data.mdb"); let dst = dst.join("data.mdb");
let mut options = heed::EnvOpenOptions::new(); let env = open_auth_store_env(&auth_path)?;
options.map_size(1_073_741_824);
let env = options.open(auth_path)?;
env.copy_to_path(dst, heed::CompactionOption::Enabled)?; env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
Ok(()) Ok(())