searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 11:57:07 +02:00
Code Issues Releases Wiki Activity

bug(snapshot): Correctly open environments in snapshots

Browse source
This commit is contained in:
ad hoc 2022-02-22 18:16:02 +01:00 • committed by Clémentine Urquizar
parent ff6a7b6007
commit 4fbb83a34d
No known key found for this signature in database
GPG key ID: D8E7CC7422E77E1A
5 changed files with 25 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

13
meilisearch-lib/src/index_controller/mod.rs
View file

@ -48,6 +48,13 @@ pub type Payload = Box<
dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
>;
pub fn open_meta_env(path: &Path, size: usize) -> heed::Result<heed::Env> {
let mut options = heed::EnvOpenOptions::new();
options.map_size(size);
options.max_dbs(20);
options.open(path)
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct IndexMetadata {
@ -202,11 +209,7 @@ impl IndexControllerBuilder {
std::fs::create_dir_all(db_path.as_ref())?;
let mut options = heed::EnvOpenOptions::new();
options.map_size(task_store_size);
options.max_dbs(20);
let meta_env = Arc::new(options.open(&db_path)?);
let meta_env = Arc::new(open_meta_env(db_path.as_ref(), task_store_size)?);
let update_file_store = UpdateFileStore::new(&db_path)?;
// Create or overwrite the version file for this DB

18
meilisearch-lib/src/snapshot.rs
View file

@ -6,11 +6,13 @@ use std::time::Duration;
use anyhow::bail;
use fs_extra::dir::{self, CopyOptions};
use log::{info, trace};
use meilisearch_auth::open_auth_store_env;
use tokio::sync::RwLock;
use tokio::time::sleep;
use walkdir::WalkDir;
use crate::compression::from_tar_gz;
use crate::index_controller::open_meta_env;
use crate::index_controller::versioning::VERSION_FILE_NAME;
use crate::tasks::task::Job;
use crate::tasks::Scheduler;
@ -39,7 +41,6 @@ impl SnapshotService {
};
let job = Job::Snapshot(snapshot_job);
self.scheduler.write().await.schedule_job(job).await;
sleep(self.snapshot_period).await;
}
}
@ -145,9 +146,7 @@ impl SnapshotJob {
}
fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> {
let mut options = heed::EnvOpenOptions::new();
options.map_size(self.meta_env_size);
let env = options.open(&self.src_path)?;
let env = open_meta_env(&self.src_path, self.meta_env_size)?;
let dst = path.join("data.mdb");
env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
@ -183,9 +182,10 @@ impl SnapshotJob {
let mut options = heed::EnvOpenOptions::new();
options.map_size(self.index_size);
let env = options.open(entry.path())?;
env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
let index = milli::Index::new(options, entry.path())?;
index
.env
.copy_to_path(dst, heed::CompactionOption::Enabled)?;
}
Ok(())
@ -197,9 +197,7 @@ impl SnapshotJob {
std::fs::create_dir_all(&dst)?;
let dst = dst.join("data.mdb");
let mut options = heed::EnvOpenOptions::new();
options.map_size(1_073_741_824);
let env = options.open(auth_path)?;
let env = open_auth_store_env(&auth_path)?;
env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
Ok(())