Reintroduce the versioning functions

This commit is contained in:
Kerollmops 2022-10-25 15:06:28 +02:00 committed by Tamo
parent 5151bde209
commit 19910f912d
8 changed files with 83 additions and 8 deletions

View file

@ -33,7 +33,7 @@ use meilisearch_types::milli::update::{
use meilisearch_types::milli::{self, BEU32};
use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
use meilisearch_types::tasks::{Details, Kind, KindWithContent, Status, Task};
use meilisearch_types::Index;
use meilisearch_types::{Index, VERSION_FILE_NAME};
use roaring::RoaringBitmap;
use time::OffsetDateTime;
use uuid::Uuid;
@ -559,9 +559,8 @@ impl IndexScheduler {
// 1. Snapshot the version file.
// TODO where can I find the path of this file and do we create it anyway?
// let dst = temp_snapshot_dir.path().join(VERSION_FILE_NAME);
// let src = self.base_path.join(VERSION_FILE_NAME);
// fs::copy(src, dst)?;
let dst = temp_snapshot_dir.path().join(VERSION_FILE_NAME);
fs::copy(&self.version_file_path, dst)?;
// TODO what is a meta-env in the previous version of the scheduler?
@ -601,7 +600,7 @@ impl IndexScheduler {
// 3. Snapshot every indexes
// TODO we are opening all of the indexes it can be too much we should unload all
// of the indexes we are trying to open. It would be even better to only unload
// the one that were opened by us. Or maybe use a LRU in the index mapper.
// the ones that were opened by us. Or maybe use a LRU in the index mapper.
for result in self.index_mapper.index_mapping.iter(&rtxn)? {
let (name, uuid) = result?;
let index = self.index_mapper.index(&rtxn, name)?;
@ -618,7 +617,8 @@ impl IndexScheduler {
// 4. Snapshot the auth LMDB env
let dst = temp_snapshot_dir.path().join("auth").join("data.mdb");
fs::create_dir_all(&dst)?;
let auth = milli::heed::EnvOpenOptions::new().open(&self.auth_path)?;
let src = self.auth_path.join("data.mdb");
let auth = milli::heed::EnvOpenOptions::new().open(src)?;
auth.copy_to_path(dst, CompactionOption::Enabled)?;
todo!("tar-gz and append .snapshot at the end of the file");

View file

@ -28,6 +28,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
dumps_path: _,
snapshots_path: _,
auth_path: _,
version_file_path: _,
test_breakpoint_sdr: _,
planned_failures: _,
run_loop_iteration: _,

View file

@ -248,6 +248,9 @@ pub struct IndexScheduler {
/// The path to the folder containing the auth LMDB env.
pub(crate) auth_path: PathBuf,
/// The path to the version file of Meilisearch.
pub(crate) version_file_path: PathBuf,
// ================= test
// The next entry is dedicated to the tests.
/// Provide a way to set a breakpoint in multiple part of the scheduler.
@ -286,6 +289,7 @@ impl IndexScheduler {
snapshots_path: self.snapshots_path.clone(),
dumps_path: self.dumps_path.clone(),
auth_path: self.auth_path.clone(),
version_file_path: self.version_file_path.clone(),
#[cfg(test)]
test_breakpoint_sdr: self.test_breakpoint_sdr.clone(),
#[cfg(test)]
@ -314,6 +318,7 @@ impl IndexScheduler {
/// Create an index scheduler and start its run loop.
///
/// ## Arguments
/// - `version_file_path`: the path to the version file of Meilisearch
/// - `auth_path`: the path to the folder containing the auth LMDB env
/// - `tasks_path`: the path to the folder containing the task databases
/// - `update_file_path`: the path to the file store containing the files associated to the tasks
@ -326,6 +331,7 @@ impl IndexScheduler {
/// together, to process multiple tasks at once.
#[allow(clippy::too_many_arguments)]
pub fn new(
version_file_path: PathBuf,
auth_path: PathBuf,
tasks_path: PathBuf,
update_file_path: PathBuf,
@ -371,6 +377,7 @@ impl IndexScheduler {
dumps_path,
snapshots_path,
auth_path,
version_file_path,
#[cfg(test)]
test_breakpoint_sdr,
@ -975,6 +982,8 @@ mod tests {
let (sender, receiver) = crossbeam::channel::bounded(0);
let index_scheduler = Self::new(
tempdir.path().join(VERSION_FILE_NAME),
tempdir.path().join("auth"),
tempdir.path().join("db_path"),
tempdir.path().join("file_store"),
tempdir.path().join("indexes"),