From 4cafc635617ea450c2e69329df98a75315734975 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 25 Oct 2022 15:06:28 +0200 Subject: [PATCH] Reintroduce the versioning functions --- Cargo.lock | 1 + index-scheduler/src/batch.rs | 12 +++--- index-scheduler/src/insta_snapshot.rs | 1 + index-scheduler/src/lib.rs | 9 ++++ meilisearch-http/src/lib.rs | 4 +- meilisearch-types/Cargo.toml | 1 + meilisearch-types/src/lib.rs | 2 + meilisearch-types/src/versioning.rs | 61 +++++++++++++++++++++++++++ 8 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 meilisearch-types/src/versioning.rs diff --git a/Cargo.lock b/Cargo.lock index 35ebb4d0c..9a7677ab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2337,6 +2337,7 @@ name = "meilisearch-types" version = "0.29.1" dependencies = [ "actix-web", + "anyhow", "csv", "either", "enum-iterator", diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index 39ab862ba..c97bbf45b 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -33,7 +33,7 @@ use meilisearch_types::milli::update::{ use meilisearch_types::milli::{self, BEU32}; use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked}; use meilisearch_types::tasks::{Details, Kind, KindWithContent, Status, Task}; -use meilisearch_types::Index; +use meilisearch_types::{Index, VERSION_FILE_NAME}; use roaring::RoaringBitmap; use time::OffsetDateTime; use uuid::Uuid; @@ -559,9 +559,8 @@ impl IndexScheduler { // 1. Snapshot the version file. // TODO where can I find the path of this file and do we create it anyway? - // let dst = temp_snapshot_dir.path().join(VERSION_FILE_NAME); - // let src = self.base_path.join(VERSION_FILE_NAME); - // fs::copy(src, dst)?; + let dst = temp_snapshot_dir.path().join(VERSION_FILE_NAME); + fs::copy(&self.version_file_path, dst)?; // TODO what is a meta-env in the previous version of the scheduler? @@ -601,7 +600,7 @@ impl IndexScheduler { // 3. Snapshot every indexes // TODO we are opening all of the indexes it can be too much we should unload all // of the indexes we are trying to open. It would be even better to only unload - // the one that were opened by us. Or maybe use a LRU in the index mapper. + // the ones that were opened by us. Or maybe use a LRU in the index mapper. for result in self.index_mapper.index_mapping.iter(&rtxn)? { let (name, uuid) = result?; let index = self.index_mapper.index(&rtxn, name)?; @@ -618,7 +617,8 @@ impl IndexScheduler { // 4. Snapshot the auth LMDB env let dst = temp_snapshot_dir.path().join("auth").join("data.mdb"); fs::create_dir_all(&dst)?; - let auth = milli::heed::EnvOpenOptions::new().open(&self.auth_path)?; + let src = self.auth_path.join("data.mdb"); + let auth = milli::heed::EnvOpenOptions::new().open(src)?; auth.copy_to_path(dst, CompactionOption::Enabled)?; todo!("tar-gz and append .snapshot at the end of the file"); diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs index 6e5dd187a..dd5b6bb8b 100644 --- a/index-scheduler/src/insta_snapshot.rs +++ b/index-scheduler/src/insta_snapshot.rs @@ -28,6 +28,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { dumps_path: _, snapshots_path: _, auth_path: _, + version_file_path: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index cdb2e09e8..940b3eb4c 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -248,6 +248,9 @@ pub struct IndexScheduler { /// The path to the folder containing the auth LMDB env. pub(crate) auth_path: PathBuf, + /// The path to the version file of Meilisearch. + pub(crate) version_file_path: PathBuf, + // ================= test // The next entry is dedicated to the tests. /// Provide a way to set a breakpoint in multiple part of the scheduler. @@ -286,6 +289,7 @@ impl IndexScheduler { snapshots_path: self.snapshots_path.clone(), dumps_path: self.dumps_path.clone(), auth_path: self.auth_path.clone(), + version_file_path: self.version_file_path.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), #[cfg(test)] @@ -314,6 +318,7 @@ impl IndexScheduler { /// Create an index scheduler and start its run loop. /// /// ## Arguments + /// - `version_file_path`: the path to the version file of Meilisearch /// - `auth_path`: the path to the folder containing the auth LMDB env /// - `tasks_path`: the path to the folder containing the task databases /// - `update_file_path`: the path to the file store containing the files associated to the tasks @@ -326,6 +331,7 @@ impl IndexScheduler { /// together, to process multiple tasks at once. #[allow(clippy::too_many_arguments)] pub fn new( + version_file_path: PathBuf, auth_path: PathBuf, tasks_path: PathBuf, update_file_path: PathBuf, @@ -371,6 +377,7 @@ impl IndexScheduler { dumps_path, snapshots_path, auth_path, + version_file_path, #[cfg(test)] test_breakpoint_sdr, @@ -975,6 +982,8 @@ mod tests { let (sender, receiver) = crossbeam::channel::bounded(0); let index_scheduler = Self::new( + tempdir.path().join(VERSION_FILE_NAME), + tempdir.path().join("auth"), tempdir.path().join("db_path"), tempdir.path().join("file_store"), tempdir.path().join("indexes"), diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 6ab205385..135c5080c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -34,8 +34,8 @@ use index_scheduler::IndexScheduler; use meilisearch_auth::AuthController; use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod}; -use meilisearch_types::milli::{self}; use meilisearch_types::settings::apply_settings_to_builder; +use meilisearch_types::{milli, VERSION_FILE_NAME}; pub use option::Opt; use crate::error::MeilisearchHttpError; @@ -110,7 +110,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(IndexScheduler, AuthContr let auth_controller_builder = || AuthController::new(&opt.db_path, &opt.master_key); let index_scheduler_builder = || { IndexScheduler::new( - // TODO find a better way to have the path of the auth store + opt.db_path.join(VERSION_FILE_NAME), opt.db_path.join("auth"), opt.db_path.join("tasks"), opt.db_path.join("update_files"), diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 520df67e8..e0e48c65e 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] actix-web = { version = "4.2.1", default-features = false } +anyhow = "1.0.65" csv = "1.1.6" either = { version = "1.6.1", features = ["serde"] } enum-iterator = "1.1.3" diff --git a/meilisearch-types/src/lib.rs b/meilisearch-types/src/lib.rs index 40f9cdff1..f141de197 100644 --- a/meilisearch-types/src/lib.rs +++ b/meilisearch-types/src/lib.rs @@ -5,10 +5,12 @@ pub mod keys; pub mod settings; pub mod star_or; pub mod tasks; +pub mod versioning; pub use milli; pub use milli::{heed, Index}; use uuid::Uuid; +pub use versioning::VERSION_FILE_NAME; pub type Document = serde_json::Map; pub type InstanceUid = Uuid; diff --git a/meilisearch-types/src/versioning.rs b/meilisearch-types/src/versioning.rs new file mode 100644 index 000000000..bf1efe1ad --- /dev/null +++ b/meilisearch-types/src/versioning.rs @@ -0,0 +1,61 @@ +use std::fs; +use std::io::{self, ErrorKind}; +use std::path::Path; + +/// The name of the file that contains the version of the database. +pub const VERSION_FILE_NAME: &str = "VERSION"; + +static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); +static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); +static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); + +/// Persists the version of the current Meilisearch binary to a VERSION file +pub fn create_version_file(db_path: &Path) -> io::Result<()> { + let version_path = db_path.join(VERSION_FILE_NAME); + fs::write(version_path, format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)) +} + +/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch. +pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> { + let version_path = db_path.join(VERSION_FILE_NAME); + + match fs::read_to_string(&version_path) { + Ok(version) => { + let version_components = version.split('.').collect::>(); + let (major, minor, patch) = match &version_components[..] { + [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()), + _ => return Err(VersionFileError::MalformedVersionFile.into()), + }; + + if major != VERSION_MAJOR || minor != VERSION_MINOR { + return Err(VersionFileError::VersionMismatch { major, minor, patch }.into()); + } + } + Err(error) => { + return match error.kind() { + ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()), + _ => Err(error.into()), + } + } + } + + Ok(()) +} + +#[derive(thiserror::Error, Debug)] +pub enum VersionFileError { + #[error( + "Meilisearch (v{}) failed to infer the version of the database. + To update Meilisearch please follow our guide on https://docs.meilisearch.com/learn/advanced/updating.html.", + env!("CARGO_PKG_VERSION").to_string() + )] + MissingVersionFile, + #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")] + MalformedVersionFile, + #[error( + "Expected Meilisearch engine version: {major}.{minor}.{patch}, current engine version: {}. + To update Meilisearch please follow our guide on https://docs.meilisearch.com/learn/advanced/updating.html.", + env!("CARGO_PKG_VERSION").to_string() + )] + VersionMismatch { major: String, minor: String, patch: String }, +}