Merge #1965

1965: Reintroduce engine version file r=MarinPostma a=irevoire

Right now if you boot up MeiliSearch and point it to a DB directory created with a previous version of MeiliSearch the existing indexes will be deleted. This [used to be](51d7c84e73) prevented by a startup check which would compare the current engine version vs what was stored in the DB directory's version file, but this functionality seems to have been lost after a few refactorings of the code.

In order to go back to the old behavior we'll need to reintroduce the `VERSION` file that used to be present; I considered reusing the `metadata.json` file used in the dumps feature, but this seemed like the simpler and more approach. As the intent is just to restore functionality, the implementation is quite basic. I imagine that in the future we could build on this and do things like compatibility across major/minor versions and even migrating between formats.

This PR was made thanks to `@mbStavola` and is basically a port of his PR #1860 after a big refacto of the code #1796.

Closes #1840

Co-authored-by: Matt Stavola <m.freitas@offensive-security.com>

meilisearch-lib/src/index_controller/mod.rs

@@ -39,6 +39,7 @@ use crate::update_file_store::UpdateFileStore;
 
 mod dump_actor;
 pub mod error;
+pub mod versioning;
 
 /// Concrete implementation of the IndexController, exposed by meilisearch-lib
 pub type MeiliSearch = IndexController<HeedMetaStore, MapIndexStore>;
@@ -162,6 +163,11 @@ impl IndexControllerBuilder {
             .max_task_store_size
             .ok_or_else(|| anyhow::anyhow!("Missing update database size"))?;
 
+        let db_exists = db_path.as_ref().exists();
+        if db_exists {
+            versioning::check_version_file(db_path.as_ref())?;
+        }
+
         if let Some(ref path) = self.import_snapshot {
             log::info!("Loading from snapshot {:?}", path);
             load_snapshot(
@@ -189,6 +195,8 @@ impl IndexControllerBuilder {
         let meta_env = options.open(&db_path)?;
 
         let update_file_store = UpdateFileStore::new(&db_path)?;
+        // Create or overwrite the version file for this DB
+        versioning::create_version_file(db_path.as_ref())?;
 
         let index_resolver = Arc::new(create_index_resolver(
             &db_path,

meilisearch-lib/src/index_controller/versioning/error.rs

@@ -0,0 +1,16 @@
+#[derive(thiserror::Error, Debug)]
+pub enum VersionFileError {
+    #[error("Version file is missing or the previous MeiliSearch engine version was below 0.24.0. Use a dump to update Meilisearch.")]
+    MissingVersionFile,
+    #[error("Version file is corrupted and thus MeiliSearch is unable to determine the version of the database.")]
+    MalformedVersionFile,
+    #[error(
+        "Expected MeiliSearch engine version: {major}.{minor}.{patch}, current engine version: {}. To update Meilisearch use a dump.",
+        env!("CARGO_PKG_VERSION").to_string()
+    )]
+    VersionMismatch {
+        major: String,
+        minor: String,
+        patch: String,
+    },
+}

meilisearch-lib/src/index_controller/versioning/mod.rs

@@ -0,0 +1,56 @@
+use std::fs;
+use std::io::ErrorKind;
+use std::path::Path;
+
+use self::error::VersionFileError;
+
+mod error;
+
+pub const VERSION_FILE_NAME: &str = "VERSION";
+
+static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR");
+static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
+static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
+
+// Persists the version of the current MeiliSearch binary to a VERSION file
+pub fn create_version_file(db_path: &Path) -> anyhow::Result<()> {
+    let version_path = db_path.join(VERSION_FILE_NAME);
+    fs::write(
+        version_path,
+        format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH),
+    )?;
+
+    Ok(())
+}
+
+// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
+pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
+    let version_path = db_path.join(VERSION_FILE_NAME);
+
+    match fs::read_to_string(&version_path) {
+        Ok(version) => {
+            let version_components = version.split('.').collect::<Vec<_>>();
+            let (major, minor, patch) = match &version_components[..] {
+                [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
+                _ => return Err(VersionFileError::MalformedVersionFile.into()),
+            };
+
+            if major != VERSION_MAJOR || minor != VERSION_MINOR {
+                return Err(VersionFileError::VersionMismatch {
+                    major,
+                    minor,
+                    patch,
+                }
+                .into());
+            }
+        }
+        Err(error) => {
+            return match error.kind() {
+                ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()),
+                _ => Err(error.into()),
+            }
+        }
+    }
+
+    Ok(())
+}

meilisearch-lib/src/snapshot.rs

@@ -9,6 +9,7 @@ use tokio::time::sleep;
 use walkdir::WalkDir;
 
 use crate::compression::from_tar_gz;
+use crate::index_controller::versioning::VERSION_FILE_NAME;
 use crate::tasks::task::Job;
 use crate::tasks::TaskStore;
 
@@ -102,6 +103,7 @@ impl SnapshotJob {
         let temp_snapshot_dir = tempfile::tempdir()?;
         let temp_snapshot_path = temp_snapshot_dir.path();
 
+        self.snapshot_version_file(temp_snapshot_path)?;
         self.snapshot_meta_env(temp_snapshot_path)?;
         self.snapshot_file_store(temp_snapshot_path)?;
         self.snapshot_indexes(temp_snapshot_path)?;
@@ -133,6 +135,15 @@ impl SnapshotJob {
         Ok(())
     }
 
+    fn snapshot_version_file(&self, path: &Path) -> anyhow::Result<()> {
+        let dst = path.join(VERSION_FILE_NAME);
+        let src = self.src_path.join(VERSION_FILE_NAME);
+
+        fs::copy(src, dst)?;
+
+        Ok(())
+    }
+
     fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> {
         let mut options = heed::EnvOpenOptions::new();
         options.map_size(self.meta_env_size);