From 7f8ee6657a3d1d6a4e447fd05e08e4383c7c10bf Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 16 Jan 2025 12:56:23 +0100 Subject: [PATCH] starts adding tests and fix the starts of meilisearch --- crates/index-scheduler/src/upgrade/mod.rs | 2 +- crates/meilisearch-types/src/versioning.rs | 31 ++++++++++++---- crates/meilisearch/src/lib.rs | 16 ++++---- crates/meilisearch/tests/integration.rs | 1 + crates/meilisearch/tests/upgrade/mod.rs | 43 ++++++++++++++++++++++ crates/meilitool/src/upgrade/mod.rs | 37 ++++++++----------- crates/meilitool/src/upgrade/v1_10.rs | 6 +-- crates/meilitool/src/upgrade/v1_11.rs | 6 +-- crates/meilitool/src/upgrade/v1_12.rs | 14 +++---- 9 files changed, 105 insertions(+), 51 deletions(-) create mode 100644 crates/meilisearch/tests/upgrade/mod.rs diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index a0ad32f57..7dc4f8055 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -20,7 +20,7 @@ pub fn upgrade_task_queue(tasks_path: &Path, from: (u32, u32, u32)) -> anyhow::R [(v1_12_to_current as fn(&Path) -> anyhow::Result<()>, "Upgrading from v1.12 to v1.13")]; let start = match from { - (1, 12, _) => 0, + (1, 12, patch) if patch < current_patch => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) diff --git a/crates/meilisearch-types/src/versioning.rs b/crates/meilisearch-types/src/versioning.rs index 081c95c6e..054d2e312 100644 --- a/crates/meilisearch-types/src/versioning.rs +++ b/crates/meilisearch-types/src/versioning.rs @@ -24,7 +24,7 @@ pub fn create_version_file( fs::write(version_path, format!("{}.{}.{}", major, minor, patch)) } -pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> { +pub fn get_version(db_path: &Path) -> Result<(u32, u32, u32), VersionFileError> { let version_path = db_path.join(VERSION_FILE_NAME); match fs::read_to_string(version_path) { @@ -36,11 +36,28 @@ pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFi } } -pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> { +pub fn parse_version(version: &str) -> Result<(u32, u32, u32), VersionFileError> { let version_components = version.trim().split('.').collect::>(); let (major, minor, patch) = match &version_components[..] { - [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()), - _ => return Err(VersionFileError::MalformedVersionFile), + [major, minor, patch] => ( + major.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the major: {e}"), + })?, + minor.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the minor: {e}"), + })?, + patch.parse().map_err(|e| VersionFileError::MalformedVersionFile { + context: format!("Could not parse the patch: {e}"), + })?, + ), + _ => { + return Err(VersionFileError::MalformedVersionFile { + context: format!( + "The version contains {} parts instead of 3 (major, minor and patch)", + version_components.len() + ), + }) + } }; Ok((major, minor, patch)) } @@ -53,14 +70,14 @@ pub enum VersionFileError { env!("CARGO_PKG_VERSION").to_string() )] MissingVersionFile, - #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")] - MalformedVersionFile, + #[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database. {context}")] + MalformedVersionFile { context: String }, #[error( "Your database version ({major}.{minor}.{patch}) is incompatible with your current engine version ({}).\n\ To migrate data between Meilisearch versions, please follow our guide on https://www.meilisearch.com/docs/learn/update_and_migration/updating.", env!("CARGO_PKG_VERSION").to_string() )] - VersionMismatch { major: String, minor: String, patch: String }, + VersionMismatch { major: u32, minor: u32, patch: u32 }, #[error(transparent)] IoError(#[from] std::io::Error), diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index ebdaab7b6..4f513532b 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -35,12 +35,13 @@ use extractors::payload::PayloadConfig; use index_scheduler::upgrade::upgrade_task_queue; use index_scheduler::{IndexScheduler, IndexSchedulerOptions}; use meilisearch_auth::AuthController; +use meilisearch_types::milli::constants::VERSION_MAJOR; use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod}; use meilisearch_types::settings::apply_settings_to_builder; use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{ - create_current_version_file, get_version, VersionFileError, VERSION_MAJOR, VERSION_MINOR, + create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH, }; use meilisearch_types::{compression, milli, VERSION_FILE_NAME}; pub use option::Opt; @@ -345,14 +346,13 @@ fn check_version_and_update_task_queue( ) -> anyhow::Result<()> { let (major, minor, patch) = get_version(db_path)?; - if major != VERSION_MAJOR || minor != VERSION_MINOR { + let version_major: u32 = VERSION_MAJOR.parse().unwrap(); + let version_minor: u32 = VERSION_MINOR.parse().unwrap(); + let version_patch: u32 = VERSION_PATCH.parse().unwrap(); + + if major != version_major || minor != version_minor || patch > version_patch { if experimental_dumpless_upgrade { - let version = ( - major.parse().map_err(|_| VersionFileError::MalformedVersionFile)?, - minor.parse().map_err(|_| VersionFileError::MalformedVersionFile)?, - patch.parse().map_err(|_| VersionFileError::MalformedVersionFile)?, - ); - return upgrade_task_queue(&db_path.join("tasks"), version); + return upgrade_task_queue(&db_path.join("tasks"), (major, minor, patch)); } else { return Err(VersionFileError::VersionMismatch { major, minor, patch }.into()); } diff --git a/crates/meilisearch/tests/integration.rs b/crates/meilisearch/tests/integration.rs index 85deb9cdf..7c3b8affe 100644 --- a/crates/meilisearch/tests/integration.rs +++ b/crates/meilisearch/tests/integration.rs @@ -14,6 +14,7 @@ mod snapshot; mod stats; mod swap_indexes; mod tasks; +mod upgrade; mod vector; // Tests are isolated by features in different modules to allow better readability, test diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs new file mode 100644 index 000000000..152f49850 --- /dev/null +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -0,0 +1,43 @@ +use meili_snap::snapshot; +use meilisearch::Opt; + +use crate::common::{default_settings, Server}; + +#[actix_rt::test] +async fn malformed_version_file() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("VERSION"), "kefir").unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Version file is corrupted and thus Meilisearch is unable to determine the version of the database."); +} + +#[actix_rt::test] +async fn version_too_old() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and imports it in the v1.12.2"); +} + +#[actix_rt::test] +async fn version_requires_downgrade() { + let temp = tempfile::tempdir().unwrap(); + let default_settings = default_settings(temp.path()); + let db_path = default_settings.db_path.clone(); + std::fs::create_dir_all(&db_path).unwrap(); + let major = meilisearch_types::versioning::VERSION_MAJOR; + let minor = meilisearch_types::versioning::VERSION_MINOR; + let patch = meilisearch_types::versioning::VERSION_PATCH.parse::().unwrap() + 1; + std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); + let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); + snapshot!(err, @"Database version 1.12.3 is higher than the binary version 1.12.2. Downgrade is not supported"); +} diff --git a/crates/meilitool/src/upgrade/mod.rs b/crates/meilitool/src/upgrade/mod.rs index 47ca2cbd9..1b060db04 100644 --- a/crates/meilitool/src/upgrade/mod.rs +++ b/crates/meilitool/src/upgrade/mod.rs @@ -14,15 +14,15 @@ use crate::upgrade::v1_11::v1_10_to_v1_11; pub struct OfflineUpgrade { pub db_path: PathBuf, - pub current_version: (String, String, String), - pub target_version: (String, String, String), + pub current_version: (u32, u32, u32), + pub target_version: (u32, u32, u32), } impl OfflineUpgrade { pub fn upgrade(self) -> anyhow::Result<()> { let upgrade_list = [ ( - v1_9_to_v1_10 as fn(&Path, &str, &str, &str) -> Result<(), anyhow::Error>, + v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, "1", "10", "0", @@ -32,32 +32,25 @@ impl OfflineUpgrade { (v1_12_to_v1_12_3, "1", "12", "3"), ]; - let (current_major, current_minor, current_patch) = &self.current_version; + let (current_major, current_minor, current_patch) = self.current_version; - let start_at = match ( - current_major.as_str(), - current_minor.as_str(), - current_patch.as_str(), - ) { - ("1", "9", _) => 0, - ("1", "10", _) => 1, - ("1", "11", _) => 2, - ("1", "12", x) if x == "0" || x == "1" || x == "2" => 3, + let start_at = match (current_major, current_minor, current_patch) { + (1, 9, _) => 0, + (1, 10, _) => 1, + (1, 11, _) => 2, + (1, 12, x) if x == 0 || x == 1 || x == 2 => 3, _ => { bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9 and v1.10") } }; - let (target_major, target_minor, target_patch) = &self.target_version; + let (target_major, target_minor, target_patch) = self.target_version; - let ends_at = match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) { - ("1", "10", _) => 0, - ("1", "11", _) => 1, - ("1", "12", x) if x == "0" || x == "1" || x == "2" => 2, - ("1", "12", "3") => 3, - (major, _, _) if major.starts_with('v') => { - bail!("Target version must not starts with a `v`. Instead of writing `v1.9.0` write `1.9.0` for example.") - } + let ends_at = match (target_major, target_minor, target_patch) { + (1, 10, _) => 0, + (1, 11, _) => 1, + (1, 12, x) if x == 0 || x == 1 || x == 2 => 2, + (1, 12, 3) => 3, _ => { bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10 and v1.11") } diff --git a/crates/meilitool/src/upgrade/v1_10.rs b/crates/meilitool/src/upgrade/v1_10.rs index a35fd4184..043520e82 100644 --- a/crates/meilitool/src/upgrade/v1_10.rs +++ b/crates/meilitool/src/upgrade/v1_10.rs @@ -153,9 +153,9 @@ fn date_round_trip( pub fn v1_9_to_v1_10( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.9.0 to v1.10.0"); // 2 changes here diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index e24a35e8b..44aeb125f 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -16,9 +16,9 @@ use crate::{try_opening_database, try_opening_poly_database}; pub fn v1_10_to_v1_11( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.10.0 to v1.11.0"); diff --git a/crates/meilitool/src/upgrade/v1_12.rs b/crates/meilitool/src/upgrade/v1_12.rs index 593fb833c..67a19c370 100644 --- a/crates/meilitool/src/upgrade/v1_12.rs +++ b/crates/meilitool/src/upgrade/v1_12.rs @@ -25,9 +25,9 @@ use crate::uuid_codec::UuidCodec; pub fn v1_11_to_v1_12( db_path: &Path, - _origin_major: &str, - _origin_minor: &str, - _origin_patch: &str, + _origin_major: u32, + _origin_minor: u32, + _origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.11.0 to v1.12.0"); @@ -38,13 +38,13 @@ pub fn v1_11_to_v1_12( pub fn v1_12_to_v1_12_3( db_path: &Path, - origin_major: &str, - origin_minor: &str, - origin_patch: &str, + origin_major: u32, + origin_minor: u32, + origin_patch: u32, ) -> anyhow::Result<()> { println!("Upgrading from v1.12.{{0, 1, 2}} to v1.12.3"); - if origin_minor == "12" { + if origin_minor == 12 { rebuild_field_distribution(db_path)?; } else { println!("Not rebuilding field distribution as it wasn't corrupted coming from v{origin_major}.{origin_minor}.{origin_patch}");