searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-01-22 11:17:28 +01:00
Code Issues Releases Wiki Activity

starts adding tests and fix the starts of meilisearch

Browse Source
This commit is contained in:
Tamo 2025-01-16 12:56:23 +01:00
parent acebf68261
commit 7f8ee6657a
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
9 changed files with 105 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/index-scheduler/src/upgrade/mod.rs
View File

@ -20,7 +20,7 @@ pub fn upgrade_task_queue(tasks_path: &Path, from: (u32, u32, u32)) -> anyhow::R
[(v1_12_to_current as fn(&Path) -> anyhow::Result<()>, "Upgrading from v1.12 to v1.13")];
let start = match from {
(1, 12, _) => 0,
(1, 12, patch) if patch < current_patch => 0,
(major, minor, patch) => {
if major > current_major
|| (major == current_major && minor > current_minor)

31
crates/meilisearch-types/src/versioning.rs
View File

@ -24,7 +24,7 @@ pub fn create_version_file(
fs::write(version_path, format!("{}.{}.{}", major, minor, patch))
}
pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> {
pub fn get_version(db_path: &Path) -> Result<(u32, u32, u32), VersionFileError> {
let version_path = db_path.join(VERSION_FILE_NAME);
match fs::read_to_string(version_path) {
@ -36,11 +36,28 @@ pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFi
}
}
pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> {
pub fn parse_version(version: &str) -> Result<(u32, u32, u32), VersionFileError> {
let version_components = version.trim().split('.').collect::<Vec<_>>();
let (major, minor, patch) = match &version_components[..] {
[major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
_ => return Err(VersionFileError::MalformedVersionFile),
[major, minor, patch] => (
major.parse().map_err(|e| VersionFileError::MalformedVersionFile {
context: format!("Could not parse the major: {e}"),
})?,
minor.parse().map_err(|e| VersionFileError::MalformedVersionFile {
context: format!("Could not parse the minor: {e}"),
})?,
patch.parse().map_err(|e| VersionFileError::MalformedVersionFile {
context: format!("Could not parse the patch: {e}"),
})?,
),
_ => {
return Err(VersionFileError::MalformedVersionFile {
context: format!(
"The version contains {} parts instead of 3 (major, minor and patch)",
version_components.len()
),
})
}
};
Ok((major, minor, patch))
}
@ -53,14 +70,14 @@ pub enum VersionFileError {
env!("CARGO_PKG_VERSION").to_string()
)]
MissingVersionFile,
#[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database.")]
MalformedVersionFile,
#[error("Version file is corrupted and thus Meilisearch is unable to determine the version of the database. {context}")]
MalformedVersionFile { context: String },
#[error(
"Your database version ({major}.{minor}.{patch}) is incompatible with your current engine version ({}).\n\
To migrate data between Meilisearch versions, please follow our guide on https://www.meilisearch.com/docs/learn/update_and_migration/updating.",
env!("CARGO_PKG_VERSION").to_string()
)]
VersionMismatch { major: String, minor: String, patch: String },
VersionMismatch { major: u32, minor: u32, patch: u32 },
#[error(transparent)]
IoError(#[from] std::io::Error),

16
crates/meilisearch/src/lib.rs
View File

@ -35,12 +35,13 @@ use extractors::payload::PayloadConfig;
use index_scheduler::upgrade::upgrade_task_queue;
use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
use meilisearch_auth::AuthController;
use meilisearch_types::milli::constants::VERSION_MAJOR;
use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
use meilisearch_types::settings::apply_settings_to_builder;
use meilisearch_types::tasks::KindWithContent;
use meilisearch_types::versioning::{
create_current_version_file, get_version, VersionFileError, VERSION_MAJOR, VERSION_MINOR,
create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH,
};
use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
pub use option::Opt;
@ -345,14 +346,13 @@ fn check_version_and_update_task_queue(
) -> anyhow::Result<()> {
let (major, minor, patch) = get_version(db_path)?;
if major != VERSION_MAJOR || minor != VERSION_MINOR {
let version_major: u32 = VERSION_MAJOR.parse().unwrap();
let version_minor: u32 = VERSION_MINOR.parse().unwrap();
let version_patch: u32 = VERSION_PATCH.parse().unwrap();
if major != version_major || minor != version_minor || patch > version_patch {
if experimental_dumpless_upgrade {
let version = (
major.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
minor.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
patch.parse().map_err(|_| VersionFileError::MalformedVersionFile)?,
);
return upgrade_task_queue(&db_path.join("tasks"), version);
return upgrade_task_queue(&db_path.join("tasks"), (major, minor, patch));
} else {
return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
}

1
crates/meilisearch/tests/integration.rs
View File

@ -14,6 +14,7 @@ mod snapshot;
mod stats;
mod swap_indexes;
mod tasks;
mod upgrade;
mod vector;
// Tests are isolated by features in different modules to allow better readability, test

43
crates/meilisearch/tests/upgrade/mod.rs Normal file
View File

@ -0,0 +1,43 @@
use meili_snap::snapshot;
use meilisearch::Opt;
use crate::common::{default_settings, Server};
#[actix_rt::test]
async fn malformed_version_file() {
let temp = tempfile::tempdir().unwrap();
let default_settings = default_settings(temp.path());
let db_path = default_settings.db_path.clone();
std::fs::create_dir_all(&db_path).unwrap();
std::fs::write(db_path.join("VERSION"), "kefir").unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Version file is corrupted and thus Meilisearch is unable to determine the version of the database.");
}
#[actix_rt::test]
async fn version_too_old() {
let temp = tempfile::tempdir().unwrap();
let default_settings = default_settings(temp.path());
let db_path = default_settings.db_path.clone();
std::fs::create_dir_all(&db_path).unwrap();
std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and imports it in the v1.12.2");
}
#[actix_rt::test]
async fn version_requires_downgrade() {
let temp = tempfile::tempdir().unwrap();
let default_settings = default_settings(temp.path());
let db_path = default_settings.db_path.clone();
std::fs::create_dir_all(&db_path).unwrap();
let major = meilisearch_types::versioning::VERSION_MAJOR;
let minor = meilisearch_types::versioning::VERSION_MINOR;
let patch = meilisearch_types::versioning::VERSION_PATCH.parse::<u32>().unwrap() + 1;
std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.12.3 is higher than the binary version 1.12.2. Downgrade is not supported");
}

37
crates/meilitool/src/upgrade/mod.rs
View File

@ -14,15 +14,15 @@ use crate::upgrade::v1_11::v1_10_to_v1_11;
pub struct OfflineUpgrade {
pub db_path: PathBuf,
pub current_version: (String, String, String),
pub target_version: (String, String, String),
pub current_version: (u32, u32, u32),
pub target_version: (u32, u32, u32),
}
impl OfflineUpgrade {
pub fn upgrade(self) -> anyhow::Result<()> {
let upgrade_list = [
(
v1_9_to_v1_10 as fn(&Path, &str, &str, &str) -> Result<(), anyhow::Error>,
v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>,
"1",
"10",
"0",
@ -32,32 +32,25 @@ impl OfflineUpgrade {
(v1_12_to_v1_12_3, "1", "12", "3"),
];
let (current_major, current_minor, current_patch) = &self.current_version;
let (current_major, current_minor, current_patch) = self.current_version;
let start_at = match (
current_major.as_str(),
current_minor.as_str(),
current_patch.as_str(),
) {
("1", "9", _) => 0,
("1", "10", _) => 1,
("1", "11", _) => 2,
("1", "12", x) if x == "0" || x == "1" || x == "2" => 3,
let start_at = match (current_major, current_minor, current_patch) {
(1, 9, _) => 0,
(1, 10, _) => 1,
(1, 11, _) => 2,
(1, 12, x) if x == 0 || x == 1 || x == 2 => 3,
_ => {
bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9 and v1.10")
}
};
let (target_major, target_minor, target_patch) = &self.target_version;
let (target_major, target_minor, target_patch) = self.target_version;
let ends_at = match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) {
("1", "10", _) => 0,
("1", "11", _) => 1,
("1", "12", x) if x == "0" || x == "1" || x == "2" => 2,
("1", "12", "3") => 3,
(major, _, _) if major.starts_with('v') => {
bail!("Target version must not starts with a `v`. Instead of writing `v1.9.0` write `1.9.0` for example.")
}
let ends_at = match (target_major, target_minor, target_patch) {
(1, 10, _) => 0,
(1, 11, _) => 1,
(1, 12, x) if x == 0 || x == 1 || x == 2 => 2,
(1, 12, 3) => 3,
_ => {
bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10 and v1.11")
}

6
crates/meilitool/src/upgrade/v1_10.rs
View File

@ -153,9 +153,9 @@ fn date_round_trip(
pub fn v1_9_to_v1_10(
db_path: &Path,
_origin_major: &str,
_origin_minor: &str,
_origin_patch: &str,
_origin_major: u32,
_origin_minor: u32,
_origin_patch: u32,
) -> anyhow::Result<()> {
println!("Upgrading from v1.9.0 to v1.10.0");
// 2 changes here

6
crates/meilitool/src/upgrade/v1_11.rs
View File

@ -16,9 +16,9 @@ use crate::{try_opening_database, try_opening_poly_database};
pub fn v1_10_to_v1_11(
db_path: &Path,
_origin_major: &str,
_origin_minor: &str,
_origin_patch: &str,
_origin_major: u32,
_origin_minor: u32,
_origin_patch: u32,
) -> anyhow::Result<()> {
println!("Upgrading from v1.10.0 to v1.11.0");

14
crates/meilitool/src/upgrade/v1_12.rs
View File

@ -25,9 +25,9 @@ use crate::uuid_codec::UuidCodec;
pub fn v1_11_to_v1_12(
db_path: &Path,
_origin_major: &str,
_origin_minor: &str,
_origin_patch: &str,
_origin_major: u32,
_origin_minor: u32,
_origin_patch: u32,
) -> anyhow::Result<()> {
println!("Upgrading from v1.11.0 to v1.12.0");
@ -38,13 +38,13 @@ pub fn v1_11_to_v1_12(
pub fn v1_12_to_v1_12_3(
db_path: &Path,
origin_major: &str,
origin_minor: &str,
origin_patch: &str,
origin_major: u32,
origin_minor: u32,
origin_patch: u32,
) -> anyhow::Result<()> {
println!("Upgrading from v1.12.{{0, 1, 2}} to v1.12.3");
if origin_minor == "12" {
if origin_minor == 12 {
rebuild_field_distribution(db_path)?;
} else {
println!("Not rebuilding field distribution as it wasn't corrupted coming from v{origin_major}.{origin_minor}.{origin_patch}");