From f5a936614a3c3705a98b03c673ed156bf552788a Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Jun 2020 17:03:56 +0200 Subject: [PATCH 1/4] error on meili database version mismatch --- meilisearch-core/src/database.rs | 52 ++++++++++++++++++++++++++++++ meilisearch-core/src/error.rs | 55 +++++++++++++++++--------------- meilisearch-http/src/data.rs | 9 +++++- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index b9e952616..5510331a1 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -3,6 +3,7 @@ use std::fs::File; use std::path::Path; use std::sync::{Arc, RwLock}; use std::{fs, thread}; +use std::io::{Read, Write, ErrorKind}; use chrono::{DateTime, Utc}; use crossbeam_channel::{Receiver, Sender}; @@ -161,11 +162,62 @@ fn update_awaiter( Ok(()) } +/// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch. +/// If create is set to true, a VERSION file is created with the current version. +fn version_guard(path: &Path, create: bool) -> MResult<()> { + let current_version_major = env!("CARGO_PKG_VERSION_MAJOR"); + let current_version_minor = env!("CARGO_PKG_VERSION_MINOR"); + let current_version_patch = env!("CARGO_PKG_VERSION_PATCH"); + let version_path = path.join("VERSION"); + + match File::open(&version_path) { + Ok(mut file) => { + let mut version = String::new(); + file.read_to_string(&mut version)?; + let mut version = version.split("."); + + let version_major = version.next().ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; + let version_minor = version.next().ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; + + if version_major != current_version_major || version_minor != current_version_minor { + return Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_major))); + } + } + Err(error) => { + match error.kind() { + ErrorKind::NotFound => { + if create { + // when no version file is found, and we've beem told to create one, + // create a new file wioth the current version in it. + let mut version_file = File::create(&version_path)?; + version_file.write_all(format!("{}.{}.{}", + current_version_major, + current_version_minor, + current_version_patch).as_bytes())?; + } else { + // when no version file is found and we were not told to create one, this + // means that the version is inferior to the one this feature was added. + return Err(Error::VersionMismatch(format!("<0.12.0"))); + } + } + _ => return Err(error.into()) + } + } + } + Ok(()) +} + impl Database { pub fn open_or_create(path: impl AsRef, options: DatabaseOptions) -> MResult { let main_path = path.as_ref().join("main"); let update_path = path.as_ref().join("update"); + //create db directory + fs::create_dir_all(&path)?; + + // create file only if main db wasn't created before (first run) + version_guard(path.as_ref(), !main_path.exists() && !update_path.exists())?; + fs::create_dir_all(&main_path)?; let env = heed::EnvOpenOptions::new() .map_size(options.main_map_size) diff --git a/meilisearch-core/src/error.rs b/meilisearch-core/src/error.rs index 477063bac..379dbd8ff 100644 --- a/meilisearch-core/src/error.rs +++ b/meilisearch-core/src/error.rs @@ -15,22 +15,23 @@ pub type MResult = Result; #[derive(Debug)] pub enum Error { - Io(io::Error), - IndexAlreadyExists, - MissingPrimaryKey, - SchemaMissing, - WordIndexMissing, - MissingDocumentId, - MaxFieldsLimitExceeded, - Schema(meilisearch_schema::Error), - Heed(heed::Error), - Fst(fst::Error), - SerdeJson(SerdeJsonError), Bincode(bincode::Error), - Serializer(SerializerError), Deserializer(DeserializerError), - FilterParseError(PestError), FacetError(FacetError), + FilterParseError(PestError), + Fst(fst::Error), + Heed(heed::Error), + IndexAlreadyExists, + Io(io::Error), + MaxFieldsLimitExceeded, + MissingDocumentId, + MissingPrimaryKey, + Schema(meilisearch_schema::Error), + SchemaMissing, + SerdeJson(SerdeJsonError), + Serializer(SerializerError), + VersionMismatch(String), + WordIndexMissing, } impl ErrorCode for Error { @@ -53,6 +54,7 @@ impl ErrorCode for Error { | Bincode(_) | Serializer(_) | Deserializer(_) + | VersionMismatch(_) | Io(_) => Code::Internal, } } @@ -141,22 +143,23 @@ impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use self::Error::*; match self { - Io(e) => write!(f, "{}", e), - IndexAlreadyExists => write!(f, "index already exists"), - MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"), - SchemaMissing => write!(f, "this index does not have a schema"), - WordIndexMissing => write!(f, "this index does not have a word index"), - MissingDocumentId => write!(f, "document id is missing"), - MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"), - Schema(e) => write!(f, "schema error; {}", e), - Heed(e) => write!(f, "heed error; {}", e), - Fst(e) => write!(f, "fst error; {}", e), - SerdeJson(e) => write!(f, "serde json error; {}", e), Bincode(e) => write!(f, "bincode error; {}", e), - Serializer(e) => write!(f, "serializer error; {}", e), Deserializer(e) => write!(f, "deserializer error; {}", e), - FilterParseError(e) => write!(f, "error parsing filter; {}", e), FacetError(e) => write!(f, "error processing facet filter: {}", e), + FilterParseError(e) => write!(f, "error parsing filter; {}", e), + Fst(e) => write!(f, "fst error; {}", e), + Heed(e) => write!(f, "heed error; {}", e), + IndexAlreadyExists => write!(f, "index already exists"), + Io(e) => write!(f, "{}", e), + MaxFieldsLimitExceeded => write!(f, "maximum number of fields in a document exceeded"), + MissingDocumentId => write!(f, "document id is missing"), + MissingPrimaryKey => write!(f, "schema cannot be built without a primary key"), + Schema(e) => write!(f, "schema error; {}", e), + SchemaMissing => write!(f, "this index does not have a schema"), + SerdeJson(e) => write!(f, "serde json error; {}", e), + Serializer(e) => write!(f, "serializer error; {}", e), + VersionMismatch(version) => write!(f, "Cannot open database, expected Meilisearch version: {}", version), + WordIndexMissing => write!(f, "this index does not have a word index"), } } } diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs index 3692b0b69..04151d3cb 100644 --- a/meilisearch-http/src/data.rs +++ b/meilisearch-http/src/data.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use meilisearch_core::{Database, DatabaseOptions}; use sha2::Digest; use sysinfo::Pid; +use log::error; use crate::index_update_callback; use crate::option::Opt; @@ -66,7 +67,13 @@ impl Data { let http_payload_size_limit = opt.http_payload_size_limit; - let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap()); + let db = match Database::open_or_create(opt.db_path, db_opt) { + Ok(db) => Arc::new(db), + Err(e) => { + error!("{}", e); + std::process::exit(1); + } + }; let mut api_keys = ApiKeys { master: opt.master_key, From 6f0b6933e696941f3741e44e5fcbe2fa5b2502eb Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 23 Jun 2020 17:15:35 +0200 Subject: [PATCH 2/4] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcc9c8fe9..14501b85c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v0.13.0 (unreleased) -- placeholder search (#771) + - placeholder search (#771) + - Add database version mismatch check (#794) ## v0.12.0 From 51d7c84e7327b14f84522a1f5521f6d98c398a6f Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 24 Jun 2020 16:06:04 +0200 Subject: [PATCH 3/4] better exit on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update meilisearch-core/src/database.rs Co-authored-by: Clément Renault Update meilisearch-core/src/database.rs Co-authored-by: Clément Renault --- meilisearch-core/src/database.rs | 8 ++++---- meilisearch-core/src/error.rs | 6 +++++- meilisearch-http/src/data.rs | 14 ++++---------- meilisearch-http/src/main.rs | 2 +- meilisearch-http/tests/common.rs | 2 +- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 5510331a1..07fc6840f 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -180,15 +180,15 @@ fn version_guard(path: &Path, create: bool) -> MResult<()> { let version_minor = version.next().ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; if version_major != current_version_major || version_minor != current_version_minor { - return Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_major))); + return Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor))); } } Err(error) => { match error.kind() { ErrorKind::NotFound => { if create { - // when no version file is found, and we've beem told to create one, - // create a new file wioth the current version in it. + // when no version file is found, and we've been told to create one, + // create a new file with the current version in it. let mut version_file = File::create(&version_path)?; version_file.write_all(format!("{}.{}.{}", current_version_major, @@ -196,7 +196,7 @@ fn version_guard(path: &Path, create: bool) -> MResult<()> { current_version_patch).as_bytes())?; } else { // when no version file is found and we were not told to create one, this - // means that the version is inferior to the one this feature was added. + // means that the version is inferior to the one this feature was added in. return Err(Error::VersionMismatch(format!("<0.12.0"))); } } diff --git a/meilisearch-core/src/error.rs b/meilisearch-core/src/error.rs index 379dbd8ff..cf6340bff 100644 --- a/meilisearch-core/src/error.rs +++ b/meilisearch-core/src/error.rs @@ -158,7 +158,11 @@ impl fmt::Display for Error { SchemaMissing => write!(f, "this index does not have a schema"), SerdeJson(e) => write!(f, "serde json error; {}", e), Serializer(e) => write!(f, "serializer error; {}", e), - VersionMismatch(version) => write!(f, "Cannot open database, expected Meilisearch version: {}", version), + VersionMismatch(version) => write!(f, "Cannot open database, expected MeiliSearch engine version: {}, currrent engine version: {}.{}.{}", + version, + env!("CARGO_PKG_VERSION_MAJOR"), + env!("CARGO_PKG_VERSION_MINOR"), + env!("CARGO_PKG_VERSION_PATCH")), WordIndexMissing => write!(f, "this index does not have a word index"), } } diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs index 04151d3cb..3460b6bc1 100644 --- a/meilisearch-http/src/data.rs +++ b/meilisearch-http/src/data.rs @@ -1,10 +1,10 @@ +use std::error::Error; use std::ops::Deref; use std::sync::Arc; use meilisearch_core::{Database, DatabaseOptions}; use sha2::Digest; use sysinfo::Pid; -use log::error; use crate::index_update_callback; use crate::option::Opt; @@ -56,7 +56,7 @@ impl ApiKeys { } impl Data { - pub fn new(opt: Opt) -> Data { + pub fn new(opt: Opt) -> Result> { let db_path = opt.db_path.clone(); let server_pid = sysinfo::get_current_pid().unwrap(); @@ -67,13 +67,7 @@ impl Data { let http_payload_size_limit = opt.http_payload_size_limit; - let db = match Database::open_or_create(opt.db_path, db_opt) { - Ok(db) => Arc::new(db), - Err(e) => { - error!("{}", e); - std::process::exit(1); - } - }; + let db = Arc::new(Database::open_or_create(opt.db_path, db_opt)?); let mut api_keys = ApiKeys { master: opt.master_key, @@ -100,6 +94,6 @@ impl Data { index_update_callback(&index_uid, &callback_context, status); })); - data + Ok(data) } } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index ec211c6ae..13ce99d57 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -51,7 +51,7 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - let data = Data::new(opt.clone()); + let data = Data::new(opt.clone())?; if !opt.no_analytics { let analytics_data = data.clone(); diff --git a/meilisearch-http/tests/common.rs b/meilisearch-http/tests/common.rs index 9d63cf8b5..1906ab44f 100644 --- a/meilisearch-http/tests/common.rs +++ b/meilisearch-http/tests/common.rs @@ -50,7 +50,7 @@ impl Server { ..Opt::default() }; - let data = Data::new(opt.clone()); + let data = Data::new(opt.clone()).unwrap(); Server { uid: uid.to_string(), From 5773c5c86502f884474494d0fdf97c8855391c8c Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 26 Jun 2020 22:39:50 +0200 Subject: [PATCH 4/4] check version file against regex --- meilisearch-core/src/database.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 07fc6840f..f7ede3552 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -7,10 +7,11 @@ use std::io::{Read, Write, ErrorKind}; use chrono::{DateTime, Utc}; use crossbeam_channel::{Receiver, Sender}; -use heed::types::{Str, Unit, SerdeBincode}; use heed::CompactionOption; +use heed::types::{Str, Unit, SerdeBincode}; use log::{debug, error}; use meilisearch_schema::Schema; +use regex::Regex; use crate::{store, update, Index, MResult, Error}; @@ -174,10 +175,17 @@ fn version_guard(path: &Path, create: bool) -> MResult<()> { Ok(mut file) => { let mut version = String::new(); file.read_to_string(&mut version)?; - let mut version = version.split("."); + // Matches strings like XX.XX.XX + let re = Regex::new(r"(\d+).(\d+).(\d+)").unwrap(); - let version_major = version.next().ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; - let version_minor = version.next().ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; + // Make sure there is a result + let version = re + .captures_iter(&version) + .next() + .ok_or(Error::VersionMismatch("bad VERSION file".to_string()))?; + // the first is always the complete match, safe to unwrap because we have a match + let version_major = version.get(1).unwrap().as_str(); + let version_minor = version.get(2).unwrap().as_str(); if version_major != current_version_major || version_minor != current_version_minor { return Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor)));