diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dc0d8d43..8168d7977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.10.2 + +- Add support for configuring the lmdb map size (#646) + ## v0.10.1 - Add support for floating points in filters (#640) diff --git a/meilisearch-core/examples/from_file.rs b/meilisearch-core/examples/from_file.rs index 2a30d1ce4..fad6f4e3c 100644 --- a/meilisearch-core/examples/from_file.rs +++ b/meilisearch-core/examples/from_file.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use structopt::StructOpt; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; -use meilisearch_core::{Database, Highlight, ProcessedUpdateResult}; +use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult}; use meilisearch_core::settings::Settings; use meilisearch_schema::FieldId; @@ -463,7 +463,7 @@ fn main() -> Result<(), Box> { env_logger::init(); let opt = Command::from_args(); - let database = Database::open_or_create(opt.path())?; + let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?; match opt { Command::Index(command) => index_command(command, database), diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 87fc24c11..eab085fc5 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -135,20 +135,34 @@ fn update_awaiter( Ok(()) } +pub struct DatabaseOptions { + pub main_map_size: usize, + pub update_map_size: usize +} + +impl Default for DatabaseOptions { + fn default() -> DatabaseOptions { + DatabaseOptions { + main_map_size: 100 * 1024 * 1024 * 1024, // 100GB + update_map_size: 100 * 1024 * 1024 * 1024 // 100GB + } + } +} + impl Database { - pub fn open_or_create(path: impl AsRef) -> MResult { + pub fn open_or_create(path: impl AsRef, options: DatabaseOptions) -> MResult { let main_path = path.as_ref().join("main"); let update_path = path.as_ref().join("update"); fs::create_dir_all(&main_path)?; let env = heed::EnvOpenOptions::new() - .map_size(100 * 1024 * 1024 * 1024) // 100GB + .map_size(options.main_map_size) .max_dbs(3000) .open(main_path)?; fs::create_dir_all(&update_path)?; let update_env = heed::EnvOpenOptions::new() - .map_size(100 * 1024 * 1024 * 1024) // 100GB + .map_size(options.update_map_size) .max_dbs(3000) .open(update_path)?; @@ -368,7 +382,7 @@ mod tests { fn valid_updates() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -433,7 +447,7 @@ mod tests { fn invalid_updates() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -495,7 +509,7 @@ mod tests { fn ignored_words_too_long() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -550,7 +564,7 @@ mod tests { fn add_schema_attributes_at_end() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -694,7 +708,7 @@ mod tests { fn deserialize_documents() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -774,7 +788,7 @@ mod tests { fn partial_document_update() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -908,7 +922,7 @@ mod tests { fn delete_index() { let dir = tempfile::tempdir().unwrap(); - let database = Arc::new(Database::open_or_create(dir.path()).unwrap()); + let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap()); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -980,7 +994,7 @@ mod tests { fn check_number_ordering() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path()).unwrap(); + let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); diff --git a/meilisearch-core/src/lib.rs b/meilisearch-core/src/lib.rs index 6ba37259b..87db276a0 100644 --- a/meilisearch-core/src/lib.rs +++ b/meilisearch-core/src/lib.rs @@ -25,7 +25,7 @@ pub mod raw_indexer; pub mod serde; pub mod store; -pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT}; +pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT}; pub use self::error::{Error, HeedError, FstError, MResult, pest_error}; pub use self::filters::Filter; pub use self::number::{Number, ParseNumberError}; diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs index 68ed97537..d95c19713 100644 --- a/meilisearch-core/src/query_builder.rs +++ b/meilisearch-core/src/query_builder.rs @@ -143,7 +143,7 @@ mod tests { use crate::DocIndex; use crate::automaton::normalize_str; use crate::bucket_sort::SimpleMatch; - use crate::database::Database; + use crate::database::{Database,DatabaseOptions}; use crate::store::Index; use meilisearch_schema::Schema; @@ -249,7 +249,7 @@ mod tests { impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase { fn from_iter>(iter: I) -> Self { let tempdir = TempDir::new().unwrap(); - let database = Database::open_or_create(&tempdir).unwrap(); + let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap(); let index = database.create_index("default").unwrap(); let db = &database; diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs index cdfdfb80d..da171b53d 100644 --- a/meilisearch-http/src/data.rs +++ b/meilisearch-http/src/data.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use heed::types::{SerdeBincode, Str}; use log::error; -use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT}; +use meilisearch_core::{Database, DatabaseOptions, Error as MError, MResult, MainT, UpdateT}; use sha2::Digest; use sysinfo::Pid; @@ -132,7 +132,12 @@ impl Data { let db_path = opt.db_path.clone(); let server_pid = sysinfo::get_current_pid().unwrap(); - let db = Arc::new(Database::open_or_create(opt.db_path).unwrap()); + let db_opt = DatabaseOptions { + main_map_size: opt.main_map_size, + update_map_size: opt.update_map_size + }; + + let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap()); let mut api_keys = ApiKeys { master: opt.master_key, diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 7fa7aa1ca..76b5b2277 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -26,4 +26,12 @@ pub struct Opt { /// Do not send analytics to Meili. #[structopt(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, -} + + /// The maximum size, in bytes, of the main lmdb database directory + #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")] + pub main_map_size: usize, + + /// The maximum size, in bytes, of the update lmdb database directory + #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")] + pub update_map_size: usize +} \ No newline at end of file diff --git a/meilisearch-http/tests/common.rs b/meilisearch-http/tests/common.rs index 8c4e5d147..2b1edcc5c 100644 --- a/meilisearch-http/tests/common.rs +++ b/meilisearch-http/tests/common.rs @@ -18,12 +18,16 @@ impl Server { pub fn with_uid(uid: &str) -> Server { let tmp_dir = TempDir::new("meilisearch").unwrap(); + let default_db_options = meilisearch_core::DatabaseOptions::default(); + let opt = Opt { db_path: tmp_dir.path().to_str().unwrap().to_string(), http_addr: "127.0.0.1:7700".to_owned(), master_key: None, env: "development".to_owned(), no_analytics: true, + main_map_size: default_db_options.main_map_size, + update_map_size: default_db_options.update_map_size }; let data = Data::new(opt.clone());