From 2e3a765dacd53d1a5b9efbc5ca84ee75fa0c50a5 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 29 Apr 2020 21:26:58 +0200 Subject: [PATCH] fix database options --- CHANGELOG.md | 2 +- meilisearch-core/examples/from_file.rs | 6 ++++- meilisearch-core/src/database.rs | 32 +++++++++++--------------- meilisearch-core/src/query_builder.rs | 7 +++++- meilisearch-http/src/option.rs | 6 ++--- meilisearch-http/tests/common.rs | 10 ++++++-- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8168d7977..2dfdce318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## v0.10.2 -- Add support for configuring the lmdb map size (#646) +- Add support for configuring the lmdb map size (#646, #647) ## v0.10.1 diff --git a/meilisearch-core/examples/from_file.rs b/meilisearch-core/examples/from_file.rs index fad6f4e3c..e2e7147cc 100644 --- a/meilisearch-core/examples/from_file.rs +++ b/meilisearch-core/examples/from_file.rs @@ -463,7 +463,11 @@ fn main() -> Result<(), Box> { env_logger::init(); let opt = Command::from_args(); - let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?; + let db_opts = DatabaseOptions { + main_map_size: 100 * 1024 * 1024 * 1024, + update_map_size: 100 * 1024 * 1024 * 1024, + }; + let database = Database::open_or_create(opt.path(), db_opts)?; match opt { Command::Index(command) => index_command(command, database), diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index eab085fc5..ddb795850 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -137,16 +137,7 @@ fn update_awaiter( pub struct DatabaseOptions { pub main_map_size: usize, - pub update_map_size: usize -} - -impl Default for DatabaseOptions { - fn default() -> DatabaseOptions { - DatabaseOptions { - main_map_size: 100 * 1024 * 1024 * 1024, // 100GB - update_map_size: 100 * 1024 * 1024 * 1024 // 100GB - } - } + pub update_map_size: usize, } impl Database { @@ -378,11 +369,16 @@ mod tests { use serde::de::IgnoredAny; use std::sync::mpsc; + const DB_OPTS: DatabaseOptions = DatabaseOptions { + main_map_size: 100 * 1024 * 1024 * 1024, + update_map_size: 100 * 1024 * 1024 * 1024, + }; + #[test] fn valid_updates() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -447,7 +443,7 @@ mod tests { fn invalid_updates() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -509,7 +505,7 @@ mod tests { fn ignored_words_too_long() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -564,7 +560,7 @@ mod tests { fn add_schema_attributes_at_end() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -708,7 +704,7 @@ mod tests { fn deserialize_documents() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -788,7 +784,7 @@ mod tests { fn partial_document_update() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -922,7 +918,7 @@ mod tests { fn delete_index() { let dir = tempfile::tempdir().unwrap(); - let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap()); + let database = Arc::new(Database::open_or_create(dir.path(), DB_OPTS).unwrap()); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); @@ -994,7 +990,7 @@ mod tests { fn check_number_ordering() { let dir = tempfile::tempdir().unwrap(); - let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap(); let db = &database; let (sender, receiver) = mpsc::sync_channel(100); diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs index d95c19713..2944fc109 100644 --- a/meilisearch-core/src/query_builder.rs +++ b/meilisearch-core/src/query_builder.rs @@ -147,6 +147,11 @@ mod tests { use crate::store::Index; use meilisearch_schema::Schema; + const DB_OPTS: DatabaseOptions = DatabaseOptions { + main_map_size: 100 * 1024 * 1024 * 1024, + update_map_size: 100 * 1024 * 1024 * 1024, + }; + fn set_from_stream<'f, I, S>(stream: I) -> Set where I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>, @@ -249,7 +254,7 @@ mod tests { impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase { fn from_iter>(iter: I) -> Self { let tempdir = TempDir::new().unwrap(); - let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap(); + let database = Database::open_or_create(&tempdir, DB_OPTS).unwrap(); let index = database.create_index("default").unwrap(); let db = &database; diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 76b5b2277..2c93f320a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -28,10 +28,10 @@ pub struct Opt { pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")] + #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "107374182400")] // 100GB pub main_map_size: usize, /// The maximum size, in bytes, of the update lmdb database directory - #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")] + #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "107374182400")] // 100GB pub update_map_size: usize -} \ No newline at end of file +} diff --git a/meilisearch-http/tests/common.rs b/meilisearch-http/tests/common.rs index 2b1edcc5c..2d8d64566 100644 --- a/meilisearch-http/tests/common.rs +++ b/meilisearch-http/tests/common.rs @@ -6,6 +6,7 @@ use std::time::Duration; use actix_web::{http::StatusCode, test}; use meilisearch_http::data::Data; use meilisearch_http::option::Opt; +use meilisearch_core::DatabaseOptions; use tempdir::TempDir; use tokio::time::delay_for; @@ -14,11 +15,16 @@ pub struct Server { data: Data, } +const DB_OPTS: DatabaseOptions = DatabaseOptions { + main_map_size: 100 * 1024 * 1024 * 1024, + update_map_size: 100 * 1024 * 1024 * 1024, +}; + impl Server { pub fn with_uid(uid: &str) -> Server { let tmp_dir = TempDir::new("meilisearch").unwrap(); - let default_db_options = meilisearch_core::DatabaseOptions::default(); + let default_db_options = DB_OPTS; let opt = Opt { db_path: tmp_dir.path().to_str().unwrap().to_string(), @@ -34,7 +40,7 @@ impl Server { Server { uid: uid.to_string(), - data: data, + data, } }