Merge pull request #646 from Wazner/configurable-map-size

Add support for configuring lmdb map size

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.10.2
+
+- Add support for configuring the lmdb map size (#646)
+
 ## v0.10.1
 
   - Add support for floating points in filters (#640)

meilisearch-core/examples/from_file.rs

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
-use meilisearch_core::{Database, Highlight, ProcessedUpdateResult};
+use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult};
 use meilisearch_core::settings::Settings;
 use meilisearch_schema::FieldId;
 
@@ -463,7 +463,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init();
 
     let opt = Command::from_args();
-    let database = Database::open_or_create(opt.path())?;
+    let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?;
 
     match opt {
         Command::Index(command) => index_command(command, database),

meilisearch-core/src/database.rs

@@ -135,20 +135,34 @@ fn update_awaiter(
     Ok(())
 }
 
+pub struct DatabaseOptions {
+    pub main_map_size: usize,
+    pub update_map_size: usize
+}
+
+impl Default for DatabaseOptions {
+    fn default() -> DatabaseOptions {
+        DatabaseOptions {
+            main_map_size: 100 * 1024 * 1024 * 1024, // 100GB
+            update_map_size: 100 * 1024 * 1024 * 1024 // 100GB
+        }
+    }
+}
+
 impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
+    pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
         let main_path = path.as_ref().join("main");
         let update_path = path.as_ref().join("update");
 
         fs::create_dir_all(&main_path)?;
         let env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.main_map_size)
             .max_dbs(3000)
             .open(main_path)?;
 
         fs::create_dir_all(&update_path)?;
         let update_env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.update_map_size)
             .max_dbs(3000)
             .open(update_path)?;
 
@@ -368,7 +382,7 @@ mod tests {
     fn valid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -433,7 +447,7 @@ mod tests {
     fn invalid_updates() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -495,7 +509,7 @@ mod tests {
     fn ignored_words_too_long() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -550,7 +564,7 @@ mod tests {
     fn add_schema_attributes_at_end() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -694,7 +708,7 @@ mod tests {
     fn deserialize_documents() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -774,7 +788,7 @@ mod tests {
     fn partial_document_update() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -908,7 +922,7 @@ mod tests {
     fn delete_index() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Arc::new(Database::open_or_create(dir.path()).unwrap());
+        let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap());
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);
@@ -980,7 +994,7 @@ mod tests {
     fn check_number_ordering() {
         let dir = tempfile::tempdir().unwrap();
 
-        let database = Database::open_or_create(dir.path()).unwrap();
+        let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
         let db = &database;
 
         let (sender, receiver) = mpsc::sync_channel(100);

meilisearch-core/src/lib.rs

@@ -25,7 +25,7 @@ pub mod raw_indexer;
 pub mod serde;
 pub mod store;
 
-pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
+pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
 pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
 pub use self::filters::Filter;
 pub use self::number::{Number, ParseNumberError};

meilisearch-core/src/query_builder.rs

@@ -143,7 +143,7 @@ mod tests {
     use crate::DocIndex;
     use crate::automaton::normalize_str;
     use crate::bucket_sort::SimpleMatch;
-    use crate::database::Database;
+    use crate::database::{Database,DatabaseOptions};
     use crate::store::Index;
     use meilisearch_schema::Schema;
 
@@ -249,7 +249,7 @@ mod tests {
     impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase {
         fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self {
             let tempdir = TempDir::new().unwrap();
-            let database = Database::open_or_create(&tempdir).unwrap();
+            let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap();
             let index = database.create_index("default").unwrap();
 
             let db = &database;

meilisearch-http/src/data.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use heed::types::{SerdeBincode, Str};
 use log::error;
-use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT};
+use meilisearch_core::{Database, DatabaseOptions, Error as MError, MResult, MainT, UpdateT};
 use sha2::Digest;
 use sysinfo::Pid;
 
@@ -132,7 +132,12 @@ impl Data {
         let db_path = opt.db_path.clone();
         let server_pid = sysinfo::get_current_pid().unwrap();
 
-        let db = Arc::new(Database::open_or_create(opt.db_path).unwrap());
+        let db_opt = DatabaseOptions {
+            main_map_size: opt.main_map_size,
+            update_map_size: opt.update_map_size
+        };
+
+        let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap());
 
         let mut api_keys = ApiKeys {
             master: opt.master_key,

meilisearch-http/src/option.rs

@@ -26,4 +26,12 @@ pub struct Opt {
     /// Do not send analytics to Meili.
     #[structopt(long, env = "MEILI_NO_ANALYTICS")]
     pub no_analytics: bool,
+
+    /// The maximum size, in bytes, of the main lmdb database directory
+    #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")]
+    pub main_map_size: usize,
+
+    /// The maximum size, in bytes, of the update lmdb database directory
+    #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")]
+    pub update_map_size: usize
 }

meilisearch-http/tests/common.rs

@@ -18,12 +18,16 @@ impl Server {
     pub fn with_uid(uid: &str) -> Server {
         let tmp_dir = TempDir::new("meilisearch").unwrap();
 
+        let default_db_options = meilisearch_core::DatabaseOptions::default();
+
         let opt = Opt {
             db_path: tmp_dir.path().to_str().unwrap().to_string(),
             http_addr: "127.0.0.1:7700".to_owned(),
             master_key: None,
             env: "development".to_owned(),
             no_analytics: true,
+            main_map_size: default_db_options.main_map_size,
+            update_map_size: default_db_options.update_map_size
         };
 
         let data = Data::new(opt.clone());