Add support for configuring lmdb map size

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.10.2
+
+- Add support for configuring the lmdb map size (#646)
+
 ## v0.10.1
 
   - Add support for floating points in filters (#640)

meilisearch-core/src/database.rs

@@ -135,20 +135,34 @@ fn update_awaiter(
     Ok(())
 }
 
+pub struct DatabaseOptions {
+    pub main_map_size: usize,
+    pub update_map_size: usize
+}
+
+impl Default for DatabaseOptions {
+    fn default() -> DatabaseOptions {
+        DatabaseOptions {
+            main_map_size: 100 * 1024 * 1024 * 1024, // 100GB
+            update_map_size: 100 * 1024 * 1024 * 1024 // 100GB
+        }
+    }
+}
+
 impl Database {
-    pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
+    pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
         let main_path = path.as_ref().join("main");
         let update_path = path.as_ref().join("update");
 
         fs::create_dir_all(&main_path)?;
         let env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.main_map_size)
             .max_dbs(3000)
             .open(main_path)?;
 
         fs::create_dir_all(&update_path)?;
         let update_env = heed::EnvOpenOptions::new()
-            .map_size(100 * 1024 * 1024 * 1024) // 100GB
+            .map_size(options.update_map_size)
             .max_dbs(3000)
             .open(update_path)?;
 

meilisearch-core/src/lib.rs

@@ -25,7 +25,7 @@ pub mod raw_indexer;
 pub mod serde;
 pub mod store;
 
-pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
+pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
 pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
 pub use self::filters::Filter;
 pub use self::number::{Number, ParseNumberError};

meilisearch-http/src/data.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use heed::types::{SerdeBincode, Str};
 use log::error;
-use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT};
+use meilisearch_core::{Database, DatabaseOptions, Error as MError, MResult, MainT, UpdateT};
 use sha2::Digest;
 use sysinfo::Pid;
 
@@ -132,7 +132,12 @@ impl Data {
         let db_path = opt.db_path.clone();
         let server_pid = sysinfo::get_current_pid().unwrap();
 
-        let db = Arc::new(Database::open_or_create(opt.db_path).unwrap());
+        let db_opt = DatabaseOptions {
+            main_map_size: opt.main_map_size,
+            update_map_size: opt.update_map_size
+        };
+
+        let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap());
 
         let mut api_keys = ApiKeys {
             master: opt.master_key,

meilisearch-http/src/option.rs

@@ -26,4 +26,12 @@ pub struct Opt {
     /// Do not send analytics to Meili.
     #[structopt(long, env = "MEILI_NO_ANALYTICS")]
     pub no_analytics: bool,
-}
+
+    /// The maximum size, in bytes, of the main lmdb database directory
+    #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")]
+    pub main_map_size: usize,
+
+    /// The maximum size, in bytes, of the update lmdb database directory
+    #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")]
+    pub update_map_size: usize
+}