Add support for configuring lmdb map size

This commit is contained in:
Jan Emmens 2020-04-29 00:40:06 +02:00
parent 899559a060
commit c4287cdfac
5 changed files with 38 additions and 7 deletions

View File

@ -1,3 +1,7 @@
## v0.10.2
- Add support for configuring the lmdb map size (#646)
## v0.10.1
- Add support for floating points in filters (#640)

View File

@ -135,20 +135,34 @@ fn update_awaiter(
Ok(())
}
pub struct DatabaseOptions {
pub main_map_size: usize,
pub update_map_size: usize
}
impl Default for DatabaseOptions {
fn default() -> DatabaseOptions {
DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024, // 100GB
update_map_size: 100 * 1024 * 1024 * 1024 // 100GB
}
}
}
impl Database {
pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> {
pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
let main_path = path.as_ref().join("main");
let update_path = path.as_ref().join("update");
fs::create_dir_all(&main_path)?;
let env = heed::EnvOpenOptions::new()
.map_size(100 * 1024 * 1024 * 1024) // 100GB
.map_size(options.main_map_size)
.max_dbs(3000)
.open(main_path)?;
fs::create_dir_all(&update_path)?;
let update_env = heed::EnvOpenOptions::new()
.map_size(100 * 1024 * 1024 * 1024) // 100GB
.map_size(options.update_map_size)
.max_dbs(3000)
.open(update_path)?;

View File

@ -25,7 +25,7 @@ pub mod raw_indexer;
pub mod serde;
pub mod store;
pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT};
pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
pub use self::filters::Filter;
pub use self::number::{Number, ParseNumberError};

View File

@ -5,7 +5,7 @@ use std::sync::Arc;
use chrono::{DateTime, Utc};
use heed::types::{SerdeBincode, Str};
use log::error;
use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT};
use meilisearch_core::{Database, DatabaseOptions, Error as MError, MResult, MainT, UpdateT};
use sha2::Digest;
use sysinfo::Pid;
@ -132,7 +132,12 @@ impl Data {
let db_path = opt.db_path.clone();
let server_pid = sysinfo::get_current_pid().unwrap();
let db = Arc::new(Database::open_or_create(opt.db_path).unwrap());
let db_opt = DatabaseOptions {
main_map_size: opt.main_map_size,
update_map_size: opt.update_map_size
};
let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap());
let mut api_keys = ApiKeys {
master: opt.master_key,

View File

@ -26,4 +26,12 @@ pub struct Opt {
/// Do not send analytics to Meili.
#[structopt(long, env = "MEILI_NO_ANALYTICS")]
pub no_analytics: bool,
/// The maximum size, in bytes, of the main lmdb database directory
#[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")]
pub main_map_size: usize,
/// The maximum size, in bytes, of the update lmdb database directory
#[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")]
pub update_map_size: usize
}