Merge pull request #646 from Wazner/configurable-map-size

Add support for configuring lmdb map size
This commit is contained in:
Clément Renault 2020-04-29 14:32:03 +02:00 committed by GitHub
commit 382e300326
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 54 additions and 19 deletions

View File

@ -1,3 +1,7 @@
## v0.10.2
- Add support for configuring the lmdb map size (#646)
## v0.10.1 ## v0.10.1
- Add support for floating points in filters (#640) - Add support for floating points in filters (#640)

View File

@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
use structopt::StructOpt; use structopt::StructOpt;
use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
use meilisearch_core::{Database, Highlight, ProcessedUpdateResult}; use meilisearch_core::{Database, DatabaseOptions, Highlight, ProcessedUpdateResult};
use meilisearch_core::settings::Settings; use meilisearch_core::settings::Settings;
use meilisearch_schema::FieldId; use meilisearch_schema::FieldId;
@ -463,7 +463,7 @@ fn main() -> Result<(), Box<dyn Error>> {
env_logger::init(); env_logger::init();
let opt = Command::from_args(); let opt = Command::from_args();
let database = Database::open_or_create(opt.path())?; let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?;
match opt { match opt {
Command::Index(command) => index_command(command, database), Command::Index(command) => index_command(command, database),

View File

@ -135,20 +135,34 @@ fn update_awaiter(
Ok(()) Ok(())
} }
pub struct DatabaseOptions {
pub main_map_size: usize,
pub update_map_size: usize
}
impl Default for DatabaseOptions {
fn default() -> DatabaseOptions {
DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024, // 100GB
update_map_size: 100 * 1024 * 1024 * 1024 // 100GB
}
}
}
impl Database { impl Database {
pub fn open_or_create(path: impl AsRef<Path>) -> MResult<Database> { pub fn open_or_create(path: impl AsRef<Path>, options: DatabaseOptions) -> MResult<Database> {
let main_path = path.as_ref().join("main"); let main_path = path.as_ref().join("main");
let update_path = path.as_ref().join("update"); let update_path = path.as_ref().join("update");
fs::create_dir_all(&main_path)?; fs::create_dir_all(&main_path)?;
let env = heed::EnvOpenOptions::new() let env = heed::EnvOpenOptions::new()
.map_size(100 * 1024 * 1024 * 1024) // 100GB .map_size(options.main_map_size)
.max_dbs(3000) .max_dbs(3000)
.open(main_path)?; .open(main_path)?;
fs::create_dir_all(&update_path)?; fs::create_dir_all(&update_path)?;
let update_env = heed::EnvOpenOptions::new() let update_env = heed::EnvOpenOptions::new()
.map_size(100 * 1024 * 1024 * 1024) // 100GB .map_size(options.update_map_size)
.max_dbs(3000) .max_dbs(3000)
.open(update_path)?; .open(update_path)?;
@ -368,7 +382,7 @@ mod tests {
fn valid_updates() { fn valid_updates() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -433,7 +447,7 @@ mod tests {
fn invalid_updates() { fn invalid_updates() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -495,7 +509,7 @@ mod tests {
fn ignored_words_too_long() { fn ignored_words_too_long() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -550,7 +564,7 @@ mod tests {
fn add_schema_attributes_at_end() { fn add_schema_attributes_at_end() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -694,7 +708,7 @@ mod tests {
fn deserialize_documents() { fn deserialize_documents() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -774,7 +788,7 @@ mod tests {
fn partial_document_update() { fn partial_document_update() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -908,7 +922,7 @@ mod tests {
fn delete_index() { fn delete_index() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Arc::new(Database::open_or_create(dir.path()).unwrap()); let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap());
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -980,7 +994,7 @@ mod tests {
fn check_number_ordering() { fn check_number_ordering() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path()).unwrap(); let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);

View File

@ -25,7 +25,7 @@ pub mod raw_indexer;
pub mod serde; pub mod serde;
pub mod store; pub mod store;
pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT}; pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
pub use self::error::{Error, HeedError, FstError, MResult, pest_error}; pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
pub use self::filters::Filter; pub use self::filters::Filter;
pub use self::number::{Number, ParseNumberError}; pub use self::number::{Number, ParseNumberError};

View File

@ -143,7 +143,7 @@ mod tests {
use crate::DocIndex; use crate::DocIndex;
use crate::automaton::normalize_str; use crate::automaton::normalize_str;
use crate::bucket_sort::SimpleMatch; use crate::bucket_sort::SimpleMatch;
use crate::database::Database; use crate::database::{Database,DatabaseOptions};
use crate::store::Index; use crate::store::Index;
use meilisearch_schema::Schema; use meilisearch_schema::Schema;
@ -249,7 +249,7 @@ mod tests {
impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase { impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase {
fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self { fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self {
let tempdir = TempDir::new().unwrap(); let tempdir = TempDir::new().unwrap();
let database = Database::open_or_create(&tempdir).unwrap(); let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap();
let index = database.create_index("default").unwrap(); let index = database.create_index("default").unwrap();
let db = &database; let db = &database;

View File

@ -5,7 +5,7 @@ use std::sync::Arc;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use heed::types::{SerdeBincode, Str}; use heed::types::{SerdeBincode, Str};
use log::error; use log::error;
use meilisearch_core::{Database, Error as MError, MResult, MainT, UpdateT}; use meilisearch_core::{Database, DatabaseOptions, Error as MError, MResult, MainT, UpdateT};
use sha2::Digest; use sha2::Digest;
use sysinfo::Pid; use sysinfo::Pid;
@ -132,7 +132,12 @@ impl Data {
let db_path = opt.db_path.clone(); let db_path = opt.db_path.clone();
let server_pid = sysinfo::get_current_pid().unwrap(); let server_pid = sysinfo::get_current_pid().unwrap();
let db = Arc::new(Database::open_or_create(opt.db_path).unwrap()); let db_opt = DatabaseOptions {
main_map_size: opt.main_map_size,
update_map_size: opt.update_map_size
};
let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap());
let mut api_keys = ApiKeys { let mut api_keys = ApiKeys {
master: opt.master_key, master: opt.master_key,

View File

@ -26,4 +26,12 @@ pub struct Opt {
/// Do not send analytics to Meili. /// Do not send analytics to Meili.
#[structopt(long, env = "MEILI_NO_ANALYTICS")] #[structopt(long, env = "MEILI_NO_ANALYTICS")]
pub no_analytics: bool, pub no_analytics: bool,
/// The maximum size, in bytes, of the main lmdb database directory
#[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")]
pub main_map_size: usize,
/// The maximum size, in bytes, of the update lmdb database directory
#[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")]
pub update_map_size: usize
} }

View File

@ -18,12 +18,16 @@ impl Server {
pub fn with_uid(uid: &str) -> Server { pub fn with_uid(uid: &str) -> Server {
let tmp_dir = TempDir::new("meilisearch").unwrap(); let tmp_dir = TempDir::new("meilisearch").unwrap();
let default_db_options = meilisearch_core::DatabaseOptions::default();
let opt = Opt { let opt = Opt {
db_path: tmp_dir.path().to_str().unwrap().to_string(), db_path: tmp_dir.path().to_str().unwrap().to_string(),
http_addr: "127.0.0.1:7700".to_owned(), http_addr: "127.0.0.1:7700".to_owned(),
master_key: None, master_key: None,
env: "development".to_owned(), env: "development".to_owned(),
no_analytics: true, no_analytics: true,
main_map_size: default_db_options.main_map_size,
update_map_size: default_db_options.update_map_size
}; };
let data = Data::new(opt.clone()); let data = Data::new(opt.clone());