Merge pull request #647 from MarinPostma/master

fix database options
This commit is contained in:
Clément Renault 2020-04-29 23:00:34 +02:00 committed by GitHub
commit 674476155a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 37 additions and 26 deletions

View File

@ -1,6 +1,6 @@
## v0.10.2 ## v0.10.2
- Add support for configuring the lmdb map size (#646) - Add support for configuring the lmdb map size (#646, #647)
## v0.10.1 ## v0.10.1

View File

@ -463,7 +463,11 @@ fn main() -> Result<(), Box<dyn Error>> {
env_logger::init(); env_logger::init();
let opt = Command::from_args(); let opt = Command::from_args();
let database = Database::open_or_create(opt.path(), DatabaseOptions::default())?; let db_opts = DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024,
update_map_size: 100 * 1024 * 1024 * 1024,
};
let database = Database::open_or_create(opt.path(), db_opts)?;
match opt { match opt {
Command::Index(command) => index_command(command, database), Command::Index(command) => index_command(command, database),

View File

@ -137,16 +137,7 @@ fn update_awaiter(
pub struct DatabaseOptions { pub struct DatabaseOptions {
pub main_map_size: usize, pub main_map_size: usize,
pub update_map_size: usize pub update_map_size: usize,
}
impl Default for DatabaseOptions {
fn default() -> DatabaseOptions {
DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024, // 100GB
update_map_size: 100 * 1024 * 1024 * 1024 // 100GB
}
}
} }
impl Database { impl Database {
@ -378,11 +369,16 @@ mod tests {
use serde::de::IgnoredAny; use serde::de::IgnoredAny;
use std::sync::mpsc; use std::sync::mpsc;
const DB_OPTS: DatabaseOptions = DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024,
update_map_size: 100 * 1024 * 1024 * 1024,
};
#[test] #[test]
fn valid_updates() { fn valid_updates() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -447,7 +443,7 @@ mod tests {
fn invalid_updates() { fn invalid_updates() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -509,7 +505,7 @@ mod tests {
fn ignored_words_too_long() { fn ignored_words_too_long() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -564,7 +560,7 @@ mod tests {
fn add_schema_attributes_at_end() { fn add_schema_attributes_at_end() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -708,7 +704,7 @@ mod tests {
fn deserialize_documents() { fn deserialize_documents() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -788,7 +784,7 @@ mod tests {
fn partial_document_update() { fn partial_document_update() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -922,7 +918,7 @@ mod tests {
fn delete_index() { fn delete_index() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Arc::new(Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap()); let database = Arc::new(Database::open_or_create(dir.path(), DB_OPTS).unwrap());
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);
@ -994,7 +990,7 @@ mod tests {
fn check_number_ordering() { fn check_number_ordering() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let database = Database::open_or_create(dir.path(), DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(dir.path(), DB_OPTS).unwrap();
let db = &database; let db = &database;
let (sender, receiver) = mpsc::sync_channel(100); let (sender, receiver) = mpsc::sync_channel(100);

View File

@ -147,6 +147,11 @@ mod tests {
use crate::store::Index; use crate::store::Index;
use meilisearch_schema::Schema; use meilisearch_schema::Schema;
const DB_OPTS: DatabaseOptions = DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024,
update_map_size: 100 * 1024 * 1024 * 1024,
};
fn set_from_stream<'f, I, S>(stream: I) -> Set fn set_from_stream<'f, I, S>(stream: I) -> Set
where where
I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>, I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>,
@ -249,7 +254,7 @@ mod tests {
impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase { impl<'a> FromIterator<(&'a str, &'a [DocIndex])> for TempDatabase {
fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self { fn from_iter<I: IntoIterator<Item = (&'a str, &'a [DocIndex])>>(iter: I) -> Self {
let tempdir = TempDir::new().unwrap(); let tempdir = TempDir::new().unwrap();
let database = Database::open_or_create(&tempdir, DatabaseOptions::default()).unwrap(); let database = Database::open_or_create(&tempdir, DB_OPTS).unwrap();
let index = database.create_index("default").unwrap(); let index = database.create_index("default").unwrap();
let db = &database; let db = &database;

View File

@ -28,10 +28,10 @@ pub struct Opt {
pub no_analytics: bool, pub no_analytics: bool,
/// The maximum size, in bytes, of the main lmdb database directory /// The maximum size, in bytes, of the main lmdb database directory
#[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().main_map_size")] #[structopt(long, env = "MEILI_MAIN_MAP_SIZE", default_value = "107374182400")] // 100GB
pub main_map_size: usize, pub main_map_size: usize,
/// The maximum size, in bytes, of the update lmdb database directory /// The maximum size, in bytes, of the update lmdb database directory
#[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "meilisearch_core::DatabaseOptions::default().update_map_size")] #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "107374182400")] // 100GB
pub update_map_size: usize pub update_map_size: usize
} }

View File

@ -6,6 +6,7 @@ use std::time::Duration;
use actix_web::{http::StatusCode, test}; use actix_web::{http::StatusCode, test};
use meilisearch_http::data::Data; use meilisearch_http::data::Data;
use meilisearch_http::option::Opt; use meilisearch_http::option::Opt;
use meilisearch_core::DatabaseOptions;
use tempdir::TempDir; use tempdir::TempDir;
use tokio::time::delay_for; use tokio::time::delay_for;
@ -14,11 +15,16 @@ pub struct Server {
data: Data, data: Data,
} }
const DB_OPTS: DatabaseOptions = DatabaseOptions {
main_map_size: 100 * 1024 * 1024 * 1024,
update_map_size: 100 * 1024 * 1024 * 1024,
};
impl Server { impl Server {
pub fn with_uid(uid: &str) -> Server { pub fn with_uid(uid: &str) -> Server {
let tmp_dir = TempDir::new("meilisearch").unwrap(); let tmp_dir = TempDir::new("meilisearch").unwrap();
let default_db_options = meilisearch_core::DatabaseOptions::default(); let default_db_options = DB_OPTS;
let opt = Opt { let opt = Opt {
db_path: tmp_dir.path().to_str().unwrap().to_string(), db_path: tmp_dir.path().to_str().unwrap().to_string(),
@ -34,7 +40,7 @@ impl Server {
Server { Server {
uid: uid.to_string(), uid: uid.to_string(),
data: data, data,
} }
} }