Merge --schedule-snapshot and --snapshot-interval-sec options

This commit is contained in:
Louis Dureuil 2022-12-27 18:17:29 +01:00
parent 12c3d432f9
commit c4aa5cc7d0
No known key found for this signature in database
5 changed files with 123 additions and 31 deletions

View File

@ -73,17 +73,15 @@ ignore_dump_if_db_exists = false
################# #################
schedule_snapshot = false schedule_snapshot = false
# Activates scheduled snapshots when provided. # Enables scheduled snapshots when true, disable when false (the default).
# If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
# between each snapshot, in seconds.
# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation # https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
snapshot_dir = "snapshots/" snapshot_dir = "snapshots/"
# Sets the directory where Meilisearch will store snapshots. # Sets the directory where Meilisearch will store snapshots.
# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination # https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
snapshot_interval_sec = 86400
# Defines the interval between each snapshot. Value must be given in seconds.
# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-interval
# import_snapshot = "./path/to/my/snapshot" # import_snapshot = "./path/to/my/snapshot"
# Launches Meilisearch after importing a previously-generated snapshot at the given filepath. # Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot # https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot

View File

@ -25,7 +25,7 @@ use uuid::Uuid;
use super::{config_user_id_path, DocumentDeletionKind, MEILISEARCH_CONFIG_PATH}; use super::{config_user_id_path, DocumentDeletionKind, MEILISEARCH_CONFIG_PATH};
use crate::analytics::Analytics; use crate::analytics::Analytics;
use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads}; use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads, ScheduleSnapshot};
use crate::routes::indexes::documents::UpdateDocumentsQuery; use crate::routes::indexes::documents::UpdateDocumentsQuery;
use crate::routes::tasks::TasksFilterQueryRaw; use crate::routes::tasks::TasksFilterQueryRaw;
use crate::routes::{create_all_stats, Stats}; use crate::routes::{create_all_stats, Stats};
@ -220,9 +220,8 @@ struct Infos {
ignore_missing_dump: bool, ignore_missing_dump: bool,
ignore_dump_if_db_exists: bool, ignore_dump_if_db_exists: bool,
import_snapshot: bool, import_snapshot: bool,
schedule_snapshot: bool, schedule_snapshot: Option<u64>,
snapshot_dir: bool, snapshot_dir: bool,
snapshot_interval_sec: u64,
ignore_missing_snapshot: bool, ignore_missing_snapshot: bool,
ignore_snapshot_if_db_exists: bool, ignore_snapshot_if_db_exists: bool,
http_addr: bool, http_addr: bool,
@ -267,7 +266,6 @@ impl From<Opt> for Infos {
ignore_snapshot_if_db_exists, ignore_snapshot_if_db_exists,
snapshot_dir, snapshot_dir,
schedule_snapshot, schedule_snapshot,
snapshot_interval_sec,
import_dump, import_dump,
ignore_missing_dump, ignore_missing_dump,
ignore_dump_if_db_exists, ignore_dump_if_db_exists,
@ -280,6 +278,11 @@ impl From<Opt> for Infos {
no_analytics: _, no_analytics: _,
} = options; } = options;
let schedule_snapshot = match schedule_snapshot {
ScheduleSnapshot::Disabled => None,
ScheduleSnapshot::Enabled(interval) => Some(interval),
};
let IndexerOpts { let IndexerOpts {
log_every_n: _, log_every_n: _,
max_nb_chunks: _, max_nb_chunks: _,
@ -299,7 +302,6 @@ impl From<Opt> for Infos {
import_snapshot: import_snapshot.is_some(), import_snapshot: import_snapshot.is_some(),
schedule_snapshot, schedule_snapshot,
snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"), snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"),
snapshot_interval_sec,
ignore_missing_snapshot, ignore_missing_snapshot,
ignore_snapshot_if_db_exists, ignore_snapshot_if_db_exists,
http_addr: http_addr != default_http_addr(), http_addr: http_addr != default_http_addr(),

View File

@ -41,6 +41,7 @@ use meilisearch_types::tasks::KindWithContent;
use meilisearch_types::versioning::{check_version_file, create_version_file}; use meilisearch_types::versioning::{check_version_file, create_version_file};
use meilisearch_types::{compression, milli, VERSION_FILE_NAME}; use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
pub use option::Opt; pub use option::Opt;
use option::ScheduleSnapshot;
use crate::error::MeilisearchHttpError; use crate::error::MeilisearchHttpError;
@ -169,8 +170,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Auth
// We create a loop in a thread that registers snapshotCreation tasks // We create a loop in a thread that registers snapshotCreation tasks
let index_scheduler = Arc::new(index_scheduler); let index_scheduler = Arc::new(index_scheduler);
if opt.schedule_snapshot { if let ScheduleSnapshot::Enabled(snapshot_delay) = opt.schedule_snapshot {
let snapshot_delay = Duration::from_secs(opt.snapshot_interval_sec); let snapshot_delay = Duration::from_secs(snapshot_delay);
let index_scheduler = index_scheduler.clone(); let index_scheduler = index_scheduler.clone();
thread::Builder::new() thread::Builder::new()
.name(String::from("register-snapshot-tasks")) .name(String::from("register-snapshot-tasks"))

View File

@ -44,7 +44,6 @@ const MEILI_IGNORE_MISSING_SNAPSHOT: &str = "MEILI_IGNORE_MISSING_SNAPSHOT";
const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS"; const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS";
const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR"; const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR";
const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT"; const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT";
const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC";
const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP"; const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP";
const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
@ -63,6 +62,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB";
const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
const DEFAULT_SNAPSHOT_INTERVAL_SEC_STR: &str = "86400";
const DEFAULT_DUMP_DIR: &str = "dumps/"; const DEFAULT_DUMP_DIR: &str = "dumps/";
const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
@ -241,14 +241,11 @@ pub struct Opt {
pub snapshot_dir: PathBuf, pub snapshot_dir: PathBuf,
/// Activates scheduled snapshots when provided. Snapshots are disabled by default. /// Activates scheduled snapshots when provided. Snapshots are disabled by default.
#[clap(long, env = MEILI_SCHEDULE_SNAPSHOT)] ///
#[serde(default)] /// When provided with a value, defines the interval between each snapshot, in seconds.
pub schedule_snapshot: bool, #[clap(long,env = MEILI_SCHEDULE_SNAPSHOT, num_args(0..=1), value_parser=parse_schedule_snapshot, default_value_t, default_missing_value=default_snapshot_interval_sec(), value_name = "SNAPSHOT_INTERVAL_SEC")]
#[serde(default, deserialize_with = "schedule_snapshot_deserialize")]
/// Defines the interval between each snapshot. Value must be given in seconds. pub schedule_snapshot: ScheduleSnapshot,
#[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value_t = default_snapshot_interval_sec())]
#[serde(default = "default_snapshot_interval_sec")]
pub snapshot_interval_sec: u64,
/// Imports the dump file located at the specified path. Path must point to a `.dump` file. /// Imports the dump file located at the specified path. Path must point to a `.dump` file.
/// If a database already exists, Meilisearch will throw an error and abort launch. /// If a database already exists, Meilisearch will throw an error and abort launch.
@ -376,7 +373,6 @@ impl Opt {
ssl_tickets, ssl_tickets,
snapshot_dir, snapshot_dir,
schedule_snapshot, schedule_snapshot,
snapshot_interval_sec,
dump_dir, dump_dir,
log_level, log_level,
indexer_options, indexer_options,
@ -425,11 +421,10 @@ impl Opt {
export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string()); export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string());
export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string()); export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string());
export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir); export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir);
export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, schedule_snapshot.to_string()); if let Some(snapshot_interval) = schedule_snapshot_to_env(schedule_snapshot) {
export_to_env_if_not_present( export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, snapshot_interval)
MEILI_SNAPSHOT_INTERVAL_SEC, }
snapshot_interval_sec.to_string(),
);
export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string()); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string());
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]
@ -744,8 +739,8 @@ fn default_snapshot_dir() -> PathBuf {
PathBuf::from(DEFAULT_SNAPSHOT_DIR) PathBuf::from(DEFAULT_SNAPSHOT_DIR)
} }
fn default_snapshot_interval_sec() -> u64 { fn default_snapshot_interval_sec() -> &'static str {
DEFAULT_SNAPSHOT_INTERVAL_SEC DEFAULT_SNAPSHOT_INTERVAL_SEC_STR
} }
fn default_dump_dir() -> PathBuf { fn default_dump_dir() -> PathBuf {
@ -756,6 +751,102 @@ fn default_log_every_n() -> usize {
DEFAULT_LOG_EVERY_N DEFAULT_LOG_EVERY_N
} }
/// Indicates if a snapshot was scheduled, and if yes with which interval.
#[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)]
pub enum ScheduleSnapshot {
/// Scheduled snapshots are disabled.
#[default]
Disabled,
/// Snapshots are scheduled at the specified interval, in seconds.
Enabled(u64),
}
impl Display for ScheduleSnapshot {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ScheduleSnapshot::Disabled => write!(f, ""),
ScheduleSnapshot::Enabled(value) => write!(f, "{}", value),
}
}
}
impl FromStr for ScheduleSnapshot {
type Err = ParseIntError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s {
"" => ScheduleSnapshot::Disabled,
s => ScheduleSnapshot::Enabled(s.parse()?),
})
}
}
fn parse_schedule_snapshot(s: &str) -> Result<ScheduleSnapshot, ParseIntError> {
Ok(if s.is_empty() { ScheduleSnapshot::Disabled } else { ScheduleSnapshot::from_str(s)? })
}
fn schedule_snapshot_to_env(schedule_snapshot: ScheduleSnapshot) -> Option<String> {
match schedule_snapshot {
ScheduleSnapshot::Enabled(snapshot_delay) => Some(snapshot_delay.to_string()),
_ => None,
}
}
fn schedule_snapshot_deserialize<'de, D>(deserializer: D) -> Result<ScheduleSnapshot, D::Error>
where
D: serde::Deserializer<'de>,
{
struct BoolOrInt;
impl<'de> serde::de::Visitor<'de> for BoolOrInt {
type Value = ScheduleSnapshot;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("integer or boolean")
}
fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(if value {
ScheduleSnapshot::Enabled(DEFAULT_SNAPSHOT_INTERVAL_SEC)
} else {
ScheduleSnapshot::Disabled
})
}
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(ScheduleSnapshot::Enabled(v as u64))
}
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(ScheduleSnapshot::Enabled(v))
}
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(ScheduleSnapshot::Disabled)
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(ScheduleSnapshot::Disabled)
}
}
deserializer.deserialize_any(BoolOrInt)
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {

View File

@ -1,5 +1,6 @@
use std::time::Duration; use std::time::Duration;
use meilisearch::option::ScheduleSnapshot;
use meilisearch::Opt; use meilisearch::Opt;
use tokio::time::sleep; use tokio::time::sleep;
@ -36,8 +37,7 @@ async fn perform_snapshot() {
let options = Opt { let options = Opt {
snapshot_dir: snapshot_dir.path().to_owned(), snapshot_dir: snapshot_dir.path().to_owned(),
snapshot_interval_sec: 1, schedule_snapshot: ScheduleSnapshot::Enabled(1),
schedule_snapshot: true,
..default_settings(temp.path()) ..default_settings(temp.path())
}; };