From c4aa5cc7d0d49e9d659130e088dfbc4da66ccc95 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 27 Dec 2022 18:17:29 +0100 Subject: [PATCH] Merge --schedule-snapshot and --snapshot-interval-sec options --- config.toml | 8 +- .../src/analytics/segment_analytics.rs | 12 +- meilisearch/src/lib.rs | 5 +- meilisearch/src/option.rs | 125 +++++++++++++++--- meilisearch/tests/snapshot/mod.rs | 4 +- 5 files changed, 123 insertions(+), 31 deletions(-) diff --git a/config.toml b/config.toml index 6f60c8774..99a073faf 100644 --- a/config.toml +++ b/config.toml @@ -73,17 +73,15 @@ ignore_dump_if_db_exists = false ################# schedule_snapshot = false -# Activates scheduled snapshots when provided. +# Enables scheduled snapshots when true, disable when false (the default). +# If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval +# between each snapshot, in seconds. # https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation snapshot_dir = "snapshots/" # Sets the directory where Meilisearch will store snapshots. # https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination -snapshot_interval_sec = 86400 -# Defines the interval between each snapshot. Value must be given in seconds. -# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-interval - # import_snapshot = "./path/to/my/snapshot" # Launches Meilisearch after importing a previously-generated snapshot at the given filepath. # https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index b1fd06dc1..57a494120 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -25,7 +25,7 @@ use uuid::Uuid; use super::{config_user_id_path, DocumentDeletionKind, MEILISEARCH_CONFIG_PATH}; use crate::analytics::Analytics; -use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads}; +use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads, ScheduleSnapshot}; use crate::routes::indexes::documents::UpdateDocumentsQuery; use crate::routes::tasks::TasksFilterQueryRaw; use crate::routes::{create_all_stats, Stats}; @@ -220,9 +220,8 @@ struct Infos { ignore_missing_dump: bool, ignore_dump_if_db_exists: bool, import_snapshot: bool, - schedule_snapshot: bool, + schedule_snapshot: Option, snapshot_dir: bool, - snapshot_interval_sec: u64, ignore_missing_snapshot: bool, ignore_snapshot_if_db_exists: bool, http_addr: bool, @@ -267,7 +266,6 @@ impl From for Infos { ignore_snapshot_if_db_exists, snapshot_dir, schedule_snapshot, - snapshot_interval_sec, import_dump, ignore_missing_dump, ignore_dump_if_db_exists, @@ -280,6 +278,11 @@ impl From for Infos { no_analytics: _, } = options; + let schedule_snapshot = match schedule_snapshot { + ScheduleSnapshot::Disabled => None, + ScheduleSnapshot::Enabled(interval) => Some(interval), + }; + let IndexerOpts { log_every_n: _, max_nb_chunks: _, @@ -299,7 +302,6 @@ impl From for Infos { import_snapshot: import_snapshot.is_some(), schedule_snapshot, snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"), - snapshot_interval_sec, ignore_missing_snapshot, ignore_snapshot_if_db_exists, http_addr: http_addr != default_http_addr(), diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index bb20c1d13..7a7555659 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -41,6 +41,7 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{check_version_file, create_version_file}; use meilisearch_types::{compression, milli, VERSION_FILE_NAME}; pub use option::Opt; +use option::ScheduleSnapshot; use crate::error::MeilisearchHttpError; @@ -169,8 +170,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Auth // We create a loop in a thread that registers snapshotCreation tasks let index_scheduler = Arc::new(index_scheduler); - if opt.schedule_snapshot { - let snapshot_delay = Duration::from_secs(opt.snapshot_interval_sec); + if let ScheduleSnapshot::Enabled(snapshot_delay) = opt.schedule_snapshot { + let snapshot_delay = Duration::from_secs(snapshot_delay); let index_scheduler = index_scheduler.clone(); thread::Builder::new() .name(String::from("register-snapshot-tasks")) diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 01b781984..f66d12af1 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -44,7 +44,6 @@ const MEILI_IGNORE_MISSING_SNAPSHOT: &str = "MEILI_IGNORE_MISSING_SNAPSHOT"; const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS"; const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR"; const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT"; -const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC"; const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP"; const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; @@ -63,6 +62,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB"; const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; +const DEFAULT_SNAPSHOT_INTERVAL_SEC_STR: &str = "86400"; const DEFAULT_DUMP_DIR: &str = "dumps/"; const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; @@ -241,14 +241,11 @@ pub struct Opt { pub snapshot_dir: PathBuf, /// Activates scheduled snapshots when provided. Snapshots are disabled by default. - #[clap(long, env = MEILI_SCHEDULE_SNAPSHOT)] - #[serde(default)] - pub schedule_snapshot: bool, - - /// Defines the interval between each snapshot. Value must be given in seconds. - #[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value_t = default_snapshot_interval_sec())] - #[serde(default = "default_snapshot_interval_sec")] - pub snapshot_interval_sec: u64, + /// + /// When provided with a value, defines the interval between each snapshot, in seconds. + #[clap(long,env = MEILI_SCHEDULE_SNAPSHOT, num_args(0..=1), value_parser=parse_schedule_snapshot, default_value_t, default_missing_value=default_snapshot_interval_sec(), value_name = "SNAPSHOT_INTERVAL_SEC")] + #[serde(default, deserialize_with = "schedule_snapshot_deserialize")] + pub schedule_snapshot: ScheduleSnapshot, /// Imports the dump file located at the specified path. Path must point to a `.dump` file. /// If a database already exists, Meilisearch will throw an error and abort launch. @@ -376,7 +373,6 @@ impl Opt { ssl_tickets, snapshot_dir, schedule_snapshot, - snapshot_interval_sec, dump_dir, log_level, indexer_options, @@ -425,11 +421,10 @@ impl Opt { export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string()); export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string()); export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir); - export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, schedule_snapshot.to_string()); - export_to_env_if_not_present( - MEILI_SNAPSHOT_INTERVAL_SEC, - snapshot_interval_sec.to_string(), - ); + if let Some(snapshot_interval) = schedule_snapshot_to_env(schedule_snapshot) { + export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, snapshot_interval) + } + export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string()); #[cfg(feature = "metrics")] @@ -744,8 +739,8 @@ fn default_snapshot_dir() -> PathBuf { PathBuf::from(DEFAULT_SNAPSHOT_DIR) } -fn default_snapshot_interval_sec() -> u64 { - DEFAULT_SNAPSHOT_INTERVAL_SEC +fn default_snapshot_interval_sec() -> &'static str { + DEFAULT_SNAPSHOT_INTERVAL_SEC_STR } fn default_dump_dir() -> PathBuf { @@ -756,6 +751,102 @@ fn default_log_every_n() -> usize { DEFAULT_LOG_EVERY_N } +/// Indicates if a snapshot was scheduled, and if yes with which interval. +#[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)] +pub enum ScheduleSnapshot { + /// Scheduled snapshots are disabled. + #[default] + Disabled, + /// Snapshots are scheduled at the specified interval, in seconds. + Enabled(u64), +} + +impl Display for ScheduleSnapshot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ScheduleSnapshot::Disabled => write!(f, ""), + ScheduleSnapshot::Enabled(value) => write!(f, "{}", value), + } + } +} + +impl FromStr for ScheduleSnapshot { + type Err = ParseIntError; + + fn from_str(s: &str) -> Result { + Ok(match s { + "" => ScheduleSnapshot::Disabled, + s => ScheduleSnapshot::Enabled(s.parse()?), + }) + } +} + +fn parse_schedule_snapshot(s: &str) -> Result { + Ok(if s.is_empty() { ScheduleSnapshot::Disabled } else { ScheduleSnapshot::from_str(s)? }) +} + +fn schedule_snapshot_to_env(schedule_snapshot: ScheduleSnapshot) -> Option { + match schedule_snapshot { + ScheduleSnapshot::Enabled(snapshot_delay) => Some(snapshot_delay.to_string()), + _ => None, + } +} + +fn schedule_snapshot_deserialize<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + struct BoolOrInt; + + impl<'de> serde::de::Visitor<'de> for BoolOrInt { + type Value = ScheduleSnapshot; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("integer or boolean") + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(if value { + ScheduleSnapshot::Enabled(DEFAULT_SNAPSHOT_INTERVAL_SEC) + } else { + ScheduleSnapshot::Disabled + }) + } + + fn visit_i64(self, v: i64) -> Result + where + E: serde::de::Error, + { + Ok(ScheduleSnapshot::Enabled(v as u64)) + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(ScheduleSnapshot::Enabled(v)) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(ScheduleSnapshot::Disabled) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(ScheduleSnapshot::Disabled) + } + } + deserializer.deserialize_any(BoolOrInt) +} + #[cfg(test)] mod test { diff --git a/meilisearch/tests/snapshot/mod.rs b/meilisearch/tests/snapshot/mod.rs index eba260722..4615f3ea9 100644 --- a/meilisearch/tests/snapshot/mod.rs +++ b/meilisearch/tests/snapshot/mod.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use meilisearch::option::ScheduleSnapshot; use meilisearch::Opt; use tokio::time::sleep; @@ -36,8 +37,7 @@ async fn perform_snapshot() { let options = Opt { snapshot_dir: snapshot_dir.path().to_owned(), - snapshot_interval_sec: 1, - schedule_snapshot: true, + schedule_snapshot: ScheduleSnapshot::Enabled(1), ..default_settings(temp.path()) };