mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-29 16:24:26 +01:00
Merge --schedule-snapshot and --snapshot-interval-sec options
This commit is contained in:
parent
12c3d432f9
commit
c4aa5cc7d0
@ -73,17 +73,15 @@ ignore_dump_if_db_exists = false
|
|||||||
#################
|
#################
|
||||||
|
|
||||||
schedule_snapshot = false
|
schedule_snapshot = false
|
||||||
# Activates scheduled snapshots when provided.
|
# Enables scheduled snapshots when true, disable when false (the default).
|
||||||
|
# If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
|
||||||
|
# between each snapshot, in seconds.
|
||||||
# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
|
# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
|
||||||
|
|
||||||
snapshot_dir = "snapshots/"
|
snapshot_dir = "snapshots/"
|
||||||
# Sets the directory where Meilisearch will store snapshots.
|
# Sets the directory where Meilisearch will store snapshots.
|
||||||
# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
|
# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
|
||||||
|
|
||||||
snapshot_interval_sec = 86400
|
|
||||||
# Defines the interval between each snapshot. Value must be given in seconds.
|
|
||||||
# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-interval
|
|
||||||
|
|
||||||
# import_snapshot = "./path/to/my/snapshot"
|
# import_snapshot = "./path/to/my/snapshot"
|
||||||
# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
|
# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
|
||||||
# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot
|
# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot
|
||||||
|
@ -25,7 +25,7 @@ use uuid::Uuid;
|
|||||||
|
|
||||||
use super::{config_user_id_path, DocumentDeletionKind, MEILISEARCH_CONFIG_PATH};
|
use super::{config_user_id_path, DocumentDeletionKind, MEILISEARCH_CONFIG_PATH};
|
||||||
use crate::analytics::Analytics;
|
use crate::analytics::Analytics;
|
||||||
use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads};
|
use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads, ScheduleSnapshot};
|
||||||
use crate::routes::indexes::documents::UpdateDocumentsQuery;
|
use crate::routes::indexes::documents::UpdateDocumentsQuery;
|
||||||
use crate::routes::tasks::TasksFilterQueryRaw;
|
use crate::routes::tasks::TasksFilterQueryRaw;
|
||||||
use crate::routes::{create_all_stats, Stats};
|
use crate::routes::{create_all_stats, Stats};
|
||||||
@ -220,9 +220,8 @@ struct Infos {
|
|||||||
ignore_missing_dump: bool,
|
ignore_missing_dump: bool,
|
||||||
ignore_dump_if_db_exists: bool,
|
ignore_dump_if_db_exists: bool,
|
||||||
import_snapshot: bool,
|
import_snapshot: bool,
|
||||||
schedule_snapshot: bool,
|
schedule_snapshot: Option<u64>,
|
||||||
snapshot_dir: bool,
|
snapshot_dir: bool,
|
||||||
snapshot_interval_sec: u64,
|
|
||||||
ignore_missing_snapshot: bool,
|
ignore_missing_snapshot: bool,
|
||||||
ignore_snapshot_if_db_exists: bool,
|
ignore_snapshot_if_db_exists: bool,
|
||||||
http_addr: bool,
|
http_addr: bool,
|
||||||
@ -267,7 +266,6 @@ impl From<Opt> for Infos {
|
|||||||
ignore_snapshot_if_db_exists,
|
ignore_snapshot_if_db_exists,
|
||||||
snapshot_dir,
|
snapshot_dir,
|
||||||
schedule_snapshot,
|
schedule_snapshot,
|
||||||
snapshot_interval_sec,
|
|
||||||
import_dump,
|
import_dump,
|
||||||
ignore_missing_dump,
|
ignore_missing_dump,
|
||||||
ignore_dump_if_db_exists,
|
ignore_dump_if_db_exists,
|
||||||
@ -280,6 +278,11 @@ impl From<Opt> for Infos {
|
|||||||
no_analytics: _,
|
no_analytics: _,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
|
let schedule_snapshot = match schedule_snapshot {
|
||||||
|
ScheduleSnapshot::Disabled => None,
|
||||||
|
ScheduleSnapshot::Enabled(interval) => Some(interval),
|
||||||
|
};
|
||||||
|
|
||||||
let IndexerOpts {
|
let IndexerOpts {
|
||||||
log_every_n: _,
|
log_every_n: _,
|
||||||
max_nb_chunks: _,
|
max_nb_chunks: _,
|
||||||
@ -299,7 +302,6 @@ impl From<Opt> for Infos {
|
|||||||
import_snapshot: import_snapshot.is_some(),
|
import_snapshot: import_snapshot.is_some(),
|
||||||
schedule_snapshot,
|
schedule_snapshot,
|
||||||
snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"),
|
snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"),
|
||||||
snapshot_interval_sec,
|
|
||||||
ignore_missing_snapshot,
|
ignore_missing_snapshot,
|
||||||
ignore_snapshot_if_db_exists,
|
ignore_snapshot_if_db_exists,
|
||||||
http_addr: http_addr != default_http_addr(),
|
http_addr: http_addr != default_http_addr(),
|
||||||
|
@ -41,6 +41,7 @@ use meilisearch_types::tasks::KindWithContent;
|
|||||||
use meilisearch_types::versioning::{check_version_file, create_version_file};
|
use meilisearch_types::versioning::{check_version_file, create_version_file};
|
||||||
use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
|
use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
|
||||||
pub use option::Opt;
|
pub use option::Opt;
|
||||||
|
use option::ScheduleSnapshot;
|
||||||
|
|
||||||
use crate::error::MeilisearchHttpError;
|
use crate::error::MeilisearchHttpError;
|
||||||
|
|
||||||
@ -169,8 +170,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Auth
|
|||||||
|
|
||||||
// We create a loop in a thread that registers snapshotCreation tasks
|
// We create a loop in a thread that registers snapshotCreation tasks
|
||||||
let index_scheduler = Arc::new(index_scheduler);
|
let index_scheduler = Arc::new(index_scheduler);
|
||||||
if opt.schedule_snapshot {
|
if let ScheduleSnapshot::Enabled(snapshot_delay) = opt.schedule_snapshot {
|
||||||
let snapshot_delay = Duration::from_secs(opt.snapshot_interval_sec);
|
let snapshot_delay = Duration::from_secs(snapshot_delay);
|
||||||
let index_scheduler = index_scheduler.clone();
|
let index_scheduler = index_scheduler.clone();
|
||||||
thread::Builder::new()
|
thread::Builder::new()
|
||||||
.name(String::from("register-snapshot-tasks"))
|
.name(String::from("register-snapshot-tasks"))
|
||||||
|
@ -44,7 +44,6 @@ const MEILI_IGNORE_MISSING_SNAPSHOT: &str = "MEILI_IGNORE_MISSING_SNAPSHOT";
|
|||||||
const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS";
|
const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS";
|
||||||
const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR";
|
const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR";
|
||||||
const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT";
|
const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT";
|
||||||
const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC";
|
|
||||||
const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP";
|
const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP";
|
||||||
const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
|
const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
|
||||||
const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
|
const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
|
||||||
@ -63,6 +62,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB";
|
|||||||
const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
|
const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
|
||||||
const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
|
const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
|
||||||
const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
|
const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
|
||||||
|
const DEFAULT_SNAPSHOT_INTERVAL_SEC_STR: &str = "86400";
|
||||||
const DEFAULT_DUMP_DIR: &str = "dumps/";
|
const DEFAULT_DUMP_DIR: &str = "dumps/";
|
||||||
|
|
||||||
const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
|
const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
|
||||||
@ -241,14 +241,11 @@ pub struct Opt {
|
|||||||
pub snapshot_dir: PathBuf,
|
pub snapshot_dir: PathBuf,
|
||||||
|
|
||||||
/// Activates scheduled snapshots when provided. Snapshots are disabled by default.
|
/// Activates scheduled snapshots when provided. Snapshots are disabled by default.
|
||||||
#[clap(long, env = MEILI_SCHEDULE_SNAPSHOT)]
|
///
|
||||||
#[serde(default)]
|
/// When provided with a value, defines the interval between each snapshot, in seconds.
|
||||||
pub schedule_snapshot: bool,
|
#[clap(long,env = MEILI_SCHEDULE_SNAPSHOT, num_args(0..=1), value_parser=parse_schedule_snapshot, default_value_t, default_missing_value=default_snapshot_interval_sec(), value_name = "SNAPSHOT_INTERVAL_SEC")]
|
||||||
|
#[serde(default, deserialize_with = "schedule_snapshot_deserialize")]
|
||||||
/// Defines the interval between each snapshot. Value must be given in seconds.
|
pub schedule_snapshot: ScheduleSnapshot,
|
||||||
#[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value_t = default_snapshot_interval_sec())]
|
|
||||||
#[serde(default = "default_snapshot_interval_sec")]
|
|
||||||
pub snapshot_interval_sec: u64,
|
|
||||||
|
|
||||||
/// Imports the dump file located at the specified path. Path must point to a `.dump` file.
|
/// Imports the dump file located at the specified path. Path must point to a `.dump` file.
|
||||||
/// If a database already exists, Meilisearch will throw an error and abort launch.
|
/// If a database already exists, Meilisearch will throw an error and abort launch.
|
||||||
@ -376,7 +373,6 @@ impl Opt {
|
|||||||
ssl_tickets,
|
ssl_tickets,
|
||||||
snapshot_dir,
|
snapshot_dir,
|
||||||
schedule_snapshot,
|
schedule_snapshot,
|
||||||
snapshot_interval_sec,
|
|
||||||
dump_dir,
|
dump_dir,
|
||||||
log_level,
|
log_level,
|
||||||
indexer_options,
|
indexer_options,
|
||||||
@ -425,11 +421,10 @@ impl Opt {
|
|||||||
export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string());
|
export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string());
|
||||||
export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string());
|
export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string());
|
||||||
export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir);
|
export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir);
|
||||||
export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, schedule_snapshot.to_string());
|
if let Some(snapshot_interval) = schedule_snapshot_to_env(schedule_snapshot) {
|
||||||
export_to_env_if_not_present(
|
export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, snapshot_interval)
|
||||||
MEILI_SNAPSHOT_INTERVAL_SEC,
|
}
|
||||||
snapshot_interval_sec.to_string(),
|
|
||||||
);
|
|
||||||
export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
|
export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
|
||||||
export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string());
|
export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string());
|
||||||
#[cfg(feature = "metrics")]
|
#[cfg(feature = "metrics")]
|
||||||
@ -744,8 +739,8 @@ fn default_snapshot_dir() -> PathBuf {
|
|||||||
PathBuf::from(DEFAULT_SNAPSHOT_DIR)
|
PathBuf::from(DEFAULT_SNAPSHOT_DIR)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_snapshot_interval_sec() -> u64 {
|
fn default_snapshot_interval_sec() -> &'static str {
|
||||||
DEFAULT_SNAPSHOT_INTERVAL_SEC
|
DEFAULT_SNAPSHOT_INTERVAL_SEC_STR
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_dump_dir() -> PathBuf {
|
fn default_dump_dir() -> PathBuf {
|
||||||
@ -756,6 +751,102 @@ fn default_log_every_n() -> usize {
|
|||||||
DEFAULT_LOG_EVERY_N
|
DEFAULT_LOG_EVERY_N
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Indicates if a snapshot was scheduled, and if yes with which interval.
|
||||||
|
#[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)]
|
||||||
|
pub enum ScheduleSnapshot {
|
||||||
|
/// Scheduled snapshots are disabled.
|
||||||
|
#[default]
|
||||||
|
Disabled,
|
||||||
|
/// Snapshots are scheduled at the specified interval, in seconds.
|
||||||
|
Enabled(u64),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Display for ScheduleSnapshot {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
ScheduleSnapshot::Disabled => write!(f, ""),
|
||||||
|
ScheduleSnapshot::Enabled(value) => write!(f, "{}", value),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FromStr for ScheduleSnapshot {
|
||||||
|
type Err = ParseIntError;
|
||||||
|
|
||||||
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
|
Ok(match s {
|
||||||
|
"" => ScheduleSnapshot::Disabled,
|
||||||
|
s => ScheduleSnapshot::Enabled(s.parse()?),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_schedule_snapshot(s: &str) -> Result<ScheduleSnapshot, ParseIntError> {
|
||||||
|
Ok(if s.is_empty() { ScheduleSnapshot::Disabled } else { ScheduleSnapshot::from_str(s)? })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schedule_snapshot_to_env(schedule_snapshot: ScheduleSnapshot) -> Option<String> {
|
||||||
|
match schedule_snapshot {
|
||||||
|
ScheduleSnapshot::Enabled(snapshot_delay) => Some(snapshot_delay.to_string()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schedule_snapshot_deserialize<'de, D>(deserializer: D) -> Result<ScheduleSnapshot, D::Error>
|
||||||
|
where
|
||||||
|
D: serde::Deserializer<'de>,
|
||||||
|
{
|
||||||
|
struct BoolOrInt;
|
||||||
|
|
||||||
|
impl<'de> serde::de::Visitor<'de> for BoolOrInt {
|
||||||
|
type Value = ScheduleSnapshot;
|
||||||
|
|
||||||
|
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
formatter.write_str("integer or boolean")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E>
|
||||||
|
where
|
||||||
|
E: serde::de::Error,
|
||||||
|
{
|
||||||
|
Ok(if value {
|
||||||
|
ScheduleSnapshot::Enabled(DEFAULT_SNAPSHOT_INTERVAL_SEC)
|
||||||
|
} else {
|
||||||
|
ScheduleSnapshot::Disabled
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
|
||||||
|
where
|
||||||
|
E: serde::de::Error,
|
||||||
|
{
|
||||||
|
Ok(ScheduleSnapshot::Enabled(v as u64))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
|
||||||
|
where
|
||||||
|
E: serde::de::Error,
|
||||||
|
{
|
||||||
|
Ok(ScheduleSnapshot::Enabled(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_none<E>(self) -> Result<Self::Value, E>
|
||||||
|
where
|
||||||
|
E: serde::de::Error,
|
||||||
|
{
|
||||||
|
Ok(ScheduleSnapshot::Disabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_unit<E>(self) -> Result<Self::Value, E>
|
||||||
|
where
|
||||||
|
E: serde::de::Error,
|
||||||
|
{
|
||||||
|
Ok(ScheduleSnapshot::Disabled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_any(BoolOrInt)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod test {
|
mod test {
|
||||||
|
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use meilisearch::option::ScheduleSnapshot;
|
||||||
use meilisearch::Opt;
|
use meilisearch::Opt;
|
||||||
use tokio::time::sleep;
|
use tokio::time::sleep;
|
||||||
|
|
||||||
@ -36,8 +37,7 @@ async fn perform_snapshot() {
|
|||||||
|
|
||||||
let options = Opt {
|
let options = Opt {
|
||||||
snapshot_dir: snapshot_dir.path().to_owned(),
|
snapshot_dir: snapshot_dir.path().to_owned(),
|
||||||
snapshot_interval_sec: 1,
|
schedule_snapshot: ScheduleSnapshot::Enabled(1),
|
||||||
schedule_snapshot: true,
|
|
||||||
..default_settings(temp.path())
|
..default_settings(temp.path())
|
||||||
};
|
};
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user