From e8c9367686680adbdda14623f1effa0fbb14f2b4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Sep 2023 11:00:05 +0200 Subject: [PATCH] implement the snapshots on demand --- meilisearch-auth/src/store.rs | 3 + meilisearch-types/src/keys.rs | 8 +++ meilisearch/src/routes/mod.rs | 2 + meilisearch/src/routes/snapshot.rs | 32 ++++++++++ meilisearch/tests/common/server.rs | 4 ++ meilisearch/tests/snapshot/mod.rs | 95 ++++++++++++++++++++++++++++++ 6 files changed, 144 insertions(+) create mode 100644 meilisearch/src/routes/snapshot.rs diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index e6e30d18d..28ec8b5e4 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -129,6 +129,9 @@ impl HeedAuthStore { Action::DumpsAll => { actions.insert(Action::DumpsCreate); } + Action::SnapshotsAll => { + actions.insert(Action::SnapshotsCreate); + } Action::TasksAll => { actions.extend([Action::TasksGet, Action::TasksDelete, Action::TasksCancel]); } diff --git a/meilisearch-types/src/keys.rs b/meilisearch-types/src/keys.rs index cfeb3aa8f..f7d80bbcb 100644 --- a/meilisearch-types/src/keys.rs +++ b/meilisearch-types/src/keys.rs @@ -257,6 +257,12 @@ pub enum Action { #[serde(rename = "dumps.create")] #[deserr(rename = "dumps.create")] DumpsCreate, + #[serde(rename = "snapshots.*")] + #[deserr(rename = "snapshots.*")] + SnapshotsAll, + #[serde(rename = "snapshots.create")] + #[deserr(rename = "snapshots.create")] + SnapshotsCreate, #[serde(rename = "version")] #[deserr(rename = "version")] Version, @@ -309,6 +315,7 @@ impl Action { METRICS_GET => Some(Self::MetricsGet), DUMPS_ALL => Some(Self::DumpsAll), DUMPS_CREATE => Some(Self::DumpsCreate), + SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate), VERSION => Some(Self::Version), KEYS_CREATE => Some(Self::KeysAdd), KEYS_GET => Some(Self::KeysGet), @@ -353,6 +360,7 @@ pub mod actions { pub const METRICS_GET: u8 = MetricsGet.repr(); pub const DUMPS_ALL: u8 = DumpsAll.repr(); pub const DUMPS_CREATE: u8 = DumpsCreate.repr(); + pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr(); pub const VERSION: u8 = Version.repr(); pub const KEYS_CREATE: u8 = KeysAdd.repr(); pub const KEYS_GET: u8 = KeysGet.repr(); diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 69bc44160..0e5623b09 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -24,6 +24,7 @@ pub mod features; pub mod indexes; mod metrics; mod multi_search; +mod snapshot; mod swap_indexes; pub mod tasks; @@ -32,6 +33,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("/health").route(web::get().to(get_health))) .service(web::scope("/keys").configure(api_key::configure)) .service(web::scope("/dumps").configure(dump::configure)) + .service(web::scope("/snapshots").configure(snapshot::configure)) .service(web::resource("/stats").route(web::get().to(get_stats))) .service(web::resource("/version").route(web::get().to(get_version))) .service(web::scope("/indexes").configure(indexes::configure)) diff --git a/meilisearch/src/routes/snapshot.rs b/meilisearch/src/routes/snapshot.rs new file mode 100644 index 000000000..7fa22658a --- /dev/null +++ b/meilisearch/src/routes/snapshot.rs @@ -0,0 +1,32 @@ +use actix_web::web::Data; +use actix_web::{web, HttpRequest, HttpResponse}; +use index_scheduler::IndexScheduler; +use log::debug; +use meilisearch_types::error::ResponseError; +use meilisearch_types::tasks::KindWithContent; +use serde_json::json; + +use crate::analytics::Analytics; +use crate::extractors::authentication::policies::*; +use crate::extractors::authentication::GuardedData; +use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::SummarizedTaskView; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot)))); +} + +pub async fn create_snapshot( + index_scheduler: GuardedData, Data>, + req: HttpRequest, + analytics: web::Data, +) -> Result { + analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req)); + + let task = KindWithContent::SnapshotCreation; + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into(); + + debug!("returns: {:?}", task); + Ok(HttpResponse::Accepted().json(task)) +} diff --git a/meilisearch/tests/common/server.rs b/meilisearch/tests/common/server.rs index 40d8e8366..d3ec71a72 100644 --- a/meilisearch/tests/common/server.rs +++ b/meilisearch/tests/common/server.rs @@ -156,6 +156,10 @@ impl Server { self.service.post("/dumps", json!(null)).await } + pub async fn create_snapshot(&self) -> (Value, StatusCode) { + self.service.post("/snapshots", json!(null)).await + } + pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) { self.service.post("/swap-indexes", value).await } diff --git a/meilisearch/tests/snapshot/mod.rs b/meilisearch/tests/snapshot/mod.rs index 97db3db8d..bdcc835ba 100644 --- a/meilisearch/tests/snapshot/mod.rs +++ b/meilisearch/tests/snapshot/mod.rs @@ -1,6 +1,7 @@ use std::time::Duration; use actix_rt::time::sleep; +use meili_snap::{json_string, snapshot}; use meilisearch::option::ScheduleSnapshot; use meilisearch::Opt; @@ -90,3 +91,97 @@ async fn perform_snapshot() { server.index("test1").settings(), ); } + +#[actix_rt::test] +async fn perform_on_demand_snapshot() { + let temp = tempfile::tempdir().unwrap(); + let snapshot_dir = tempfile::tempdir().unwrap(); + + let options = + Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) }; + + let server = Server::new_with_options(options).await.unwrap(); + + let index = server.index("catto"); + index + .update_settings(serde_json::json! ({ + "searchableAttributes": [], + })) + .await; + + index.load_test_set().await; + + server.index("doggo").create(Some("bone")).await; + index.wait_task(2).await; + + server.index("doggo").create(Some("bone")).await; + index.wait_task(2).await; + + let (task, code) = server.create_snapshot().await; + snapshot!(code, @"202 Accepted"); + snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###" + { + "taskUid": 4, + "indexUid": null, + "status": "enqueued", + "type": "snapshotCreation", + "enqueuedAt": "[date]" + } + "###); + let task = index.wait_task(3).await; + snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###" + { + "uid": 3, + "indexUid": "doggo", + "status": "failed", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": "bone" + }, + "error": { + "message": "Index `doggo` already exists.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "###); + + let temp = tempfile::tempdir().unwrap(); + + let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot"); + #[cfg_attr(windows, allow(unused))] + let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = snapshot_meta.permissions().mode(); + // rwxrwxrwx + meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100"); + } + + let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) }; + + let snapshot_server = Server::new_with_options(options).await.unwrap(); + + verify_snapshot!(server, snapshot_server, |server| => + server.list_indexes(None, None), + // for some reason the db sizes differ. this may be due to the compaction options we have + // set when performing the snapshot + //server.stats(), + + // The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4 + server.tasks_filter("?from=2"), + + server.index("test").get_all_documents(GetAllDocumentsOptions::default()), + server.index("test").settings(), + server.index("test1").get_all_documents(GetAllDocumentsOptions::default()), + server.index("test1").settings(), + ); +}