MeiliSearch/crates/meilisearch/tests/snapshot/mod.rs

206 lines
7.0 KiB
Rust
Raw Normal View History

2021-03-25 10:24:33 +01:00
use std::time::Duration;
2023-09-11 11:00:05 +02:00
use meili_snap::{json_string, snapshot};
use meilisearch::option::ScheduleSnapshot;
use meilisearch::Opt;
2021-03-25 10:24:33 +01:00
2022-10-20 18:00:07 +02:00
use crate::common::server::default_settings;
use crate::common::{GetAllDocumentsOptions, Server};
2023-09-11 16:50:53 +02:00
use crate::json;
2021-03-25 10:24:33 +01:00
macro_rules! verify_snapshot {
(
$orig:expr,
$snapshot: expr,
|$server:ident| =>
$($e:expr,)+) => {
use std::sync::Arc;
let snapshot = Arc::new($snapshot);
let orig = Arc::new($orig);
$(
{
let test= |$server: Arc<Server>| async move {
$e.await
};
let (snapshot, _) = test(snapshot.clone()).await;
let (orig, _) = test(orig.clone()).await;
assert_eq!(snapshot, orig, "Got \n{}\nWhile expecting:\n{}", serde_json::to_string_pretty(&snapshot).unwrap(), serde_json::to_string_pretty(&orig).unwrap());
}
)*
};
}
2021-03-25 10:24:33 +01:00
#[actix_rt::test]
#[cfg_attr(target_os = "windows", ignore)]
2021-03-25 10:24:33 +01:00
async fn perform_snapshot() {
2021-09-28 18:10:09 +02:00
let temp = tempfile::tempdir().unwrap();
let snapshot_dir = tempfile::tempdir().unwrap();
2021-03-25 10:24:33 +01:00
let options = Opt {
snapshot_dir: snapshot_dir.path().to_owned(),
schedule_snapshot: ScheduleSnapshot::Enabled(2),
2021-03-25 10:24:33 +01:00
..default_settings(temp.path())
};
2022-04-28 10:48:57 +02:00
let server = Server::new_with_options(options).await.unwrap();
2021-03-25 10:24:33 +01:00
let index = server.index("test");
index
2023-09-11 16:50:53 +02:00
.update_settings(json! ({
"searchableAttributes": [],
}))
.await;
2021-03-25 10:24:33 +01:00
index.load_test_set().await;
2024-05-16 11:28:14 +02:00
let (task, code) = server.index("test1").create(Some("prim")).await;
meili_snap::snapshot!(code, @"202 Accepted");
2024-05-16 11:28:14 +02:00
index.wait_task(task.uid()).await;
2021-03-25 10:24:33 +01:00
2024-05-16 11:28:14 +02:00
// wait for the _next task_ to process, aka the snapshot that should be enqueued at some point
println!("waited for the next task to finish");
let now = std::time::Instant::now();
let next_task = task.uid() + 1;
loop {
let (value, code) = index.get_task(next_task).await;
dbg!(&value);
if code != 404 && value["status"].as_str() == Some("succeeded") {
break;
}
if now.elapsed() > Duration::from_secs(30) {
panic!("The snapshot didn't schedule in 30s even though it was supposed to be scheduled every 2s: {}",
serde_json::to_string_pretty(&value).unwrap()
);
}
}
2021-03-25 10:24:33 +01:00
2021-09-28 18:10:09 +02:00
let temp = tempfile::tempdir().unwrap();
2021-03-25 10:24:33 +01:00
2022-02-28 14:43:22 +01:00
let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
#[cfg_attr(windows, allow(unused))]
let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = snapshot_meta.permissions().mode();
// rwxrwxrwx
2023-09-11 11:00:05 +02:00
meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
}
let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
let snapshot_server = Server::new_with_options(options).await.unwrap();
verify_snapshot!(server, snapshot_server, |server| =>
server.list_indexes(None, None),
// for some reason the db sizes differ. this may be due to the compaction options we have
// set when performing the snapshot
//server.stats(),
// The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
server.tasks_filter("?from=2"),
server.index("test").get_all_documents(GetAllDocumentsOptions::default()),
server.index("test").settings(),
server.index("test1").get_all_documents(GetAllDocumentsOptions::default()),
server.index("test1").settings(),
);
}
#[actix_rt::test]
async fn perform_on_demand_snapshot() {
let temp = tempfile::tempdir().unwrap();
let snapshot_dir = tempfile::tempdir().unwrap();
let options =
Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) };
let server = Server::new_with_options(options).await.unwrap();
let index = server.index("catto");
index
2023-09-11 16:50:53 +02:00
.update_settings(json! ({
2023-09-11 11:00:05 +02:00
"searchableAttributes": [],
}))
.await;
index.load_test_set().await;
server.index("doggo").create(Some("bone")).await;
index.wait_task(2).await;
server.index("doggo").create(Some("bone")).await;
index.wait_task(2).await;
let (task, code) = server.create_snapshot().await;
snapshot!(code, @"202 Accepted");
snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###"
{
"taskUid": 4,
"indexUid": null,
"status": "enqueued",
"type": "snapshotCreation",
"enqueuedAt": "[date]"
}
"###);
2023-09-11 16:56:16 +02:00
let task = index.wait_task(task.uid()).await;
2023-09-11 11:00:05 +02:00
snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
{
2023-09-11 16:50:53 +02:00
"uid": 4,
2024-11-13 11:27:12 +01:00
"batchUid": 4,
2023-09-11 16:50:53 +02:00
"indexUid": null,
"status": "succeeded",
"type": "snapshotCreation",
2023-09-11 11:00:05 +02:00
"canceledBy": null,
2023-09-11 16:50:53 +02:00
"error": null,
2023-09-11 11:00:05 +02:00
"duration": "[duration]",
"enqueuedAt": "[date]",
"startedAt": "[date]",
"finishedAt": "[date]"
}
"###);
let temp = tempfile::tempdir().unwrap();
let snapshots: Vec<String> = std::fs::read_dir(&snapshot_dir)
.unwrap()
.map(|entry| entry.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
.collect();
meili_snap::snapshot!(format!("{snapshots:?}"), @r###"["db.snapshot"]"###);
2023-09-11 11:00:05 +02:00
let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
#[cfg_attr(windows, allow(unused))]
let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = snapshot_meta.permissions().mode();
// rwxrwxrwx
meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
}
2021-03-25 10:24:33 +01:00
2022-10-20 18:00:07 +02:00
let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
2021-03-25 10:24:33 +01:00
2022-04-28 10:48:57 +02:00
let snapshot_server = Server::new_with_options(options).await.unwrap();
2021-03-25 10:24:33 +01:00
verify_snapshot!(server, snapshot_server, |server| =>
server.list_indexes(None, None),
// for some reason the db sizes differ. this may be due to the compaction options we have
// set when performing the snapshot
//server.stats(),
2023-03-06 17:04:24 +01:00
// The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
server.tasks_filter("?from=2"),
2023-09-11 13:52:07 +02:00
server.index("catto").get_all_documents(GetAllDocumentsOptions::default()),
server.index("catto").settings(),
server.index("doggo").get_all_documents(GetAllDocumentsOptions::default()),
server.index("doggo").settings(),
);
2021-03-25 10:24:33 +01:00
}