use assert_json_diff::{assert_json_eq, assert_json_include}; use meilisearch_http::helpers::compression; use serde_json::{json, Value}; use std::fs::File; use std::path::Path; use std::thread; use std::time::Duration; use tempfile::TempDir; #[macro_use] mod common; async fn trigger_and_wait_dump(server: &mut common::Server) -> String { let (value, status_code) = server.trigger_dump().await; assert_eq!(status_code, 202); let dump_uid = value["uid"].as_str().unwrap().to_string(); for _ in 0..20 as u8 { let (value, status_code) = server.get_dump_status(&dump_uid).await; assert_eq!(status_code, 200); assert_ne!(value["status"].as_str(), Some("dump_process_failed")); if value["status"].as_str() == Some("done") { return dump_uid } thread::sleep(Duration::from_millis(100)); } unreachable!("dump creation runned out of time") } fn current_db_version() -> (String, String, String) { let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); (current_version_major, current_version_minor, current_version_patch) } fn current_dump_version() -> String { "V1".into() } fn read_all_jsonline(r: R) -> Value { let deserializer = serde_json::Deserializer::from_reader(r); let iterator = deserializer.into_iter::(); json!(iterator.map(|v| v.unwrap()).collect::>()) } #[actix_rt::test] #[ignore] async fn trigger_dump_should_return_ok() { let server = common::Server::test_server().await; let (_, status_code) = server.trigger_dump().await; assert_eq!(status_code, 202); } #[actix_rt::test] #[ignore] async fn trigger_dump_twice_should_return_conflict() { let server = common::Server::test_server().await; let expected = json!({ "message": "Another dump is already in progress", "errorCode": "dump_already_in_progress", "errorType": "invalid_request_error", "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" }); let (_, status_code) = server.trigger_dump().await; assert_eq!(status_code, 202); let (value, status_code) = server.trigger_dump().await; assert_json_eq!(expected, value, ordered: false); assert_eq!(status_code, 409); } #[actix_rt::test] #[ignore] async fn trigger_dump_concurently_should_return_conflict() { let server = common::Server::test_server().await; let expected = json!({ "message": "Another dump is already in progress", "errorCode": "dump_already_in_progress", "errorType": "invalid_request_error", "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" }); let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); assert_json_eq!(expected, value_2, ordered: false); assert_eq!(status_code_2, 409); } #[actix_rt::test] #[ignore] async fn get_dump_status_early_should_return_in_progress() { let mut server = common::Server::test_server().await; let (value, status_code) = server.trigger_dump().await; assert_eq!(status_code, 202); let dump_uid = value["uid"].as_str().unwrap().to_string(); let (value, status_code) = server.get_dump_status(&dump_uid).await; let expected = json!({ "uid": dump_uid, "status": "in_progress" }); assert_eq!(status_code, 200); assert_json_eq!(expected, value, ordered: false); } #[actix_rt::test] #[ignore] async fn get_dump_status_should_return_done() { let mut server = common::Server::test_server().await; let (value, status_code) = server.trigger_dump().await; assert_eq!(status_code, 202); let dump_uid = value["uid"].as_str().unwrap().to_string(); let expected = json!({ "uid": dump_uid.clone(), "status": "done" }); thread::sleep(Duration::from_secs(1)); // wait dump until process end let (value, status_code) = server.get_dump_status(&dump_uid).await; assert_eq!(status_code, 200); assert_json_eq!(expected, value, ordered: false); } #[actix_rt::test] #[ignore] async fn get_dump_status_should_return_error_provoking_it() { let mut server = common::Server::test_server().await; let (value, status_code) = server.trigger_dump().await; // removing destination directory provoking `No such file or directory` error std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); assert_eq!(status_code, 202); let dump_uid = value["uid"].as_str().unwrap().to_string(); let expected = json!({ "uid": dump_uid.clone(), "status": "failed", "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", "errorCode": "dump_process_failed", "errorType": "internal_error", "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" }); thread::sleep(Duration::from_secs(1)); // wait dump until process end let (value, status_code) = server.get_dump_status(&dump_uid).await; assert_eq!(status_code, 200); assert_json_eq!(expected, value, ordered: false); } #[actix_rt::test] #[ignore] async fn dump_metadata_should_be_valid() { let mut server = common::Server::test_server().await; let body = json!({ "uid": "test2", "primaryKey": "test2_id", }); server.create_index(body).await; let uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); // fields are randomly ordered metadata.get_mut("indexes").unwrap() .as_array_mut().unwrap() .sort_by(|a, b| a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) ); let (major, minor, patch) = current_db_version(); let expected = json!({ "indexes": [{ "uid": "test", "primaryKey": "id", }, { "uid": "test2", "primaryKey": "test2_id", } ], "dbVersion": format!("{}.{}.{}", major, minor, patch), "dumpVersion": current_dump_version() }); assert_json_include!(expected: expected, actual: metadata); } #[actix_rt::test] #[ignore] async fn dump_gzip_should_have_been_created() { let mut server = common::Server::test_server().await; let dump_uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); assert!(File::open(compressed_path).is_ok()); } #[actix_rt::test] #[ignore] async fn dump_index_settings_should_be_valid() { let mut server = common::Server::test_server().await; let expected = json!({ "rankingRules": [ "typo", "words", "proximity", "attribute", "wordsPosition", "exactness" ], "distinctAttribute": "email", "searchableAttributes": [ "balance", "picture", "age", "color", "name", "gender", "email", "phone", "address", "about", "registered", "latitude", "longitude", "tags" ], "displayedAttributes": [ "id", "isActive", "balance", "picture", "age", "color", "name", "gender", "email", "phone", "address", "about", "registered", "latitude", "longitude", "tags" ], "stopWords": [ "in", "ad" ], "synonyms": { "wolverine": ["xmen", "logan"], "logan": ["wolverine", "xmen"] }, "attributesForFaceting": [ "gender", "color", "tags" ] }); server.update_all_settings(expected.clone()).await; let uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); assert_json_eq!(expected, settings, ordered: false); } #[actix_rt::test] #[ignore] async fn dump_index_documents_should_be_valid() { let mut server = common::Server::test_server().await; let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); let mut slice: &[u8] = dataset; let expected: Value = read_all_jsonline(&mut slice); let uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); let documents = read_all_jsonline(file); assert_json_eq!(expected, documents, ordered: false); } #[actix_rt::test] #[ignore] async fn dump_index_updates_should_be_valid() { let mut server = common::Server::test_server().await; let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); let mut slice: &[u8] = dataset; let expected: Value = read_all_jsonline(&mut slice); let uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); let mut updates = read_all_jsonline(file); // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) updates.as_array_mut().unwrap() .get_mut(0).unwrap() .get_mut("type").unwrap() .get_mut("settings").unwrap() .get_mut("displayed_attributes").unwrap() .get_mut("Update").unwrap() .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); eprintln!("{}\n", updates.to_string()); eprintln!("{}", expected.to_string()); assert_json_include!(expected: expected, actual: updates); } #[actix_rt::test] #[ignore] async fn get_unexisting_dump_status_should_return_not_found() { let mut server = common::Server::test_server().await; let (_, status_code) = server.get_dump_status("4242").await; assert_eq!(status_code, 404); }