MeiliSearch/meilisearch-http/tests/dump.rs

use assert_json_diff::{assert_json_eq, assert_json_include};
use meilisearch_http::helpers::compression;
use serde_json::{json, Value};
use std::fs::File;
use std::path::Path;
use std::thread;
use std::time::Duration;
use tempfile::TempDir;

#[macro_use] mod common;

async fn trigger_and_wait_dump(server: &mut common::Server) -> String {
    let (value, status_code) = server.trigger_dump().await;

    assert_eq!(status_code, 202);

    let dump_uid = value["uid"].as_str().unwrap().to_string();

    for _ in 0..20 as u8 {
        let (value, status_code) = server.get_dump_status(&dump_uid).await;
    
        assert_eq!(status_code, 200);
        assert_ne!(value["status"].as_str(), Some("dump_process_failed"));
        
        if value["status"].as_str() == Some("done") { return dump_uid }
        thread::sleep(Duration::from_millis(100));
    }

    unreachable!("dump creation runned out of time")
}

fn current_db_version() -> (String, String, String) {
    let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string();
    let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string();
    let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string();

    (current_version_major, current_version_minor, current_version_patch)
}

fn current_dump_version() -> String {
    "V1".into()
}

fn read_all_jsonline<R: std::io::Read>(r: R) -> Value {
    let deserializer = serde_json::Deserializer::from_reader(r);
    let iterator = deserializer.into_iter::<serde_json::Value>();

    json!(iterator.map(|v| v.unwrap()).collect::<Vec<Value>>())
}

#[actix_rt::test]
#[ignore]
async fn trigger_dump_should_return_ok() {
    let server = common::Server::test_server().await;

    let (_, status_code) = server.trigger_dump().await;

    assert_eq!(status_code, 202);
}

#[actix_rt::test]
#[ignore]
async fn trigger_dump_twice_should_return_conflict() {
    let server = common::Server::test_server().await;

    let expected = json!({
        "message": "Another dump is already in progress",
        "errorCode": "dump_already_in_progress",
        "errorType": "invalid_request_error",
        "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress"
    });

    let (_, status_code) = server.trigger_dump().await;

    assert_eq!(status_code, 202);

    let (value, status_code) = server.trigger_dump().await;

    
    assert_json_eq!(expected, value, ordered: false);
    assert_eq!(status_code, 409);
}

#[actix_rt::test]
#[ignore]
async fn trigger_dump_concurently_should_return_conflict() {
    let server = common::Server::test_server().await;

    let expected = json!({
        "message": "Another dump is already in progress",
        "errorCode": "dump_already_in_progress",
        "errorType": "invalid_request_error",
        "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress"
    });

    let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump());
    
    assert_json_eq!(expected, value_2, ordered: false);
    assert_eq!(status_code_2, 409);
}

#[actix_rt::test]
#[ignore]
async fn get_dump_status_early_should_return_in_progress() {
    let mut server = common::Server::test_server().await;
    


    let (value, status_code) = server.trigger_dump().await;

    assert_eq!(status_code, 202);

    let dump_uid = value["uid"].as_str().unwrap().to_string();

    let (value, status_code) = server.get_dump_status(&dump_uid).await;

    let expected = json!({
        "uid": dump_uid,
        "status": "in_progress"
    });

    assert_eq!(status_code, 200);

    assert_json_eq!(expected, value, ordered: false);
}

#[actix_rt::test]
#[ignore]
async fn get_dump_status_should_return_done() {
    let mut server = common::Server::test_server().await;


    let (value, status_code) = server.trigger_dump().await;

    assert_eq!(status_code, 202);

    let dump_uid = value["uid"].as_str().unwrap().to_string();
    
    let expected = json!({
        "uid": dump_uid.clone(),
        "status": "done"
    });

    thread::sleep(Duration::from_secs(1)); // wait dump until process end

    let (value, status_code) = server.get_dump_status(&dump_uid).await;

    assert_eq!(status_code, 200);

    assert_json_eq!(expected, value, ordered: false);
}

#[actix_rt::test]
#[ignore]
async fn get_dump_status_should_return_error_provoking_it() {
    let mut server = common::Server::test_server().await;


    let (value, status_code) = server.trigger_dump().await;

    // removing destination directory provoking `No such file or directory` error
    std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap();

    assert_eq!(status_code, 202);

    let dump_uid = value["uid"].as_str().unwrap().to_string();
    
    let expected = json!({
        "uid": dump_uid.clone(),
        "status": "failed",
        "message": "Dump process failed: compressing dump; No such file or directory (os error 2)",
        "errorCode": "dump_process_failed",
        "errorType": "internal_error",
        "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed"
    });

    thread::sleep(Duration::from_secs(1)); // wait dump until process end

    let (value, status_code) = server.get_dump_status(&dump_uid).await;

    assert_eq!(status_code, 200);

    assert_json_eq!(expected, value, ordered: false);
}

#[actix_rt::test]
#[ignore]
async fn dump_metadata_should_be_valid() {
    let mut server = common::Server::test_server().await;
    
    let body = json!({
        "uid": "test2",
        "primaryKey": "test2_id",
    });

    server.create_index(body).await;
    
    let uid = trigger_and_wait_dump(&mut server).await;

    let dumps_dir = Path::new(&server.data().dumps_dir);
    let tmp_dir = TempDir::new().unwrap();
    let tmp_dir_path = tmp_dir.path();

    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();

    let file = File::open(tmp_dir_path.join("metadata.json")).unwrap();
    let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap();

     // fields are randomly ordered
     metadata.get_mut("indexes").unwrap()
        .as_array_mut().unwrap()
        .sort_by(|a, b| 
            a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str())
        );

    let (major, minor, patch) = current_db_version();

    let expected = json!({
        "indexes": [{
                "uid": "test",
                "primaryKey": "id",
            }, {
                "uid": "test2",
                "primaryKey": "test2_id",
            }
        ],
        "dbVersion": format!("{}.{}.{}", major, minor, patch),
        "dumpVersion": current_dump_version()
    });

    assert_json_include!(expected: expected, actual: metadata);
}

#[actix_rt::test]
#[ignore]
async fn dump_gzip_should_have_been_created() {
    let mut server = common::Server::test_server().await;
    

    let dump_uid = trigger_and_wait_dump(&mut server).await;
    let dumps_dir = Path::new(&server.data().dumps_dir);

    let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid));
    assert!(File::open(compressed_path).is_ok());
}

#[actix_rt::test]
#[ignore]
async fn dump_index_settings_should_be_valid() {
    let mut server = common::Server::test_server().await;

    let expected = json!({
        "rankingRules": [
            "typo",
            "words",
            "proximity",
            "attribute",
            "wordsPosition",
            "exactness"
        ],
        "distinctAttribute": "email",
        "searchableAttributes": [
            "balance",
            "picture",
            "age",
            "color",
            "name",
            "gender",
            "email",
            "phone",
            "address",
            "about",
            "registered",
            "latitude",
            "longitude",
            "tags"
        ],
        "displayedAttributes": [
            "id",
            "isActive",
            "balance",
            "picture",
            "age",
            "color",
            "name",
            "gender",
            "email",
            "phone",
            "address",
            "about",
            "registered",
            "latitude",
            "longitude",
            "tags"
        ],
        "stopWords": [
            "in",
            "ad"
        ],
        "synonyms": {
            "wolverine": ["xmen", "logan"],
            "logan": ["wolverine", "xmen"]
        },
        "attributesForFaceting": [
            "gender",
            "color",
            "tags"
        ]
    });

    server.update_all_settings(expected.clone()).await;

    let uid = trigger_and_wait_dump(&mut server).await;

    let dumps_dir = Path::new(&server.data().dumps_dir);
    let tmp_dir = TempDir::new().unwrap();
    let tmp_dir_path = tmp_dir.path();

    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();

    let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap();
    let settings: serde_json::Value = serde_json::from_reader(file).unwrap();

    assert_json_eq!(expected, settings, ordered: false);
}

#[actix_rt::test]
#[ignore]
async fn dump_index_documents_should_be_valid() {
    let mut server = common::Server::test_server().await;

    let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl");
    let mut slice: &[u8] = dataset;

    let expected: Value = read_all_jsonline(&mut slice);

    let uid = trigger_and_wait_dump(&mut server).await;

    let dumps_dir = Path::new(&server.data().dumps_dir);
    let tmp_dir = TempDir::new().unwrap();
    let tmp_dir_path = tmp_dir.path();

    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();

    let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap();
    let documents = read_all_jsonline(file);

    assert_json_eq!(expected, documents, ordered: false);
}

#[actix_rt::test]
#[ignore]
async fn dump_index_updates_should_be_valid() {
    let mut server = common::Server::test_server().await;

    let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl");
    let mut slice: &[u8] = dataset;

    let expected: Value = read_all_jsonline(&mut slice);

    let uid = trigger_and_wait_dump(&mut server).await;

    let dumps_dir = Path::new(&server.data().dumps_dir);
    let tmp_dir = TempDir::new().unwrap();
    let tmp_dir_path = tmp_dir.path();

    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();

    let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap();
    let mut updates = read_all_jsonline(file);


    // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943)
    updates.as_array_mut().unwrap()
            .get_mut(0).unwrap()
            .get_mut("type").unwrap()
            .get_mut("settings").unwrap()
            .get_mut("displayed_attributes").unwrap()
            .get_mut("Update").unwrap()
            .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str()));

    eprintln!("{}\n", updates.to_string());
    eprintln!("{}", expected.to_string());
    assert_json_include!(expected: expected, actual: updates);
}
 
#[actix_rt::test]
#[ignore]
async fn get_unexisting_dump_status_should_return_not_found() {
    let mut server = common::Server::test_server().await;

    let (_, status_code) = server.get_dump_status("4242").await;

    assert_eq!(status_code, 404);
}