diff --git a/crates/meilisearch/tests/search/multi/mod.rs b/crates/meilisearch/tests/search/multi/mod.rs index 4fc0aed7f..2a95a5dd2 100644 --- a/crates/meilisearch/tests/search/multi/mod.rs +++ b/crates/meilisearch/tests/search/multi/mod.rs @@ -5,6 +5,8 @@ use crate::common::Server; use crate::json; use crate::search::{SCORE_DOCUMENTS, VECTOR_DOCUMENTS}; +mod proxy; + #[actix_rt::test] async fn search_empty_list() { let server = Server::new().await; diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs new file mode 100644 index 000000000..2c3b31bf1 --- /dev/null +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -0,0 +1,2591 @@ +use std::sync::Arc; + +use actix_http::StatusCode; +use meili_snap::{json_string, snapshot}; +use wiremock::matchers::AnyMatcher; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +use crate::common::{Server, Value, SCORE_DOCUMENTS}; +use crate::json; + +#[actix_rt::test] +async fn error_feature() { + let server = Server::new().await; + + let (response, code) = server + .multi_search(json!({ + "federation": {}, + "queries": [ + { + "indexUid": "test", + "federationOptions": { + "remote": "toto" + } + } + ]})) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Performing a remote federated search requires enabling the `network` experimental feature. See https://github.com/orgs/meilisearch/discussions/805", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); + + let (response, code) = server + .multi_search(json!({ + "federation": {}, + "queries": [ + { + "indexUid": "test", + "federationOptions": { + "queryPosition": 42, + } + } + ]})) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Using `federationOptions.queryPosition` requires enabling the `network` experimental feature. See https://github.com/orgs/meilisearch/discussions/805", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); +} + +#[actix_rt::test] +async fn error_params() { + let server = Server::new().await; + + let (response, code) = server + .multi_search(json!({ + "federation": {}, + "queries": [ + { + "indexUid": "test", + "federationOptions": { + "remote": 42 + } + } + ]})) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Invalid value type at `.queries[0].federationOptions.remote`: expected a string, but found a positive integer: `42`", + "code": "invalid_multi_search_remote", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_multi_search_remote" + } + "###); + + let (response, code) = server + .multi_search(json!({ + "federation": {}, + "queries": [ + { + "indexUid": "test", + "federationOptions": { + "queryPosition": "toto", + } + } + ]})) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Invalid value type at `.queries[0].federationOptions.queryPosition`: expected a positive integer, but found a string: `\"toto\"`", + "code": "invalid_multi_search_query_position", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_multi_search_query_position" + } + "###); +} + +#[actix_rt::test] +async fn remote_sharding() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + let ms2 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms2.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms2", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let index2 = ms2.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index2.add_documents(json!(documents[3..5]), None).await; + index2.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + let ms2 = Arc::new(ms2); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + let rms2 = LocalMeili::new(ms2.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + "ms2": { + "url": rms2.url() + } + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms2.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Badman", + "id": "E", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.5, + "remote": "ms2" + } + }, + { + "title": "Batman", + "id": "D", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.23106060606060605, + "remote": "ms2" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 5, + "remoteErrors": {} + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Badman", + "id": "E", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.5, + "remote": "ms2" + } + }, + { + "title": "Batman", + "id": "D", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.23106060606060605, + "remote": "ms2" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 5, + "remoteErrors": {} + } + "###); + let (response, _status_code) = ms2.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Badman", + "id": "E", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.5, + "remote": "ms2" + } + }, + { + "title": "Batman", + "id": "D", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.23106060606060605, + "remote": "ms2" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 5, + "remoteErrors": {} + } + "###); +} + +#[actix_rt::test] +async fn error_unregistered_remote() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "message": "Invalid `queries[2].federation_options.remote`: remote `ms2` is not registered", + "code": "invalid_multi_search_remote", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_multi_search_remote" + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "message": "Invalid `queries[2].federation_options.remote`: remote `ms2` is not registered", + "code": "invalid_multi_search_remote", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_multi_search_remote" + } + "###); +} + +#[actix_rt::test] +async fn error_no_weighted_score() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::with_params( + ms1.clone(), + LocalMeiliParams { gobble_headers: true, ..Default::default() }, + ) + .await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "remote hit does not contain `._federation.weightedScoreValues`\n - hint: check that the remote instance is a Meilisearch instance running the same version", + "code": "remote_bad_response", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_bad_response" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_bad_response() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::with_params( + ms1.clone(), + LocalMeiliParams { + override_response_body: Some("Returning an HTML page".into()), + ..Default::default() + }, + ) + .await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "could not parse response from the remote host as a federated search response:\n - response from remote: Returning an HTML page\n - hint: check that the remote instance is a Meilisearch instance running the same version", + "code": "remote_bad_response", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_bad_response" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_bad_request() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "nottest", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "remote host responded with code 400:\n - response from remote: {\"message\":\"Inside `.queries[1]`: Index `nottest` not found.\",\"code\":\"index_not_found\",\"type\":\"invalid_request\",\"link\":\"https://docs.meilisearch.com/errors#index_not_found\"}\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", + "code": "remote_bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#remote_bad_request" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_bad_request_facets_by_index() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test0"); + let index1 = ms1.index("test1"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": { + "facetsByIndex": { + "test0": [] + } + }, + "queries": [ + { + "q": query, + "indexUid": "test0", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test1", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test0", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test0", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "facetsByIndex": { + "test0": { + "distribution": {}, + "stats": {} + } + }, + "remoteErrors": { + "ms1": { + "message": "remote host responded with code 400:\n - response from remote: {\"message\":\"Inside `.federation.facetsByIndex.test0`: Index `test0` not found.\\n - Note: index `test0` is not used in queries\",\"code\":\"index_not_found\",\"type\":\"invalid_request\",\"link\":\"https://docs.meilisearch.com/errors#index_not_found\"}\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", + "code": "remote_bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#remote_bad_request" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_bad_request_facets_by_index_facet() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + + let (task, _status_code) = index0.update_settings_filterable_attributes(json!(["id"])).await; + index0.wait_task(task.uid()).await.succeeded(); + + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": { + "facetsByIndex": { + "test": ["id"] + } + }, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "facetsByIndex": { + "test": { + "distribution": { + "id": { + "A": 1, + "B": 1 + } + }, + "stats": {} + } + }, + "remoteErrors": { + "ms1": { + "message": "remote host responded with code 400:\n - response from remote: {\"message\":\"Inside `.federation.facetsByIndex.test`: Invalid facet distribution, this index does not have configured filterable attributes.\\n - Note: index `test` used in `.queries[1]`\",\"code\":\"invalid_multi_search_facets\",\"type\":\"invalid_request\",\"link\":\"https://docs.meilisearch.com/errors#invalid_multi_search_facets\"}\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", + "code": "remote_bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#remote_bad_request" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_remote_does_not_answer() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + "ms2": { + "url": "https://thiswebsitedoesnotexist.example" + } + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": { + "ms2": { + "message": "error sending request", + "code": "remote_could_not_send_request", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_could_not_send_request" + } + } + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": { + "ms2": { + "message": "error sending request", + "code": "remote_could_not_send_request", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_could_not_send_request" + } + } + } + "###); +} + +#[actix_rt::test] +async fn error_remote_404() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": format!("{}/this-route-does-not-exists/", rms1.url()) + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "remote host responded with code 404:\n - response from remote: null\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", + "code": "remote_bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#remote_bad_request" + } + } + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": {} + } + "###); +} + +#[actix_rt::test] +async fn error_remote_sharding_auth() { + let ms0 = Server::new().await; + let mut ms1 = Server::new_auth().await; + ms1.use_api_key("MASTER_KEY"); + + let (search_api_key_not_enough_indexes, code) = ms1 + .add_api_key(json!({ + "actions": ["search"], + "indexes": ["nottest"], + "expiresAt": serde_json::Value::Null + })) + .await; + meili_snap::snapshot!(code, @"201 Created"); + let search_api_key_not_enough_indexes = search_api_key_not_enough_indexes["key"].clone(); + + let (api_key_not_search, code) = ms1 + .add_api_key(json!({ + "actions": ["documents.*"], + "indexes": ["*"], + "expiresAt": serde_json::Value::Null + })) + .await; + meili_snap::snapshot!(code, @"201 Created"); + let api_key_not_search = api_key_not_search["key"].clone(); + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + ms1.clear_api_key(); + + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1-nottest": { + "url": rms1.url(), + "searchApiKey": search_api_key_not_enough_indexes + }, + "ms1-notsearch": { + "url": rms1.url(), + "searchApiKey": api_key_not_search + } + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1-nottest" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1-notsearch" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1-notsearch": { + "message": "could not authenticate against the remote host\n - hint: check that the remote instance was registered with a valid API key having the `search` action", + "code": "remote_invalid_api_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#remote_invalid_api_key" + }, + "ms1-nottest": { + "message": "could not authenticate against the remote host\n - hint: check that the remote instance was registered with a valid API key having the `search` action", + "code": "remote_invalid_api_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#remote_invalid_api_key" + } + } + } + "###); +} + +#[actix_rt::test] +async fn remote_sharding_auth() { + let ms0 = Server::new().await; + let mut ms1 = Server::new_auth().await; + ms1.use_api_key("MASTER_KEY"); + + let (search_api_key, code) = ms1 + .add_api_key(json!({ + "actions": ["search"], + "indexes": ["*"], + "expiresAt": serde_json::Value::Null + })) + .await; + meili_snap::snapshot!(code, @"201 Created"); + let search_api_key = search_api_key["key"].clone(); + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + ms1.clear_api_key(); + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url(), + "searchApiKey": "MASTER_KEY" + }, + "ms1-alias": { + "url": rms1.url(), + "searchApiKey": search_api_key + } + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1-alias" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 2, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1-alias" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4, + "remoteErrors": {} + } + "###); +} + +#[actix_rt::test] +async fn error_remote_500() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::with_params( + ms1.clone(), + LocalMeiliParams { fails: FailurePolicy::Always, ..Default::default() }, + ) + .await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + } + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "remote host responded with code 500:\n - response from remote: {\"error\":\"provoked error\",\"code\":\"test_error\",\"link\":\"https://docs.meilisearch.com/errors#test_error\"}", + "code": "remote_remote_error", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_remote_error" + } + } + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + // the response if full because we queried the instance that works + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": {} + } + "###); +} + +#[actix_rt::test] +async fn error_remote_500_once() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::with_params( + ms1.clone(), + LocalMeiliParams { fails: FailurePolicy::Once, ..Default::default() }, + ) + .await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + } + ] + }); + + // Meilisearch is tolerant to a single failure + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": {} + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": {} + } + "###); +} + +#[actix_rt::test] +async fn error_remote_timeout() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + + // add documents + let documents = SCORE_DOCUMENTS.clone(); + let documents = documents.as_array().unwrap(); + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let (task, _status_code) = index0.add_documents(json!(documents[0..2]), None).await; + index0.wait_task(task.uid()).await.succeeded(); + let (task, _status_code) = index1.add_documents(json!(documents[2..3]), None).await; + index1.wait_task(task.uid()).await.succeeded(); + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::with_params( + ms1.clone(), + LocalMeiliParams { delay: Some(std::time::Duration::from_secs(6)), ..Default::default() }, + ) + .await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + }}); + + println!("{}", serde_json::to_string_pretty(&network).unwrap()); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "federationOptions": { + "remote": "ms1" + } + } + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "remoteErrors": { + "ms1": { + "message": "remote host did not answer before the deadline", + "code": "remote_timeout", + "type": "system", + "link": "https://docs.meilisearch.com/errors#remote_timeout" + } + } + } + "###); + let (response, _status_code) = ms1.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r###" + { + "hits": [ + { + "title": "Batman Returns", + "id": "C", + "_federation": { + "indexUid": "test", + "queriesPosition": 1, + "weightedRankingScore": 0.8317901234567902, + "remote": "ms1" + } + }, + { + "title": "Batman the dark knight returns: Part 1", + "id": "A", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + }, + { + "title": "Batman the dark knight returns: Part 2", + "id": "B", + "_federation": { + "indexUid": "test", + "queriesPosition": 0, + "weightedRankingScore": 0.7028218694885362, + "remote": "ms0" + } + } + ], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "remoteErrors": {} + } + "###); +} + +// test: try all the flattened structs in queries + +// working facet tests with and without merge + +#[derive(Default)] +pub enum FailurePolicy { + #[default] + Never, + Once, + Always, +} + +/// Parameters to change the behavior of the [`LocalMeili`] server. +#[derive(Default)] +pub struct LocalMeiliParams { + /// delay the response by the specified duration + pub delay: Option, + pub fails: FailurePolicy, + /// replace the reponse body with the provided String + pub override_response_body: Option, + pub gobble_headers: bool, +} + +/// A server that exploits [`MockServer`] to provide an URL for testing network and the network. +pub struct LocalMeili { + mock_server: MockServer, +} + +impl LocalMeili { + pub async fn new(server: Arc) -> Self { + Self::with_params(server, Default::default()).await + } + + pub async fn with_params(server: Arc, params: LocalMeiliParams) -> Self { + let mock_server = MockServer::start().await; + + // tokio won't let us execute asynchronous code from a sync function inside of an async test, + // so instead we spawn another thread that will call the service on a brand new tokio runtime + // and communicate via channels... + let (request_sender, request_receiver) = crossbeam_channel::bounded::(0); + let (response_sender, response_receiver) = + crossbeam_channel::bounded::<(Value, StatusCode)>(0); + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); + while let Ok(req) = request_receiver.recv() { + let body = std::str::from_utf8(&req.body).unwrap(); + let headers: Vec<(&str, &str)> = if params.gobble_headers { + vec![("Content-Type", "application/json")] + } else { + req.headers + .iter() + .map(|(name, value)| (name.as_str(), value.to_str().unwrap())) + .collect() + }; + let (value, code) = rt.block_on(async { + match req.method.as_str() { + "POST" => server.service.post_str(&req.url, body, headers.clone()).await, + "PUT" => server.service.put_str(&req.url, body, headers).await, + "PATCH" => server.service.patch(&req.url, req.body_json().unwrap()).await, + "GET" => server.service.get(&req.url).await, + "DELETE" => server.service.delete(&req.url).await, + _ => unimplemented!(), + } + }); + if response_sender.send((value, code)).is_err() { + break; + } + } + println!("exiting mock thread") + }); + + let failed_already = std::sync::atomic::AtomicBool::new(false); + + Mock::given(AnyMatcher) + .respond_with(move |req: &wiremock::Request| { + if let Some(delay) = params.delay { + std::thread::sleep(delay); + } + match params.fails { + FailurePolicy::Never => {} + FailurePolicy::Once => { + let failed_already = + failed_already.fetch_or(true, std::sync::atomic::Ordering::AcqRel); + if !failed_already { + return fail(params.override_response_body.as_deref()); + } + } + FailurePolicy::Always => return fail(params.override_response_body.as_deref()), + } + request_sender.send(req.clone()).unwrap(); + let (value, code) = response_receiver.recv().unwrap(); + let response = ResponseTemplate::new(code.as_u16()); + if let Some(override_response_body) = params.override_response_body.as_deref() { + response.set_body_string(override_response_body) + } else { + response.set_body_json(value) + } + }) + .mount(&mock_server) + .await; + Self { mock_server } + } + + pub fn url(&self) -> String { + self.mock_server.uri() + } +} + +fn fail(override_response_body: Option<&str>) -> ResponseTemplate { + let response = ResponseTemplate::new(500); + if let Some(override_response_body) = override_response_body { + response.set_body_string(override_response_body) + } else { + response.set_body_json(json!({"error": "provoked error", "code": "test_error", "link": "https://docs.meilisearch.com/errors#test_error"})) + } +}