From 9dbc71cb6da17c7f00964442793705778e068475 Mon Sep 17 00:00:00 2001 From: "Andrey \"MOU\" Larionov" Date: Sun, 9 Oct 2022 21:55:14 +0200 Subject: [PATCH] Added support for encoded payload Actix provides different content encodings out of the box, but only if we use built-in content wrappers and containers. This patch wraps its own Payload implementation with an actix decoder, which enables request compression support. --- meilisearch-http/src/extractors/payload.rs | 5 +- .../tests/documents/add_documents.rs | 93 +++++++++++++++++++ .../tests/documents/update_documents.rs | 42 +++++++++ 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/extractors/payload.rs b/meilisearch-http/src/extractors/payload.rs index 6cd8df190..f16fdd67b 100644 --- a/meilisearch-http/src/extractors/payload.rs +++ b/meilisearch-http/src/extractors/payload.rs @@ -1,13 +1,14 @@ use std::pin::Pin; use std::task::{Context, Poll}; +use actix_http::encoding::Decoder as Decompress; use actix_web::error::PayloadError; use actix_web::{dev, web, FromRequest, HttpRequest}; use futures::future::{ready, Ready}; use futures::Stream; pub struct Payload { - payload: dev::Payload, + payload: Decompress, limit: usize, } @@ -39,7 +40,7 @@ impl FromRequest for Payload { .map(|c| c.limit) .unwrap_or(PayloadConfig::default().limit); ready(Ok(Payload { - payload: payload.take(), + payload: Decompress::from_headers(payload.take(), req.headers()), limit, })) } diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index f2126e6b4..f4cb3d36f 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -4,6 +4,7 @@ use actix_web::test; use meilisearch_http::{analytics, create_app}; use serde_json::{json, Value}; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; +use crate::common::encoder::Encoder; /// This is the basic usage of our API and every other tests uses the content-type application/json #[actix_rt::test] @@ -97,6 +98,98 @@ async fn add_single_document_test_json_content_types() { assert_eq!(response["taskUid"], 1); } +/// Here we try sending encoded (compressed) document request +#[actix_rt::test] +async fn add_single_document_gzip_encoded() { + let document = json!({ + "id": 1, + "content": "Bouvier Bernois", + }); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + // post + let document = serde_json::to_string(&document).unwrap(); + let encoder = Encoder::Gzip; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document.clone())) + .insert_header(("content-type", "application/json")) + .insert_header(encoder.header().unwrap()) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], 0); + + // put + let req = test::TestRequest::put() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document)) + .insert_header(("content-type", "application/json")) + .insert_header(encoder.header().unwrap()) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], 1); +} + +/// Here we try document request with every encoding +#[actix_rt::test] +async fn add_single_document_with_every_encoding() { + let document = json!({ + "id": 1, + "content": "Bouvier Bernois", + }); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + // post + let mut task_uid = 0; + let document = serde_json::to_string(&document).unwrap(); + + for encoder in Encoder::iterator() { + let mut req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document.clone())) + .insert_header(("content-type", "application/json")); + req = match encoder.header() { + Some(header) => req.insert_header(header), + None => req + }; + let req = req.to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], task_uid); + task_uid += 1; + } + +} + /// any other content-type is must be refused #[actix_rt::test] async fn error_add_documents_test_bad_content_types() { diff --git a/meilisearch-http/tests/documents/update_documents.rs b/meilisearch-http/tests/documents/update_documents.rs index d3d56b444..da932ed26 100644 --- a/meilisearch-http/tests/documents/update_documents.rs +++ b/meilisearch-http/tests/documents/update_documents.rs @@ -1,6 +1,7 @@ use crate::common::{GetAllDocumentsOptions, Server}; use serde_json::json; +use crate::common::encoder::Encoder; #[actix_rt::test] async fn error_document_update_create_index_bad_uid() { @@ -89,6 +90,47 @@ async fn update_document() { ); } +#[actix_rt::test] +async fn update_document_gzip_encoded() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + + index.wait_task(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_task(1).await; + + let (response, code) = index.get_task(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "succeeded"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); +} + #[actix_rt::test] async fn update_larger_dataset() { let server = Server::new().await;