Merge #2876

2876: Full support for compressed (Gzip, Brotli, Zlib) API requests r=Kerollmops a=mou

# Pull Request

## Related issue
Fixes #2802 

## What does this PR do?
- Adds missed content-encoding support for streamed requests (documents)
- Adds additional tests to validate content-encoding support is in place
- Adds new tests to validate content-encoding support for previously existing code (built-in actix functionality for unmarshaling JSON payloads)

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!

Co-authored-by: Andrey "MOU" Larionov <anlarionov@gmail.com>

Cargo.lock

@@ -2038,6 +2038,7 @@ name = "meilisearch-http"
 version = "0.29.1"
 dependencies = [
  "actix-cors",
+ "actix-http",
  "actix-rt",
  "actix-web",
  "actix-web-static-files",
@@ -2045,6 +2046,7 @@ dependencies = [
  "assert-json-diff",
  "async-stream",
  "async-trait",
+ "brotli",
  "bstr 1.0.1",
  "byte-unit",
  "bytes",

meilisearch-http/Cargo.toml

@@ -24,6 +24,7 @@ zip = { version = "0.6.2", optional = true }
 [dependencies]
 actix-cors = "0.6.3"
 actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] }
+actix-http = "3.2.2"
 actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
 anyhow = { version = "1.0.65", features = ["backtrace"] }
 async-stream = "0.3.3"
@@ -85,6 +86,7 @@ lazy_static = "1.4.0"
 [dev-dependencies]
 actix-rt = "2.7.0"
 assert-json-diff = "2.0.2"
+brotli = "3.3.4"
 manifest-dir-macros = "0.1.16"
 maplit = "1.0.2"
 urlencoding = "2.1.2"

meilisearch-http/src/extractors/payload.rs

@@ -1,13 +1,14 @@
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
+use actix_http::encoding::Decoder as Decompress;
 use actix_web::error::PayloadError;
 use actix_web::{dev, web, FromRequest, HttpRequest};
 use futures::future::{ready, Ready};
 use futures::Stream;
 
 pub struct Payload {
-    payload: dev::Payload,
+    payload: Decompress<dev::Payload>,
     limit: usize,
 }
 
@@ -39,7 +40,7 @@ impl FromRequest for Payload {
             .map(|c| c.limit)
             .unwrap_or(PayloadConfig::default().limit);
         ready(Ok(Payload {
-            payload: payload.take(),
+            payload: Decompress::from_headers(payload.take(), req.headers()),
             limit,
         }))
     }

meilisearch-http/tests/common/encoder.rs

@@ -0,0 +1,87 @@
+use actix_http::header::TryIntoHeaderPair;
+use bytes::Bytes;
+use flate2::read::{GzDecoder, ZlibDecoder};
+use flate2::write::{GzEncoder, ZlibEncoder};
+use flate2::Compression;
+use std::io::{Read, Write};
+
+#[derive(Clone, Copy)]
+pub enum Encoder {
+    Plain,
+    Gzip,
+    Deflate,
+    Brotli,
+}
+
+impl Encoder {
+    pub fn encode(self: &Encoder, body: impl Into<Bytes>) -> impl Into<Bytes> {
+        match self {
+            Self::Gzip => {
+                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+                encoder
+                    .write_all(&body.into())
+                    .expect("Failed to encode request body");
+                encoder.finish().expect("Failed to encode request body")
+            }
+            Self::Deflate => {
+                let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
+                encoder
+                    .write_all(&body.into())
+                    .expect("Failed to encode request body");
+                encoder.finish().unwrap()
+            }
+            Self::Plain => Vec::from(body.into()),
+            Self::Brotli => {
+                let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22);
+                encoder
+                    .write_all(&body.into())
+                    .expect("Failed to encode request body");
+                encoder.flush().expect("Failed to encode request body");
+                encoder.into_inner()
+            }
+        }
+    }
+
+    pub fn decode(self: &Encoder, bytes: impl Into<Bytes>) -> impl Into<Bytes> {
+        let mut buffer = Vec::new();
+        let input = bytes.into();
+        match self {
+            Self::Gzip => {
+                GzDecoder::new(input.as_ref())
+                    .read_to_end(&mut buffer)
+                    .expect("Invalid gzip stream");
+            }
+            Self::Deflate => {
+                ZlibDecoder::new(input.as_ref())
+                    .read_to_end(&mut buffer)
+                    .expect("Invalid zlib stream");
+            }
+            Self::Plain => {
+                buffer
+                    .write_all(input.as_ref())
+                    .expect("Unexpected memory copying issue");
+            }
+            Self::Brotli => {
+                brotli::Decompressor::new(input.as_ref(), 4096)
+                    .read_to_end(&mut buffer)
+                    .expect("Invalid brotli stream");
+            }
+        };
+        buffer
+    }
+
+    pub fn header(self: &Encoder) -> Option<impl TryIntoHeaderPair> {
+        match self {
+            Self::Plain => None,
+            Self::Gzip => Some(("Content-Encoding", "gzip")),
+            Self::Deflate => Some(("Content-Encoding", "deflate")),
+            Self::Brotli => Some(("Content-Encoding", "br")),
+        }
+    }
+
+    pub fn iterator() -> impl Iterator<Item = Self> {
+        [Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli]
+            .iter()
+            .copied()
+    }
+}

meilisearch-http/tests/common/index.rs

@@ -7,24 +7,27 @@ use std::{
 use actix_web::http::StatusCode;
 use serde_json::{json, Value};
 use tokio::time::sleep;
-use urlencoding::encode;
+use urlencoding::encode as urlencode;
 
 use super::service::Service;
 
+use super::encoder::Encoder;
+
 pub struct Index<'a> {
     pub uid: String,
     pub service: &'a Service,
+    pub encoder: Encoder,
 }
 
 #[allow(dead_code)]
 impl Index<'_> {
     pub async fn get(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
     pub async fn load_test_set(&self) -> u64 {
-        let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
         let (response, code) = self
             .service
             .post_str(url, include_str!("../assets/test_set.json"))
@@ -40,20 +43,22 @@ impl Index<'_> {
             "uid": self.uid,
             "primaryKey": primary_key,
         });
-        self.service.post("/indexes", body).await
+        self.service
+            .post_encoded("/indexes", body, self.encoder)
+            .await
     }
 
     pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
         let body = json!({
             "primaryKey": primary_key,
         });
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
 
-        self.service.patch(url, body).await
+        self.service.patch_encoded(url, body, self.encoder).await
     }
 
     pub async fn delete(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
@@ -65,12 +70,14 @@ impl Index<'_> {
         let url = match primary_key {
             Some(key) => format!(
                 "/indexes/{}/documents?primaryKey={}",
-                encode(self.uid.as_ref()),
+                urlencode(self.uid.as_ref()),
                 key
             ),
-            None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
         };
-        self.service.post(url, documents).await
+        self.service
+            .post_encoded(url, documents, self.encoder)
+            .await
     }
 
     pub async fn update_documents(
@@ -81,12 +88,12 @@ impl Index<'_> {
         let url = match primary_key {
             Some(key) => format!(
                 "/indexes/{}/documents?primaryKey={}",
-                encode(self.uid.as_ref()),
+                urlencode(self.uid.as_ref()),
                 key
             ),
-            None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
         };
-        self.service.put(url, documents).await
+        self.service.put_encoded(url, documents, self.encoder).await
     }
 
     pub async fn wait_task(&self, update_id: u64) -> Value {
@@ -132,7 +139,7 @@ impl Index<'_> {
         id: u64,
         options: Option<GetDocumentOptions>,
     ) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         if let Some(fields) = options.and_then(|o| o.fields) {
             let _ = write!(url, "?fields={}", fields.join(","));
         }
@@ -140,7 +147,7 @@ impl Index<'_> {
     }
 
     pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref()));
+        let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref()));
         if let Some(limit) = options.limit {
             let _ = write!(url, "limit={}&", limit);
         }
@@ -157,42 +164,44 @@ impl Index<'_> {
     }
 
     pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         self.service.delete(url).await
     }
 
     pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/documents/delete-batch",
-            encode(self.uid.as_ref())
+            urlencode(self.uid.as_ref())
         );
         self.service
-            .post(url, serde_json::to_value(&ids).unwrap())
+            .post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder)
             .await
     }
 
     pub async fn settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
     pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.patch(url, settings).await
+        self.service
+            .patch_encoded(url, settings, self.encoder)
+            .await
     }
 
     pub async fn delete_settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
     pub async fn stats(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
@@ -217,29 +226,33 @@ impl Index<'_> {
     }
 
     pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/search", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref()));
-        self.service.post(url, query).await
+        self.service.post_encoded(url, query, self.encoder).await
     }
 
     pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
         let params = yaup::to_string(&query).unwrap();
-        let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params);
+        let url = format!(
+            "/indexes/{}/search?{}",
+            urlencode(self.uid.as_ref()),
+            params
+        );
         self.service.get(url).await
     }
 
     pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/settings/{}",
-            encode(self.uid.as_ref()),
+            urlencode(self.uid.as_ref()),
             "distinct-attribute"
         );
-        self.service.put(url, value).await
+        self.service.put_encoded(url, value, self.encoder).await
     }
 
     pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/settings/{}",
-            encode(self.uid.as_ref()),
+            urlencode(self.uid.as_ref()),
             "distinct-attribute"
         );
         self.service.get(url).await

meilisearch-http/tests/common/mod.rs

@@ -1,3 +1,4 @@
+pub mod encoder;
 pub mod index;
 pub mod server;
 pub mod service;

meilisearch-http/tests/common/server.rs

@@ -12,6 +12,7 @@ use once_cell::sync::Lazy;
 use serde_json::Value;
 use tempfile::TempDir;
 
+use crate::common::encoder::Encoder;
 use meilisearch_http::option::Opt;
 
 use super::index::Index;
@@ -100,9 +101,14 @@ impl Server {
 
     /// Returns a view to an index. There is no guarantee that the index exists.
     pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
+        self.index_with_encoder(uid, Encoder::Plain)
+    }
+
+    pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
         Index {
             uid: uid.as_ref().to_string(),
             service: &self.service,
+            encoder,
         }
     }
 

meilisearch-http/tests/common/service.rs

@@ -1,8 +1,11 @@
+use actix_web::http::header::ContentType;
+use actix_web::test::TestRequest;
 use actix_web::{http::StatusCode, test};
 use meilisearch_auth::AuthController;
 use meilisearch_lib::MeiliSearch;
 use serde_json::Value;
 
+use crate::common::encoder::Encoder;
 use meilisearch_http::{analytics, create_app, Opt};
 
 pub struct Service {
@@ -14,26 +17,18 @@ pub struct Service {
 
 impl Service {
     pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.post_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
+    }
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
 
-        let mut req = test::TestRequest::post().uri(url.as_ref()).set_json(&body);
+    pub async fn post_encoded(
-        if let Some(api_key) = &self.api_key {
+        &self,
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
+        url: impl AsRef<str>,
-        }
+        body: Value,
-        let req = req.to_request();
+        encoder: Encoder,
-        let res = test::call_service(&app, req).await;
+    ) -> (Value, StatusCode) {
-        let status_code = res.status();
+        let mut req = test::TestRequest::post().uri(url.as_ref());
-
+        req = self.encode(req, body, encoder);
-        let body = test::read_body(res).await;
+        self.request(req).await
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     /// Send a test post request from a text body, with a `content-type:application/json` header.
@@ -42,101 +37,54 @@ impl Service {
         url: impl AsRef<str>,
         body: impl AsRef<str>,
     ) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        let req = test::TestRequest::post()
-            &self.meilisearch,
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::post()
             .uri(url.as_ref())
             .set_payload(body.as_ref().to_string())
             .insert_header(("content-type", "application/json"));
-        if let Some(api_key) = &self.api_key {
+        self.request(req).await
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
-        }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
-
-        let body = test::read_body(res).await;
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        let req = test::TestRequest::get().uri(url.as_ref());
-            &self.meilisearch,
+        self.request(req).await
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::get().uri(url.as_ref());
-        if let Some(api_key) = &self.api_key {
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
-        }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
-
-        let body = test::read_body(res).await;
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.put_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
+    }
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
 
-        let mut req = test::TestRequest::put().uri(url.as_ref()).set_json(&body);
+    pub async fn put_encoded(
-        if let Some(api_key) = &self.api_key {
+        &self,
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
+        url: impl AsRef<str>,
-        }
+        body: Value,
-        let req = req.to_request();
+        encoder: Encoder,
-        let res = test::call_service(&app, req).await;
+    ) -> (Value, StatusCode) {
-        let status_code = res.status();
+        let mut req = test::TestRequest::put().uri(url.as_ref());
-
+        req = self.encode(req, body, encoder);
-        let body = test::read_body(res).await;
+        self.request(req).await
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn patch(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.patch_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
+    }
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
 
-        let mut req = test::TestRequest::patch().uri(url.as_ref()).set_json(&body);
+    pub async fn patch_encoded(
-        if let Some(api_key) = &self.api_key {
+        &self,
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
+        url: impl AsRef<str>,
-        }
+        body: Value,
-        let req = req.to_request();
+        encoder: Encoder,
-        let res = test::call_service(&app, req).await;
+    ) -> (Value, StatusCode) {
-        let status_code = res.status();
+        let mut req = test::TestRequest::patch().uri(url.as_ref());
-
+        req = self.encode(req, body, encoder);
-        let body = test::read_body(res).await;
+        self.request(req).await
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
+        let req = test::TestRequest::delete().uri(url.as_ref());
+        self.request(req).await
+    }
+
+    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
         let app = test::init_service(create_app!(
             &self.meilisearch,
             &self.auth,
@@ -146,7 +94,6 @@ impl Service {
         ))
         .await;
 
-        let mut req = test::TestRequest::delete().uri(url.as_ref());
         if let Some(api_key) = &self.api_key {
             req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
         }
@@ -158,4 +105,16 @@ impl Service {
         let response = serde_json::from_slice(&body).unwrap_or_default();
         (response, status_code)
     }
+
+    fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest {
+        let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json");
+        let encoded_body = encoder.encode(bytes);
+        let header = encoder.header();
+        match header {
+            Some(header) => req.insert_header(header),
+            None => req,
+        }
+        .set_payload(encoded_body)
+        .insert_header(ContentType::json())
+    }
 }

meilisearch-http/tests/documents/add_documents.rs

@@ -1,6 +1,7 @@
 use crate::common::{GetAllDocumentsOptions, Server};
 use actix_web::test;
 
+use crate::common::encoder::Encoder;
 use meilisearch_http::{analytics, create_app};
 use serde_json::{json, Value};
 use time::{format_description::well_known::Rfc3339, OffsetDateTime};
@@ -97,6 +98,95 @@ async fn add_single_document_test_json_content_types() {
     assert_eq!(response["taskUid"], 1);
 }
 
+/// Here we try sending encoded (compressed) document request
+#[actix_rt::test]
+async fn add_single_document_gzip_encoded() {
+    let document = json!({
+        "id": 1,
+        "content": "Bouvier Bernois",
+    });
+
+    // this is a what is expected and should work
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        &server.service.auth,
+        true,
+        server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let document = serde_json::to_string(&document).unwrap();
+    let encoder = Encoder::Gzip;
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(encoder.encode(document.clone()))
+        .insert_header(("content-type", "application/json"))
+        .insert_header(encoder.header().unwrap())
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 202);
+    assert_eq!(response["taskUid"], 0);
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(encoder.encode(document))
+        .insert_header(("content-type", "application/json"))
+        .insert_header(encoder.header().unwrap())
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 202);
+    assert_eq!(response["taskUid"], 1);
+}
+
+/// Here we try document request with every encoding
+#[actix_rt::test]
+async fn add_single_document_with_every_encoding() {
+    let document = json!({
+        "id": 1,
+        "content": "Bouvier Bernois",
+    });
+
+    // this is a what is expected and should work
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        &server.service.auth,
+        true,
+        server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let document = serde_json::to_string(&document).unwrap();
+
+    for (task_uid, encoder) in Encoder::iterator().enumerate() {
+        let mut req = test::TestRequest::post()
+            .uri("/indexes/dog/documents")
+            .set_payload(encoder.encode(document.clone()))
+            .insert_header(("content-type", "application/json"));
+        req = match encoder.header() {
+            Some(header) => req.insert_header(header),
+            None => req,
+        };
+        let req = req.to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+        let body = test::read_body(res).await;
+        let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+        assert_eq!(status_code, 202);
+        assert_eq!(response["taskUid"], task_uid);
+    }
+}
+
 /// any other content-type is must be refused
 #[actix_rt::test]
 async fn error_add_documents_test_bad_content_types() {
@@ -690,23 +780,6 @@ async fn error_document_add_create_index_bad_uid() {
     assert_eq!(response, expected_response);
 }
 
-#[actix_rt::test]
-async fn error_document_update_create_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("883  fj!");
-    let (response, code) = index.update_documents(json!([{"id": 1}]), None).await;
-
-    let expected_response = json!({
-        "message": "invalid index uid `883  fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.",
-        "code": "invalid_index_uid",
-        "type": "invalid_request",
-        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
-    });
-
-    assert_eq!(code, 400);
-    assert_eq!(response, expected_response);
-}
-
 #[actix_rt::test]
 async fn document_addition_with_primary_key() {
     let server = Server::new().await;
@@ -736,35 +809,6 @@ async fn document_addition_with_primary_key() {
     assert_eq!(response["primaryKey"], "primary");
 }
 
-#[actix_rt::test]
-async fn document_update_with_primary_key() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = json!([
-        {
-            "primary": 1,
-            "content": "foo",
-        }
-    ]);
-    let (_response, code) = index.update_documents(documents, Some("primary")).await;
-    assert_eq!(code, 202);
-
-    index.wait_task(0).await;
-
-    let (response, code) = index.get_task(0).await;
-    assert_eq!(code, 200);
-    assert_eq!(response["status"], "succeeded");
-    assert_eq!(response["uid"], 0);
-    assert_eq!(response["type"], "documentAdditionOrUpdate");
-    assert_eq!(response["details"]["indexedDocuments"], 1);
-    assert_eq!(response["details"]["receivedDocuments"], 1);
-
-    let (response, code) = index.get().await;
-    assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], "primary");
-}
-
 #[actix_rt::test]
 async fn replace_document() {
     let server = Server::new().await;
@@ -811,47 +855,6 @@ async fn add_no_documents() {
     assert_eq!(code, 202);
 }
 
-#[actix_rt::test]
-async fn update_document() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = json!([
-        {
-            "doc_id": 1,
-            "content": "foo",
-        }
-    ]);
-
-    let (_response, code) = index.add_documents(documents, None).await;
-    assert_eq!(code, 202);
-
-    index.wait_task(0).await;
-
-    let documents = json!([
-        {
-            "doc_id": 1,
-            "other": "bar",
-        }
-    ]);
-
-    let (response, code) = index.update_documents(documents, None).await;
-    assert_eq!(code, 202, "response: {}", response);
-
-    index.wait_task(1).await;
-
-    let (response, code) = index.get_task(1).await;
-    assert_eq!(code, 200);
-    assert_eq!(response["status"], "succeeded");
-
-    let (response, code) = index.get_document(1, None).await;
-    assert_eq!(code, 200);
-    assert_eq!(
-        response.to_string(),
-        r##"{"doc_id":1,"content":"foo","other":"bar"}"##
-    );
-}
-
 #[actix_rt::test]
 async fn add_larger_dataset() {
     let server = Server::new().await;
@@ -873,27 +876,6 @@ async fn add_larger_dataset() {
     assert_eq!(response["results"].as_array().unwrap().len(), 77);
 }
 
-#[actix_rt::test]
-async fn update_larger_dataset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap();
-    index.update_documents(documents, None).await;
-    index.wait_task(0).await;
-    let (response, code) = index.get_task(0).await;
-    assert_eq!(code, 200);
-    assert_eq!(response["type"], "documentAdditionOrUpdate");
-    assert_eq!(response["details"]["indexedDocuments"], 77);
-    let (response, code) = index
-        .get_all_documents(GetAllDocumentsOptions {
-            limit: Some(1000),
-            ..Default::default()
-        })
-        .await;
-    assert_eq!(code, 200);
-    assert_eq!(response["results"].as_array().unwrap().len(), 77);
-}
-
 #[actix_rt::test]
 async fn error_add_documents_bad_document_id() {
     let server = Server::new().await;
@@ -924,34 +906,6 @@ async fn error_add_documents_bad_document_id() {
     );
 }
 
-#[actix_rt::test]
-async fn error_update_documents_bad_document_id() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.create(Some("docid")).await;
-    let documents = json!([
-        {
-            "docid": "foo & bar",
-            "content": "foobar"
-        }
-    ]);
-    index.update_documents(documents, None).await;
-    let response = index.wait_task(1).await;
-    assert_eq!(response["status"], json!("failed"));
-    assert_eq!(
-        response["error"]["message"],
-        json!(
-            r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."#
-        )
-    );
-    assert_eq!(response["error"]["code"], json!("invalid_document_id"));
-    assert_eq!(response["error"]["type"], json!("invalid_request"));
-    assert_eq!(
-        response["error"]["link"],
-        json!("https://docs.meilisearch.com/errors#invalid_document_id")
-    );
-}
-
 #[actix_rt::test]
 async fn error_add_documents_missing_document_id() {
     let server = Server::new().await;
@@ -980,32 +934,6 @@ async fn error_add_documents_missing_document_id() {
     );
 }
 
-#[actix_rt::test]
-async fn error_update_documents_missing_document_id() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.create(Some("docid")).await;
-    let documents = json!([
-        {
-            "id": "11",
-            "content": "foobar"
-        }
-    ]);
-    index.update_documents(documents, None).await;
-    let response = index.wait_task(1).await;
-    assert_eq!(response["status"], "failed");
-    assert_eq!(
-        response["error"]["message"],
-        r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#
-    );
-    assert_eq!(response["error"]["code"], "missing_document_id");
-    assert_eq!(response["error"]["type"], "invalid_request");
-    assert_eq!(
-        response["error"]["link"],
-        "https://docs.meilisearch.com/errors#missing_document_id"
-    );
-}
-
 #[actix_rt::test]
 #[ignore] // // TODO: Fix in an other PR: this does not provoke any error.
 async fn error_document_field_limit_reached() {

meilisearch-http/tests/documents/get_documents.rs

@@ -1,6 +1,11 @@
 use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
+use actix_web::test;
+use http::header::ACCEPT_ENCODING;
 
-use serde_json::json;
+use crate::common::encoder::Encoder;
+use meilisearch_http::{analytics, create_app};
+use serde_json::{json, Value};
+use urlencoding::encode as urlencode;
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
 // transplant
@@ -155,6 +160,40 @@ async fn get_all_documents_no_options() {
     assert_eq!(first, arr[0]);
 }
 
+#[actix_rt::test]
+async fn get_all_documents_no_options_with_response_compression() {
+    let server = Server::new().await;
+    let index_uid = "test";
+    let index = server.index(index_uid);
+    index.load_test_set().await;
+
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        &server.service.auth,
+        true,
+        server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+
+    let req = test::TestRequest::get()
+        .uri(&format!("/indexes/{}/documents?", urlencode(index_uid)))
+        .insert_header((ACCEPT_ENCODING, "gzip"))
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+
+    assert_eq!(res.status(), 200);
+
+    let bytes = test::read_body(res).await;
+    let decoded = Encoder::Gzip.decode(bytes);
+    let parsed_response =
+        serde_json::from_slice::<Value>(decoded.into().as_ref()).expect("Expecting valid json");
+
+    let arr = parsed_response["results"].as_array().unwrap();
+    assert_eq!(arr.len(), 20);
+}
+
 #[actix_rt::test]
 async fn test_get_all_documents_limit() {
     let server = Server::new().await;

meilisearch-http/tests/documents/mod.rs

@@ -1,3 +1,4 @@
 mod add_documents;
 mod delete_documents;
 mod get_documents;
+mod update_documents;

meilisearch-http/tests/documents/update_documents.rs

@@ -0,0 +1,207 @@
+use crate::common::{GetAllDocumentsOptions, Server};
+
+use crate::common::encoder::Encoder;
+use serde_json::json;
+
+#[actix_rt::test]
+async fn error_document_update_create_index_bad_uid() {
+    let server = Server::new().await;
+    let index = server.index("883  fj!");
+    let (response, code) = index.update_documents(json!([{"id": 1}]), None).await;
+
+    let expected_response = json!({
+        "message": "invalid index uid `883  fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.",
+        "code": "invalid_index_uid",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
+    });
+
+    assert_eq!(code, 400);
+    assert_eq!(response, expected_response);
+}
+
+#[actix_rt::test]
+async fn document_update_with_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "primary": 1,
+            "content": "foo",
+        }
+    ]);
+    let (_response, code) = index.update_documents(documents, Some("primary")).await;
+    assert_eq!(code, 202);
+
+    index.wait_task(0).await;
+
+    let (response, code) = index.get_task(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["uid"], 0);
+    assert_eq!(response["type"], "documentAdditionOrUpdate");
+    assert_eq!(response["details"]["indexedDocuments"], 1);
+    assert_eq!(response["details"]["receivedDocuments"], 1);
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "primary");
+}
+
+#[actix_rt::test]
+async fn update_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (_response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+
+    index.wait_task(0).await;
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "other": "bar",
+        }
+    ]);
+
+    let (response, code) = index.update_documents(documents, None).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_task(1).await;
+
+    let (response, code) = index.get_task(1).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "succeeded");
+
+    let (response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response.to_string(),
+        r##"{"doc_id":1,"content":"foo","other":"bar"}"##
+    );
+}
+
+#[actix_rt::test]
+async fn update_document_gzip_encoded() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Gzip);
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (_response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+
+    index.wait_task(0).await;
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "other": "bar",
+        }
+    ]);
+
+    let (response, code) = index.update_documents(documents, None).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_task(1).await;
+
+    let (response, code) = index.get_task(1).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "succeeded");
+
+    let (response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response.to_string(),
+        r##"{"doc_id":1,"content":"foo","other":"bar"}"##
+    );
+}
+
+#[actix_rt::test]
+async fn update_larger_dataset() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap();
+    index.update_documents(documents, None).await;
+    index.wait_task(0).await;
+    let (response, code) = index.get_task(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["type"], "documentAdditionOrUpdate");
+    assert_eq!(response["details"]["indexedDocuments"], 77);
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            limit: Some(1000),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response["results"].as_array().unwrap().len(), 77);
+}
+
+#[actix_rt::test]
+async fn error_update_documents_bad_document_id() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "docid": "foo & bar",
+            "content": "foobar"
+        }
+    ]);
+    index.update_documents(documents, None).await;
+    let response = index.wait_task(1).await;
+    assert_eq!(response["status"], json!("failed"));
+    assert_eq!(
+        response["error"]["message"],
+        json!(
+            r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."#
+        )
+    );
+    assert_eq!(response["error"]["code"], json!("invalid_document_id"));
+    assert_eq!(response["error"]["type"], json!("invalid_request"));
+    assert_eq!(
+        response["error"]["link"],
+        json!("https://docs.meilisearch.com/errors#invalid_document_id")
+    );
+}
+
+#[actix_rt::test]
+async fn error_update_documents_missing_document_id() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "id": "11",
+            "content": "foobar"
+        }
+    ]);
+    index.update_documents(documents, None).await;
+    let response = index.wait_task(1).await;
+    assert_eq!(response["status"], "failed");
+    assert_eq!(
+        response["error"]["message"],
+        r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#
+    );
+    assert_eq!(response["error"]["code"], "missing_document_id");
+    assert_eq!(response["error"]["type"], "invalid_request");
+    assert_eq!(
+        response["error"]["link"],
+        "https://docs.meilisearch.com/errors#missing_document_id"
+    );
+}

meilisearch-http/tests/index/create_index.rs

@@ -1,4 +1,9 @@
+use crate::common::encoder::Encoder;
 use crate::common::Server;
+use actix_web::http::header::ContentType;
+use actix_web::test;
+use http::header::ACCEPT_ENCODING;
+use meilisearch_http::{analytics, create_app};
 use serde_json::{json, Value};
 
 #[actix_rt::test]
@@ -18,6 +23,95 @@ async fn create_index_no_primary_key() {
     assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
+#[actix_rt::test]
+async fn create_index_with_gzip_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Gzip);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
+#[actix_rt::test]
+async fn create_index_with_gzip_encoded_request_and_receiving_brotli_encoded_response() {
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        &server.service.auth,
+        true,
+        server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+
+    let body = serde_json::to_string(&json!({
+        "uid": "test",
+        "primaryKey": None::<&str>,
+    }))
+    .unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes")
+        .insert_header(Encoder::Gzip.header().unwrap())
+        .insert_header((ACCEPT_ENCODING, "br"))
+        .insert_header(ContentType::json())
+        .set_payload(Encoder::Gzip.encode(body))
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+
+    assert_eq!(res.status(), 202);
+
+    let bytes = test::read_body(res).await;
+    let decoded = Encoder::Brotli.decode(bytes);
+    let parsed_response =
+        serde_json::from_slice::<Value>(decoded.into().as_ref()).expect("Expecting valid json");
+
+    assert_eq!(parsed_response["taskUid"], 0);
+    assert_eq!(parsed_response["indexUid"], "test");
+}
+
+#[actix_rt::test]
+async fn create_index_with_zlib_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Deflate);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
+#[actix_rt::test]
+async fn create_index_with_brotli_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Brotli);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
 #[actix_rt::test]
 async fn create_index_with_primary_key() {
     let server = Server::new().await;

meilisearch-http/tests/index/update_index.rs

@@ -1,3 +1,4 @@
+use crate::common::encoder::Encoder;
 use crate::common::Server;
 use serde_json::json;
 use time::{format_description::well_known::Rfc3339, OffsetDateTime};
@@ -34,6 +35,22 @@ async fn update_primary_key() {
     assert_eq!(response.as_object().unwrap().len(), 4);
 }
 
+#[actix_rt::test]
+async fn create_and_update_with_different_encoding() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Gzip);
+    let (_, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    let index = server.index_with_encoder("test", Encoder::Brotli);
+    index.update(Some("primary")).await;
+
+    let response = index.wait_task(1).await;
+
+    assert_eq!(response["status"], "succeeded");
+}
+
 #[actix_rt::test]
 async fn update_nothing() {
     let server = Server::new().await;