From 11b986a81d3dd716556c9e34a52393e4073d9899 Mon Sep 17 00:00:00 2001 From: "Andrey \"MOU\" Larionov" Date: Sun, 9 Oct 2022 19:43:51 +0200 Subject: [PATCH] Added support for specifying compression in tests Refactored tests code to allow to specify compression (content-encoding) algorithm. Added tests to verify what actix actually handle different content encodings properly. --- Cargo.lock | 2 + meilisearch-http/Cargo.toml | 2 + meilisearch-http/tests/common/encoder.rs | 50 ++++++ meilisearch-http/tests/common/index.rs | 67 ++++---- meilisearch-http/tests/common/mod.rs | 1 + meilisearch-http/tests/common/server.rs | 6 + meilisearch-http/tests/common/service.rs | 155 +++++++------------ meilisearch-http/tests/index/create_index.rs | 52 +++++++ meilisearch-http/tests/index/update_index.rs | 17 ++ 9 files changed, 224 insertions(+), 128 deletions(-) create mode 100644 meilisearch-http/tests/common/encoder.rs diff --git a/Cargo.lock b/Cargo.lock index b26636a01..2ba905455 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2038,6 +2038,7 @@ name = "meilisearch-http" version = "0.29.1" dependencies = [ "actix-cors", + "actix-http", "actix-rt", "actix-web", "actix-web-static-files", @@ -2045,6 +2046,7 @@ dependencies = [ "assert-json-diff", "async-stream", "async-trait", + "brotli", "bstr 1.0.1", "byte-unit", "bytes", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index cabb6ed48..6dbad4354 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -24,6 +24,7 @@ zip = { version = "0.6.2", optional = true } [dependencies] actix-cors = "0.6.3" actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] } +actix-http = { version = "3.2.2" } actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true } anyhow = { version = "1.0.65", features = ["backtrace"] } async-stream = "0.3.3" @@ -89,6 +90,7 @@ manifest-dir-macros = "0.1.16" maplit = "1.0.2" urlencoding = "2.1.2" yaup = "0.2.1" +brotli = "3.3.4" [features] default = ["analytics", "meilisearch-lib/default", "mini-dashboard"] diff --git a/meilisearch-http/tests/common/encoder.rs b/meilisearch-http/tests/common/encoder.rs new file mode 100644 index 000000000..41d560253 --- /dev/null +++ b/meilisearch-http/tests/common/encoder.rs @@ -0,0 +1,50 @@ +use std::io::Write; +use actix_http::header::TryIntoHeaderPair; +use bytes::Bytes; +use flate2::write::{GzEncoder, ZlibEncoder}; +use flate2::Compression; + +#[derive(Clone,Copy)] +pub enum Encoder { + Plain, + Gzip, + Deflate, + Brotli, +} + +impl Encoder { + pub fn encode(self: &Encoder, body: impl Into) -> impl Into { + match self { + Self::Gzip => { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&body.into()).expect("Failed to encode request body"); + encoder.finish().expect("Failed to encode request body") + } + Self::Deflate => { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&body.into()).expect("Failed to encode request body"); + encoder.finish().unwrap() + } + Self::Plain => Vec::from(body.into()), + Self::Brotli => { + let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22); + encoder.write_all(&body.into()).expect("Failed to encode request body"); + encoder.flush().expect("Failed to encode request body"); + encoder.into_inner() + } + } + } + + pub fn header(self: &Encoder) -> Option { + match self { + Self::Plain => None, + Self::Gzip => Some(("Content-Encoding", "gzip")), + Self::Deflate => Some(("Content-Encoding", "deflate")), + Self::Brotli => Some(("Content-Encoding", "br")), + } + } + + pub fn iterator() -> impl Iterator { + [Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli].iter().copied() + } +} diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 90d138ced..c084d1f09 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -7,24 +7,27 @@ use std::{ use actix_web::http::StatusCode; use serde_json::{json, Value}; use tokio::time::sleep; -use urlencoding::encode; +use urlencoding::encode as urlencode; use super::service::Service; +use super::encoder::Encoder; + pub struct Index<'a> { pub uid: String, pub service: &'a Service, + pub encoder: Encoder, } #[allow(dead_code)] impl Index<'_> { pub async fn get(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.get(url).await } pub async fn load_test_set(&self) -> u64 { - let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref())); let (response, code) = self .service .post_str(url, include_str!("../assets/test_set.json")) @@ -40,20 +43,22 @@ impl Index<'_> { "uid": self.uid, "primaryKey": primary_key, }); - self.service.post("/indexes", body).await + self.service + .post_encoded("/indexes", body, self.encoder) + .await } pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "primaryKey": primary_key, }); - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); - self.service.patch(url, body).await + self.service.patch_encoded(url, body, self.encoder).await } pub async fn delete(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.delete(url).await } @@ -65,12 +70,14 @@ impl Index<'_> { let url = match primary_key { Some(key) => format!( "/indexes/{}/documents?primaryKey={}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), key ), - None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), + None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())), }; - self.service.post(url, documents).await + self.service + .post_encoded(url, documents, self.encoder) + .await } pub async fn update_documents( @@ -81,12 +88,12 @@ impl Index<'_> { let url = match primary_key { Some(key) => format!( "/indexes/{}/documents?primaryKey={}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), key ), - None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), + None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())), }; - self.service.put(url, documents).await + self.service.put_encoded(url, documents, self.encoder).await } pub async fn wait_task(&self, update_id: u64) -> Value { @@ -132,7 +139,7 @@ impl Index<'_> { id: u64, options: Option, ) -> (Value, StatusCode) { - let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id); if let Some(fields) = options.and_then(|o| o.fields) { let _ = write!(url, "?fields={}", fields.join(",")); } @@ -140,7 +147,7 @@ impl Index<'_> { } pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { - let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref())); + let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref())); if let Some(limit) = options.limit { let _ = write!(url, "limit={}&", limit); } @@ -157,42 +164,42 @@ impl Index<'_> { } pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id); self.service.delete(url).await } pub async fn clear_all_documents(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref())); self.service.delete(url).await } pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { let url = format!( "/indexes/{}/documents/delete-batch", - encode(self.uid.as_ref()) + urlencode(self.uid.as_ref()) ); self.service - .post(url, serde_json::to_value(&ids).unwrap()) + .post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder) .await } pub async fn settings(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); self.service.get(url).await } pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); - self.service.patch(url, settings).await + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); + self.service.patch_encoded(url, settings, self.encoder).await } pub async fn delete_settings(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); self.service.delete(url).await } pub async fn stats(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/stats", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref())); self.service.get(url).await } @@ -217,29 +224,29 @@ impl Index<'_> { } pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search", encode(self.uid.as_ref())); - self.service.post(url, query).await + let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref())); + self.service.post_encoded(url, query, self.encoder).await } pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { let params = yaup::to_string(&query).unwrap(); - let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params); + let url = format!("/indexes/{}/search?{}", urlencode(self.uid.as_ref()), params); self.service.get(url).await } pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) { let url = format!( "/indexes/{}/settings/{}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), "distinct-attribute" ); - self.service.put(url, value).await + self.service.put_encoded(url, value, self.encoder).await } pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) { let url = format!( "/indexes/{}/settings/{}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), "distinct-attribute" ); self.service.get(url).await diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index b076b0ea5..c4793a0cb 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -1,3 +1,4 @@ +pub mod encoder; pub mod index; pub mod server; pub mod service; diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 30a64c90e..f243a11b9 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -12,6 +12,7 @@ use once_cell::sync::Lazy; use serde_json::Value; use tempfile::TempDir; +use crate::common::encoder::Encoder; use meilisearch_http::option::Opt; use super::index::Index; @@ -100,9 +101,14 @@ impl Server { /// Returns a view to an index. There is no guarantee that the index exists. pub fn index(&self, uid: impl AsRef) -> Index<'_> { + self.index_with_encoder(uid, Encoder::Plain) + } + + pub fn index_with_encoder(&self, uid: impl AsRef, encoder: Encoder) -> Index<'_> { Index { uid: uid.as_ref().to_string(), service: &self.service, + encoder, } } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index 0834abf8d..3a3f6021c 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -1,8 +1,11 @@ +use actix_web::http::header::ContentType; +use actix_web::test::TestRequest; use actix_web::{http::StatusCode, test}; use meilisearch_auth::AuthController; use meilisearch_lib::MeiliSearch; use serde_json::Value; +use crate::common::encoder::Encoder; use meilisearch_http::{analytics, create_app, Opt}; pub struct Service { @@ -14,26 +17,18 @@ pub struct Service { impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.post_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::post().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn post_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::post().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } /// Send a test post request from a text body, with a `content-type:application/json` header. @@ -42,101 +37,54 @@ impl Service { url: impl AsRef, body: impl AsRef, ) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; - - let mut req = test::TestRequest::post() + let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) .insert_header(("content-type", "application/json")); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + self.request(req).await } pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; - - let mut req = test::TestRequest::get().uri(url.as_ref()); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + let req = test::TestRequest::get().uri(url.as_ref()); + self.request(req).await } pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.put_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::put().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn put_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::put().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } pub async fn patch(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.patch_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::patch().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn patch_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::patch().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { + let req = test::TestRequest::delete().uri(url.as_ref()); + self.request(req).await + } + + pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) { let app = test::init_service(create_app!( &self.meilisearch, &self.auth, @@ -146,7 +94,6 @@ impl Service { )) .await; - let mut req = test::TestRequest::delete().uri(url.as_ref()); if let Some(api_key) = &self.api_key { req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); } @@ -158,4 +105,16 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } + + fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest { + let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json"); + let encoded_body = encoder.encode(bytes); + let header = encoder.header(); + match header { + Some(header) => req.insert_header(header), + None => req, + } + .set_payload(encoded_body) + .insert_header(ContentType::json()) + } } diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index a1c508e1f..f6ca76ee7 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -1,3 +1,4 @@ +use crate::common::encoder::Encoder; use crate::common::Server; use serde_json::{json, Value}; @@ -18,6 +19,57 @@ async fn create_index_no_primary_key() { assert_eq!(response["details"]["primaryKey"], Value::Null); } +#[actix_rt::test] +async fn create_index_with_gzip_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn create_index_with_zlib_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Deflate); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn create_index_with_brotli_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Brotli); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + #[actix_rt::test] async fn create_index_with_primary_key() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 48fde5608..97eecbf83 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -1,3 +1,4 @@ +use crate::common::encoder::Encoder; use crate::common::Server; use serde_json::json; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; @@ -34,6 +35,22 @@ async fn update_primary_key() { assert_eq!(response.as_object().unwrap().len(), 4); } +#[actix_rt::test] +async fn create_and_update_with_different_encoding() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + let (_, code) = index.create(None).await; + + assert_eq!(code, 202); + + let index = server.index_with_encoder("test", Encoder::Brotli); + index.update(Some("primary")).await; + + let response = index.wait_task(1).await; + + assert_eq!(response["status"], "succeeded"); +} + #[actix_rt::test] async fn update_nothing() { let server = Server::new().await;