Merge pull request #661 from meilisearch/add-actix-middleware

Add actix middleware
This commit is contained in:
Clément Renault 2020-05-12 16:04:29 +02:00 committed by GitHub
commit a88f6c3241
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 157 additions and 11 deletions

View File

@ -3,7 +3,10 @@
- Add support for faceted search (#631) - Add support for faceted search (#631)
- Add support for configuring the lmdb map size (#646, #647) - Add support for configuring the lmdb map size (#646, #647)
- Add exposed port for Dockerfile (#654) - Add exposed port for Dockerfile (#654)
- Add sentry probe - Add sentry probe (#664)
- Fix url trailing slash and double slash issues (#659)
- Fix accept all Content-Type by default (#653)
- Return the error message from Serde when a deserialization error is encountered (#661)
## v0.10.1 ## v0.10.1

2
Cargo.lock generated
View File

@ -1641,6 +1641,7 @@ dependencies = [
"actix-web", "actix-web",
"actix-web-macros", "actix-web-macros",
"assert-json-diff", "assert-json-diff",
"bytes 0.5.4",
"chrono", "chrono",
"crossbeam-channel", "crossbeam-channel",
"env_logger", "env_logger",
@ -1660,6 +1661,7 @@ dependencies = [
"pretty-bytes", "pretty-bytes",
"rand 0.7.3", "rand 0.7.3",
"sentry", "sentry",
"regex",
"serde", "serde",
"serde_json", "serde_json",
"serde_qs", "serde_qs",

View File

@ -24,6 +24,7 @@ actix-rt = "1"
actix-service = "1.0.5" actix-service = "1.0.5"
actix-web = "2" actix-web = "2"
actix-web-macros = "0.1.0" actix-web-macros = "0.1.0"
bytes = "0.5.4"
chrono = { version = "0.4.11", features = ["serde"] } chrono = { version = "0.4.11", features = ["serde"] }
crossbeam-channel = "0.4.2" crossbeam-channel = "0.4.2"
env_logger = "0.7.1" env_logger = "0.7.1"
@ -39,6 +40,7 @@ meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.10.1"}
mime = "0.3.16" mime = "0.3.16"
pretty-bytes = "0.2.2" pretty-bytes = "0.2.2"
rand = "0.7.3" rand = "0.7.3"
regex = "1.3.6"
serde = { version = "1.0.105", features = ["derive"] } serde = { version = "1.0.105", features = ["derive"] }
serde_json = { version = "1.0.50", features = ["preserve_order"] } serde_json = { version = "1.0.50", features = ["preserve_order"] }
serde_qs = "0.5.2" serde_qs = "0.5.2"

View File

@ -134,7 +134,7 @@ impl Data {
let db_opt = DatabaseOptions { let db_opt = DatabaseOptions {
main_map_size: opt.main_map_size, main_map_size: opt.main_map_size,
update_map_size: opt.update_map_size update_map_size: opt.update_map_size,
}; };
let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap()); let db = Arc::new(Database::open_or_create(opt.db_path, db_opt).unwrap());

View File

@ -4,6 +4,7 @@ use actix_http::ResponseBuilder;
use actix_web as aweb; use actix_web as aweb;
use actix_web::http::StatusCode; use actix_web::http::StatusCode;
use serde_json::json; use serde_json::json;
use actix_web::error::JsonPayloadError;
#[derive(Debug)] #[derive(Debug)]
pub enum ResponseError { pub enum ResponseError {
@ -23,6 +24,8 @@ pub enum ResponseError {
FilterParsing(String), FilterParsing(String),
RetrieveDocument(u64, String), RetrieveDocument(u64, String),
SearchDocuments(String), SearchDocuments(String),
PayloadTooLarge,
UnsupportedMediaType,
FacetExpression(String), FacetExpression(String),
} }
@ -108,6 +111,8 @@ impl fmt::Display for ResponseError {
Self::RetrieveDocument(id, err) => write!(f, "impossible to retrieve the document with id: {}; {}", id, err), Self::RetrieveDocument(id, err) => write!(f, "impossible to retrieve the document with id: {}; {}", id, err),
Self::SearchDocuments(err) => write!(f, "impossible to search documents; {}", err), Self::SearchDocuments(err) => write!(f, "impossible to search documents; {}", err),
Self::FacetExpression(e) => write!(f, "error parsing facet filter expression: {}", e), Self::FacetExpression(e) => write!(f, "error parsing facet filter expression: {}", e),
Self::PayloadTooLarge => f.write_str("Payload to large"),
Self::UnsupportedMediaType => f.write_str("Unsupported media type")
} }
} }
} }
@ -138,6 +143,8 @@ impl aweb::error::ResponseError for ResponseError {
Self::MissingAuthorizationHeader => StatusCode::FORBIDDEN, Self::MissingAuthorizationHeader => StatusCode::FORBIDDEN,
Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
Self::Maintenance => StatusCode::SERVICE_UNAVAILABLE, Self::Maintenance => StatusCode::SERVICE_UNAVAILABLE,
Self::PayloadTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
Self::UnsupportedMediaType => StatusCode::UNSUPPORTED_MEDIA_TYPE,
} }
} }
} }
@ -190,3 +197,19 @@ impl From<actix_http::Error> for ResponseError {
ResponseError::Internal(err.to_string()) ResponseError::Internal(err.to_string())
} }
} }
impl From<JsonPayloadError> for ResponseError {
fn from(err: JsonPayloadError) -> ResponseError {
match err {
JsonPayloadError::Deserialize(err) => ResponseError::BadRequest(format!("Invalid JSON: {}", err)),
JsonPayloadError::Overflow => ResponseError::PayloadTooLarge,
JsonPayloadError::ContentType => ResponseError::UnsupportedMediaType,
JsonPayloadError::Payload(err) => ResponseError::BadRequest(format!("Problem while decoding the request: {}", err)),
}
}
}
pub fn json_error_handler(err: JsonPayloadError) -> ResponseError {
err.into()
}

View File

@ -17,7 +17,6 @@ pub enum Authentication {
Admin, Admin,
} }
impl<S: 'static, B> Transform<S> for Authentication impl<S: 'static, B> Transform<S> for Authentication
where where
S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>, S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error>,

View File

@ -1,4 +1,6 @@
pub mod authentication; pub mod authentication;
pub mod meilisearch; pub mod meilisearch;
pub mod normalize_slashes;
pub use authentication::Authentication; pub use authentication::Authentication;
pub use normalize_slashes::NormalizeSlashes;

View File

@ -0,0 +1,88 @@
///
/// This middleware normalizes slashes in paths
/// * consecutive instances of `/` get collapsed into one `/`
/// * any ending `/` is removed.
/// Original source from: https://gitlab.com/snippets/1884466
///
/// Ex:
/// /this///url/
/// becomes : /this/url
///
use actix_service::{Service, Transform};
use actix_web::{
dev::ServiceRequest,
dev::ServiceResponse,
http::uri::{PathAndQuery, Uri},
Error as ActixError,
};
use futures::future::{ok, Ready};
use regex::Regex;
use std::task::{Context, Poll};
pub struct NormalizeSlashes;
impl<S, B> Transform<S> for NormalizeSlashes
where
S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = ActixError>,
S::Future: 'static,
{
type Request = ServiceRequest;
type Response = ServiceResponse<B>;
type Error = ActixError;
type InitError = ();
type Transform = SlashNormalization<S>;
type Future = Ready<Result<Self::Transform, Self::InitError>>;
fn new_transform(&self, service: S) -> Self::Future {
ok(SlashNormalization { service })
}
}
pub struct SlashNormalization<S> {
service: S,
}
impl<S, B> Service for SlashNormalization<S>
where
S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = ActixError>,
S::Future: 'static,
{
type Request = ServiceRequest;
type Response = ServiceResponse<B>;
type Error = ActixError;
type Future = S::Future;
fn poll_ready(&mut self, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
self.service.poll_ready(cx)
}
fn call(&mut self, mut req: ServiceRequest) -> Self::Future {
let head = req.head();
let path = head.uri.path();
let original_len = path.len();
let slash_regex = Regex::new("//+").unwrap();
let new_path = slash_regex.replace_all(path, "/");
let new_path = new_path.trim_end_matches("/");
if original_len != new_path.len() {
let mut parts = head.uri.clone().into_parts();
let path = match parts.path_and_query.as_ref().map(|pq| pq.query()).flatten() {
Some(q) => bytes::Bytes::from(format!("{}?{}", new_path, q)),
None => bytes::Bytes::from(new_path.to_string()),
};
if let Ok(pq) = PathAndQuery::from_maybe_shared(path) {
parts.path_and_query = Some(pq);
if let Ok(uri) = Uri::from_parts(parts) {
req.match_info_mut().get_mut().update(&uri);
req.head_mut().uri = uri;
}
}
}
self.service.call(req)
}
}

View File

@ -8,6 +8,7 @@ pub mod option;
pub mod routes; pub mod routes;
pub use self::data::Data; pub use self::data::Data;
use self::error::json_error_handler;
use actix_http::Error; use actix_http::Error;
use actix_service::ServiceFactory; use actix_service::ServiceFactory;
use actix_web::{dev, web, App}; use actix_web::{dev, web, App};
@ -28,7 +29,12 @@ pub fn create_app(
> { > {
App::new() App::new()
.app_data(web::Data::new(data.clone())) .app_data(web::Data::new(data.clone()))
.app_data(web::JsonConfig::default().limit(1024 * 1024 * 10)) // Json Limit of 10Mb .app_data(
web::JsonConfig::default()
.limit(1024 * 1024 * 10) // Json Limit of 10Mb
.content_type(|_mime| true) // Accept all mime types
.error_handler(|err, _req| json_error_handler(err).into()),
)
.service(routes::load_html) .service(routes::load_html)
.service(routes::load_css) .service(routes::load_css)
.configure(routes::document::services) .configure(routes::document::services)

View File

@ -5,6 +5,7 @@ use actix_web::{middleware, HttpServer};
use log::info; use log::info;
use main_error::MainError; use main_error::MainError;
use meilisearch_http::data::Data; use meilisearch_http::data::Data;
use meilisearch_http::helpers::NormalizeSlashes;
use meilisearch_http::option::Opt; use meilisearch_http::option::Opt;
use meilisearch_http::{create_app, index_update_callback}; use meilisearch_http::{create_app, index_update_callback};
use structopt::StructOpt; use structopt::StructOpt;
@ -72,6 +73,7 @@ async fn main() -> Result<(), MainError> {
) )
.wrap(middleware::Logger::default()) .wrap(middleware::Logger::default())
.wrap(middleware::Compress::default()) .wrap(middleware::Compress::default())
.wrap(NormalizeSlashes)
}) })
.bind(opt.http_addr)? .bind(opt.http_addr)?
.run() .run()

View File

@ -33,5 +33,5 @@ pub struct Opt {
/// The maximum size, in bytes, of the update lmdb database directory /// The maximum size, in bytes, of the update lmdb database directory
#[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "107374182400")] // 100GB #[structopt(long, env = "MEILI_UPDATE_MAP_SIZE", default_value = "107374182400")] // 100GB
pub update_map_size: usize pub update_map_size: usize,
} }

View File

@ -4,9 +4,10 @@ use serde_json::{json, Value};
use std::time::Duration; use std::time::Duration;
use actix_web::{http::StatusCode, test}; use actix_web::{http::StatusCode, test};
use meilisearch_core::DatabaseOptions;
use meilisearch_http::data::Data; use meilisearch_http::data::Data;
use meilisearch_http::option::Opt; use meilisearch_http::option::Opt;
use meilisearch_core::DatabaseOptions; use meilisearch_http::helpers::NormalizeSlashes;
use tempdir::TempDir; use tempdir::TempDir;
use tokio::time::delay_for; use tokio::time::delay_for;
@ -28,7 +29,7 @@ impl Server {
env: "development".to_owned(), env: "development".to_owned(),
no_analytics: true, no_analytics: true,
main_map_size: default_db_options.main_map_size, main_map_size: default_db_options.main_map_size,
update_map_size: default_db_options.update_map_size update_map_size: default_db_options.update_map_size,
}; };
let data = Data::new(opt.clone()); let data = Data::new(opt.clone());
@ -126,7 +127,7 @@ impl Server {
pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) {
eprintln!("get_request: {}", url); eprintln!("get_request: {}", url);
let mut app = test::init_service(meilisearch_http::create_app(&self.data)).await; let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizeSlashes)).await;
let req = test::TestRequest::get().uri(url).to_request(); let req = test::TestRequest::get().uri(url).to_request();
let res = test::call_service(&mut app, req).await; let res = test::call_service(&mut app, req).await;
@ -140,7 +141,7 @@ impl Server {
pub async fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub async fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("post_request: {}", url); eprintln!("post_request: {}", url);
let mut app = test::init_service(meilisearch_http::create_app(&self.data)).await; let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizeSlashes)).await;
let req = test::TestRequest::post() let req = test::TestRequest::post()
.uri(url) .uri(url)
@ -169,7 +170,7 @@ impl Server {
pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("put_request: {}", url); eprintln!("put_request: {}", url);
let mut app = test::init_service(meilisearch_http::create_app(&self.data)).await; let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizeSlashes)).await;
let req = test::TestRequest::put() let req = test::TestRequest::put()
.uri(url) .uri(url)
@ -197,7 +198,7 @@ impl Server {
pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) {
eprintln!("delete_request: {}", url); eprintln!("delete_request: {}", url);
let mut app = test::init_service(meilisearch_http::create_app(&self.data)).await; let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizeSlashes)).await;
let req = test::TestRequest::delete().uri(url).to_request(); let req = test::TestRequest::delete().uri(url).to_request();
let res = test::call_service(&mut app, req).await; let res = test::call_service(&mut app, req).await;

View File

@ -0,0 +1,18 @@
mod common;
#[actix_rt::test]
async fn url_normalizer() {
let mut server = common::Server::with_uid("movies");
let (_response, status_code) = server.get_request("/version").await;
assert_eq!(status_code, 200);
let (_response, status_code) = server.get_request("//version").await;
assert_eq!(status_code, 200);
let (_response, status_code) = server.get_request("/version/").await;
assert_eq!(status_code, 200);
let (_response, status_code) = server.get_request("//version/").await;
assert_eq!(status_code, 200);
}