Merge #232

232: Fix payload size limit r=MarinPostma a=MarinPostma

Fix #223

This was due to the fact that Payload ignores the limit payload size limit. I fixed it by implementing my own `Payload` extractor that checks that the size of the payload is not too large.

I also refactored the `create_app` a bit.

Co-authored-by: marin postma <postma.marin@protonmail.com>

meilisearch-http/src/index_controller/mod.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 
-use actix_web::web::{Bytes, Payload};
+use actix_web::web::Bytes;
 use chrono::{DateTime, Utc};
 use futures::stream::StreamExt;
 use log::error;
@@ -22,6 +22,7 @@ use update_actor::UpdateActorHandle;
 pub use updates::*;
 use uuid_resolver::{error::UuidResolverError, UuidResolverHandle};
 
+use crate::extractors::payload::Payload;
 use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
 use crate::option::Opt;
 use error::Result;

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -122,7 +122,7 @@ where
         &self,
         uuid: Uuid,
         meta: UpdateMeta,
-        mut payload: mpsc::Receiver<PayloadData<D>>,
+        payload: mpsc::Receiver<PayloadData<D>>,
     ) -> Result<UpdateStatus> {
         let file_path = match meta {
             UpdateMeta::DocumentsAddition { .. } => {
@@ -137,21 +137,41 @@ where
                     .open(&path)
                     .await?;
 
-                let mut file_len = 0;
-                while let Some(bytes) = payload.recv().await {
-                    let bytes = bytes?;
-                    file_len += bytes.as_ref().len();
-                    file.write_all(bytes.as_ref()).await?;
+                async fn write_to_file<D>(
+                    file: &mut fs::File,
+                    mut payload: mpsc::Receiver<PayloadData<D>>,
+                ) -> Result<usize>
+                where
+                    D: AsRef<[u8]> + Sized + 'static,
+                {
+                    let mut file_len = 0;
+
+                    while let Some(bytes) = payload.recv().await {
+                        let bytes = bytes?;
+                        file_len += bytes.as_ref().len();
+                        file.write_all(bytes.as_ref()).await?;
+                    }
+
+                    file.flush().await?;
+
+                    Ok(file_len)
                 }
 
-                if file_len != 0 {
-                    file.flush().await?;
-                    let file = file.into_std().await;
-                    Some((file, update_file_id))
-                } else {
-                    // empty update, delete the empty file.
-                    fs::remove_file(&path).await?;
-                    None
+                let file_len = write_to_file(&mut file, payload).await;
+
+                match file_len {
+                    Ok(len) if len > 0 => {
+                        let file = file.into_std().await;
+                        Some((file, update_file_id))
+                    }
+                    Err(e) => {
+                        fs::remove_file(&path).await?;
+                        return Err(e);
+                    }
+                    _ => {
+                        fs::remove_file(&path).await?;
+                        None
+                    }
                 }
             }
             _ => None,

meilisearch-http/src/index_controller/update_actor/error.rs

@@ -21,6 +21,8 @@ pub enum UpdateActorError {
     FatalUpdateStoreError,
     #[error("invalid payload: {0}")]
     InvalidPayload(Box<dyn Error + Send + Sync + 'static>),
+    #[error("payload error: {0}")]
+    PayloadError(#[from] actix_web::error::PayloadError),
 }
 
 impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateActorError {
@@ -39,7 +41,6 @@ internal_error!(
     UpdateActorError: heed::Error,
     std::io::Error,
     serde_json::Error,
-    actix_http::error::PayloadError,
     tokio::task::JoinError
 );
 
@@ -51,6 +52,10 @@ impl ErrorCode for UpdateActorError {
             UpdateActorError::IndexActor(e) => e.error_code(),
             UpdateActorError::FatalUpdateStoreError => Code::Internal,
             UpdateActorError::InvalidPayload(_) => Code::BadRequest,
+            UpdateActorError::PayloadError(error) => match error {
+                actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge,
+                _ => Code::Internal,
+            },
         }
     }
 }

meilisearch-http/src/index_controller/update_actor/store/mod.rs

@@ -572,7 +572,10 @@ fn update_uuid_to_file_path(root: impl AsRef<Path>, uuid: Uuid) -> PathBuf {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::index_controller::{UpdateResult, index_actor::{MockIndexActorHandle, error::IndexActorError}};
+    use crate::index_controller::{
+        index_actor::{error::IndexActorError, MockIndexActorHandle},
+        UpdateResult,
+    };
 
     use futures::future::ok;
 
@@ -651,7 +654,9 @@ mod test {
                 if processing.id() == 0 {
                     Box::pin(ok(Ok(processing.process(UpdateResult::Other))))
                 } else {
-                    Box::pin(ok(Err(processing.fail(IndexActorError::ExistingPrimaryKey.into()))))
+                    Box::pin(ok(Err(
+                        processing.fail(IndexActorError::ExistingPrimaryKey.into())
+                    )))
                 }
             });
 

meilisearch-http/src/index_controller/updates.rs

@@ -3,7 +3,10 @@ use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat};
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
-use crate::{error::ResponseError, index::{Settings, Unchecked}};
+use crate::{
+    error::ResponseError,
+    index::{Settings, Unchecked},
+};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum UpdateResult {