From 911630000f4b5eb3d25064b813293edf1b427869 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 29 Sep 2021 00:12:25 +0200 Subject: [PATCH] split csv and json document routes --- .../src/routes/indexes/documents.rs | 82 +++++++++---------- meilisearch-http/src/routes/mod.rs | 6 +- .../tests/documents/add_documents.rs | 4 +- .../src/index_controller/updates/mod.rs | 5 +- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/meilisearch-http/src/routes/indexes/documents.rs b/meilisearch-http/src/routes/indexes/documents.rs index e89b75f28..dfa1244d1 100644 --- a/meilisearch-http/src/routes/indexes/documents.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -18,7 +18,6 @@ use crate::routes::IndexParam; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; -/* macro_rules! guard_content_type { ($fn_name:ident, $guard_value:literal) => { fn $fn_name(head: &actix_web::dev::RequestHead) -> bool { @@ -33,9 +32,8 @@ macro_rules! guard_content_type { } }; } - guard_content_type!(guard_json, "application/json"); -*/ +guard_content_type!(guard_csv, "application/csv"); /// This is required because Payload is not Sync nor Send fn payload_to_stream(mut payload: Payload) -> impl Stream> { @@ -48,22 +46,6 @@ fn payload_to_stream(mut payload: Payload) -> impl Stream bool { - if let Some(_content_type) = head.headers.get("Content-Type") { - // CURRENTLY AND FOR THIS RELEASE ONLY WE DECIDED TO INTERPRET ALL CONTENT-TYPES AS JSON - true - /* - content_type - .to_str() - .map(|v| v.contains("application/json")) - .unwrap_or(false) - */ - } else { - // if no content-type is specified we still accept the data as json! - true - } -} - #[derive(Deserialize)] pub struct DocumentParam { index_uid: String, @@ -74,8 +56,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_all_documents)) - .route(web::post().guard(guard_json).to(add_documents)) - .route(web::put().guard(guard_json).to(update_documents)) + .route(web::post().guard(guard_json).to(add_documents_json)) + .route(web::post().guard(guard_csv).to(add_documents_csv)) + .route(web::post().guard(guard_json).to(update_documents_json)) + .route(web::post().guard(guard_csv).to(update_documents_csv)) .route(web::delete().to(clear_all_documents)), ) // this route needs to be before the /documents/{document_id} to match properly @@ -159,43 +143,57 @@ pub struct UpdateDocumentsQuery { primary_key: Option, } -/// Route used when the payload type is "application/json" -/// Used to add or replace documents -pub async fn add_documents( +pub async fn add_documents_json( meilisearch: GuardedData, path: web::Path, params: web::Query, body: Payload, ) -> Result { - debug!("called with params: {:?}", params); - let update = Update::DocumentAddition { - payload: Box::new(payload_to_stream(body)), - primary_key: params.primary_key.clone(), - method: IndexDocumentsMethod::ReplaceDocuments, - format: DocumentAdditionFormat::Json, - }; - let update_status = meilisearch - .register_update(path.into_inner().index_uid, update, true) - .await?; - - debug!("returns: {:?}", update_status); - Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) + document_addition(meilisearch, path, params, body, DocumentAdditionFormat::Json, IndexDocumentsMethod::ReplaceDocuments).await } -/// Route used when the payload type is "application/json" -/// Used to add or replace documents -pub async fn update_documents( +pub async fn add_documents_csv( meilisearch: GuardedData, path: web::Path, params: web::Query, body: Payload, +) -> Result { + document_addition(meilisearch, path, params, body, DocumentAdditionFormat::Csv, IndexDocumentsMethod::ReplaceDocuments).await +} + +pub async fn update_documents_json( + meilisearch: GuardedData, + path: web::Path, + params: web::Query, + body: Payload, +) -> Result { + document_addition(meilisearch, path, params, body, DocumentAdditionFormat::Json, IndexDocumentsMethod::UpdateDocuments).await +} + +pub async fn update_documents_csv( + meilisearch: GuardedData, + path: web::Path, + params: web::Query, + body: Payload, +) -> Result { + document_addition(meilisearch, path, params, body, DocumentAdditionFormat::Csv, IndexDocumentsMethod::UpdateDocuments).await +} +/// Route used when the payload type is "application/json" +/// Used to add or replace documents +async fn document_addition( + meilisearch: GuardedData, + path: web::Path, + params: web::Query, + body: Payload, + format: DocumentAdditionFormat, + method: IndexDocumentsMethod, ) -> Result { debug!("called with params: {:?}", params); let update = Update::DocumentAddition { payload: Box::new(payload_to_stream(body)), primary_key: params.primary_key.clone(), - method: IndexDocumentsMethod::UpdateDocuments, - format: DocumentAdditionFormat::Json, + method, + format, }; let update_status = meilisearch .register_update(path.into_inner().index_uid, update, true) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 382147f31..9c0e0bed1 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -361,8 +361,10 @@ mod test { indexes::documents::clear_all_documents, indexes::documents::delete_documents, - indexes::documents::update_documents, - indexes::documents::add_documents, + indexes::documents::update_documents_json, + indexes::documents::update_documents_csv, + indexes::documents::add_documents_json, + indexes::documents::add_documents_csv, indexes::documents::delete_document, indexes::updates::get_all_updates_status, diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 42fdc7509..13265dcfd 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -152,8 +152,8 @@ async fn document_add_create_index_bad_uid() { async fn document_update_create_index_bad_uid() { let server = Server::new().await; let index = server.index("883 fj!"); - let (_response, code) = index.update_documents(json!([]), None).await; - assert_eq!(code, 400); + let (response, code) = index.update_documents(json!([]), None).await; + assert_eq!(code, 400, "{}", response); } #[actix_rt::test] diff --git a/meilisearch-lib/src/index_controller/updates/mod.rs b/meilisearch-lib/src/index_controller/updates/mod.rs index 56ea779de..474f0e77e 100644 --- a/meilisearch-lib/src/index_controller/updates/mod.rs +++ b/meilisearch-lib/src/index_controller/updates/mod.rs @@ -67,8 +67,9 @@ impl> + Unpin> io::Rea // TODO: optimize buf filling match self.current.take() { Some(mut bytes) => { - let copied = bytes.split_to(buf.len()); - buf.copy_from_slice(&copied); + let split_at = bytes.len().min(buf.len()); + let copied = bytes.split_to(split_at); + buf[..split_at].copy_from_slice(&copied); if !bytes.is_empty() { self.current.replace(bytes); }