mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-03 03:47:02 +02:00
feat(all): Task store
implements: https://github.com/meilisearch/specifications/blob/develop/text/0060-refashion-updates-apis.md linked PR: - #1889 - #1891 - #1892 - #1902 - #1906 - #1911 - #1914 - #1915 - #1916 - #1918 - #1924 - #1925 - #1926 - #1930 - #1936 - #1937 - #1942 - #1944 - #1945 - #1946 - #1947 - #1950 - #1951 - #1957 - #1959 - #1960 - #1961 - #1962 - #1964 - https://github.com/meilisearch/milli/pull/414 - https://github.com/meilisearch/milli/pull/409 - https://github.com/meilisearch/milli/pull/406 - https://github.com/meilisearch/milli/pull/418 - close #1687 - close #1786 - close #1940 - close #1948 - close #1949 - close #1932 - close #1956
This commit is contained in:
Marin Postma
parent
c9f3726447
commit
a30e02c18c
88 changed files with 5553 additions and 4496 deletions
|
|
@ -1,25 +1,38 @@
|
|||
use actix_web::error::PayloadError;
|
||||
use actix_web::http::header::CONTENT_TYPE;
|
||||
use actix_web::web::Bytes;
|
||||
use actix_web::HttpMessage;
|
||||
use actix_web::{web, HttpRequest, HttpResponse};
|
||||
use bstr::ByteSlice;
|
||||
use futures::{Stream, StreamExt};
|
||||
use log::debug;
|
||||
use meilisearch_error::ResponseError;
|
||||
use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
|
||||
use meilisearch_lib::milli::update::IndexDocumentsMethod;
|
||||
use meilisearch_lib::MeiliSearch;
|
||||
use mime::Mime;
|
||||
use once_cell::sync::Lazy;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::analytics::Analytics;
|
||||
use crate::error::{MeilisearchHttpError, ResponseError};
|
||||
use crate::error::MeilisearchHttpError;
|
||||
use crate::extractors::authentication::{policies::*, GuardedData};
|
||||
use crate::extractors::payload::Payload;
|
||||
use crate::routes::IndexParam;
|
||||
use crate::task::SummarizedTaskView;
|
||||
|
||||
const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0;
|
||||
const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20;
|
||||
|
||||
static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
|
||||
vec![
|
||||
"application/json".to_string(),
|
||||
"application/x-ndjson".to_string(),
|
||||
"text/csv".to_string(),
|
||||
]
|
||||
});
|
||||
|
||||
/// This is required because Payload is not Sync nor Send
|
||||
fn payload_to_stream(mut payload: Payload) -> impl Stream<Item = Result<Bytes, PayloadError>> {
|
||||
let (snd, recv) = mpsc::channel(1);
|
||||
|
|
@ -31,6 +44,24 @@ fn payload_to_stream(mut payload: Payload) -> impl Stream<Item = Result<Bytes, P
|
|||
tokio_stream::wrappers::ReceiverStream::new(recv)
|
||||
}
|
||||
|
||||
/// Extracts the mime type from the content type and return
|
||||
/// a meilisearch error if anyhthing bad happen.
|
||||
fn extract_mime_type(req: &HttpRequest) -> Result<Option<Mime>, MeilisearchHttpError> {
|
||||
match req.mime_type() {
|
||||
Ok(Some(mime)) => Ok(Some(mime)),
|
||||
Ok(None) => Ok(None),
|
||||
Err(_) => match req.headers().get(CONTENT_TYPE) {
|
||||
Some(content_type) => Err(MeilisearchHttpError::InvalidContentType(
|
||||
content_type.as_bytes().as_bstr().to_string(),
|
||||
ACCEPTED_CONTENT_TYPE.clone(),
|
||||
)),
|
||||
None => Err(MeilisearchHttpError::MissingContentType(
|
||||
ACCEPTED_CONTENT_TYPE.clone(),
|
||||
)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DocumentParam {
|
||||
index_uid: String,
|
||||
|
|
@ -76,11 +107,9 @@ pub async fn delete_document(
|
|||
index_uid,
|
||||
} = path.into_inner();
|
||||
let update = Update::DeleteDocuments(vec![document_id]);
|
||||
let update_status = meilisearch
|
||||
.register_update(index_uid, update, false)
|
||||
.await?;
|
||||
debug!("returns: {:?}", update_status);
|
||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||
let task: SummarizedTaskView = meilisearch.register_update(index_uid, update).await?.into();
|
||||
debug!("returns: {:?}", task);
|
||||
Ok(HttpResponse::Accepted().json(task))
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
|
|
@ -93,7 +122,7 @@ pub struct BrowseQuery {
|
|||
|
||||
pub async fn get_all_documents(
|
||||
meilisearch: GuardedData<Public, MeiliSearch>,
|
||||
path: web::Path<IndexParam>,
|
||||
path: web::Path<String>,
|
||||
params: web::Query<BrowseQuery>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
debug!("called with params: {:?}", params);
|
||||
|
|
@ -110,7 +139,7 @@ pub async fn get_all_documents(
|
|||
|
||||
let documents = meilisearch
|
||||
.documents(
|
||||
path.index_uid.clone(),
|
||||
path.into_inner(),
|
||||
params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET),
|
||||
params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT),
|
||||
attributes_to_retrieve,
|
||||
|
|
@ -128,91 +157,83 @@ pub struct UpdateDocumentsQuery {
|
|||
|
||||
pub async fn add_documents(
|
||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||
path: web::Path<IndexParam>,
|
||||
path: web::Path<String>,
|
||||
params: web::Query<UpdateDocumentsQuery>,
|
||||
body: Payload,
|
||||
req: HttpRequest,
|
||||
analytics: web::Data<dyn Analytics>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
debug!("called with params: {:?}", params);
|
||||
let content_type = req
|
||||
.headers()
|
||||
.get("Content-type")
|
||||
.map(|s| s.to_str().unwrap_or("unkown"));
|
||||
let params = params.into_inner();
|
||||
let index_uid = path.into_inner();
|
||||
|
||||
analytics.add_documents(
|
||||
¶ms,
|
||||
meilisearch.get_index(path.index_uid.clone()).await.is_err(),
|
||||
meilisearch.get_index(index_uid.clone()).await.is_err(),
|
||||
&req,
|
||||
);
|
||||
|
||||
document_addition(
|
||||
content_type,
|
||||
let task = document_addition(
|
||||
extract_mime_type(&req)?,
|
||||
meilisearch,
|
||||
path.index_uid.clone(),
|
||||
index_uid,
|
||||
params.primary_key,
|
||||
body,
|
||||
IndexDocumentsMethod::ReplaceDocuments,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
Ok(HttpResponse::Accepted().json(task))
|
||||
}
|
||||
|
||||
pub async fn update_documents(
|
||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||
path: web::Path<IndexParam>,
|
||||
path: web::Path<String>,
|
||||
params: web::Query<UpdateDocumentsQuery>,
|
||||
body: Payload,
|
||||
req: HttpRequest,
|
||||
analytics: web::Data<dyn Analytics>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
debug!("called with params: {:?}", params);
|
||||
let content_type = req
|
||||
.headers()
|
||||
.get("Content-type")
|
||||
.map(|s| s.to_str().unwrap_or("unkown"));
|
||||
let index_uid = path.into_inner();
|
||||
|
||||
analytics.update_documents(
|
||||
¶ms,
|
||||
meilisearch.get_index(path.index_uid.clone()).await.is_err(),
|
||||
meilisearch.get_index(index_uid.clone()).await.is_err(),
|
||||
&req,
|
||||
);
|
||||
|
||||
document_addition(
|
||||
content_type,
|
||||
let task = document_addition(
|
||||
extract_mime_type(&req)?,
|
||||
meilisearch,
|
||||
path.into_inner().index_uid,
|
||||
index_uid,
|
||||
params.into_inner().primary_key,
|
||||
body,
|
||||
IndexDocumentsMethod::UpdateDocuments,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
Ok(HttpResponse::Accepted().json(task))
|
||||
}
|
||||
|
||||
/// Route used when the payload type is "application/json"
|
||||
/// Used to add or replace documents
|
||||
async fn document_addition(
|
||||
content_type: Option<&str>,
|
||||
mime_type: Option<Mime>,
|
||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||
index_uid: String,
|
||||
primary_key: Option<String>,
|
||||
body: Payload,
|
||||
method: IndexDocumentsMethod,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
|
||||
vec![
|
||||
"application/json".to_string(),
|
||||
"application/x-ndjson".to_string(),
|
||||
"text/csv".to_string(),
|
||||
]
|
||||
});
|
||||
let format = match content_type {
|
||||
Some("application/json") => DocumentAdditionFormat::Json,
|
||||
Some("application/x-ndjson") => DocumentAdditionFormat::Ndjson,
|
||||
Some("text/csv") => DocumentAdditionFormat::Csv,
|
||||
Some(other) => {
|
||||
) -> Result<SummarizedTaskView, ResponseError> {
|
||||
let format = match mime_type
|
||||
.as_ref()
|
||||
.map(|m| (m.type_().as_str(), m.subtype().as_str()))
|
||||
{
|
||||
Some(("application", "json")) => DocumentAdditionFormat::Json,
|
||||
Some(("application", "x-ndjson")) => DocumentAdditionFormat::Ndjson,
|
||||
Some(("text", "csv")) => DocumentAdditionFormat::Csv,
|
||||
Some((type_, subtype)) => {
|
||||
return Err(MeilisearchHttpError::InvalidContentType(
|
||||
other.to_string(),
|
||||
format!("{}/{}", type_, subtype),
|
||||
ACCEPTED_CONTENT_TYPE.clone(),
|
||||
)
|
||||
.into())
|
||||
|
|
@ -231,15 +252,15 @@ async fn document_addition(
|
|||
format,
|
||||
};
|
||||
|
||||
let update_status = meilisearch.register_update(index_uid, update, true).await?;
|
||||
let task = meilisearch.register_update(index_uid, update).await?.into();
|
||||
|
||||
debug!("returns: {:?}", update_status);
|
||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||
debug!("returns: {:?}", task);
|
||||
Ok(task)
|
||||
}
|
||||
|
||||
pub async fn delete_documents(
|
||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||
path: web::Path<IndexParam>,
|
||||
path: web::Path<String>,
|
||||
body: web::Json<Vec<Value>>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
debug!("called with params: {:?}", body);
|
||||
|
|
@ -253,21 +274,25 @@ pub async fn delete_documents(
|
|||
.collect();
|
||||
|
||||
let update = Update::DeleteDocuments(ids);
|
||||
let update_status = meilisearch
|
||||
.register_update(path.into_inner().index_uid, update, false)
|
||||
.await?;
|
||||
debug!("returns: {:?}", update_status);
|
||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||
let task: SummarizedTaskView = meilisearch
|
||||
.register_update(path.into_inner(), update)
|
||||
.await?
|
||||
.into();
|
||||
|
||||
debug!("returns: {:?}", task);
|
||||
Ok(HttpResponse::Accepted().json(task))
|
||||
}
|
||||
|
||||
pub async fn clear_all_documents(
|
||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||
path: web::Path<IndexParam>,
|
||||
path: web::Path<String>,
|
||||
) -> Result<HttpResponse, ResponseError> {
|
||||
let update = Update::ClearDocuments;
|
||||
let update_status = meilisearch
|
||||
.register_update(path.into_inner().index_uid, update, false)
|
||||
.await?;
|
||||
debug!("returns: {:?}", update_status);
|
||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||
let task: SummarizedTaskView = meilisearch
|
||||
.register_update(path.into_inner(), update)
|
||||
.await?
|
||||
.into();
|
||||
|
||||
debug!("returns: {:?}", task);
|
||||
Ok(HttpResponse::Accepted().json(task))
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue