refactor meilisearch

This commit is contained in:
mpostma 2021-09-14 18:39:02 +02:00
parent 6fafdb7711
commit e14640e530
33 changed files with 1222 additions and 1166 deletions

View file

@ -1,12 +1,17 @@
use actix_web::error::PayloadError;
use actix_web::{web, HttpResponse};
use actix_web::web::Bytes;
use futures::{Stream, StreamExt};
use log::debug;
use milli::update::{IndexDocumentsMethod, UpdateFormat};
use milli::update::IndexDocumentsMethod;
use serde::Deserialize;
use serde_json::Value;
//use serde_json::Value;
use tokio::sync::mpsc;
use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::extractors::payload::Payload;
use crate::index_controller::{DocumentAdditionFormat, Update};
use crate::routes::IndexParam;
use crate::Data;
@ -32,6 +37,17 @@ macro_rules! guard_content_type {
guard_content_type!(guard_json, "application/json");
*/
/// This is required because Payload is not Sync nor Send
fn payload_to_stream(mut payload: Payload) -> impl Stream<Item=Result<Bytes, PayloadError>> {
let (snd, recv) = mpsc::channel(1);
tokio::task::spawn_local(async move {
while let Some(data) = payload.next().await {
let _ = snd.send(data).await;
}
});
tokio_stream::wrappers::ReceiverStream::new(recv)
}
fn guard_json(head: &actix_web::dev::RequestHead) -> bool {
if let Some(_content_type) = head.headers.get("Content-Type") {
// CURRENTLY AND FOR THIS RELEASE ONLY WE DECIDED TO INTERPRET ALL CONTENT-TYPES AS JSON
@ -60,14 +76,14 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
.route(web::get().to(get_all_documents))
.route(web::post().guard(guard_json).to(add_documents))
.route(web::put().guard(guard_json).to(update_documents))
.route(web::delete().to(clear_all_documents)),
//.route(web::delete().to(clear_all_documents)),
)
// this route needs to be before the /documents/{document_id} to match properly
.service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
//.service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
.service(
web::resource("/{document_id}")
.route(web::get().to(get_document))
.route(web::delete().to(delete_document)),
//.route(web::delete().to(delete_document)),
);
}
@ -84,16 +100,16 @@ pub async fn get_document(
Ok(HttpResponse::Ok().json(document))
}
pub async fn delete_document(
data: GuardedData<Private, Data>,
path: web::Path<DocumentParam>,
) -> Result<HttpResponse, ResponseError> {
let update_status = data
.delete_documents(path.index_uid.clone(), vec![path.document_id.clone()])
.await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
//pub async fn delete_document(
//data: GuardedData<Private, Data>,
//path: web::Path<DocumentParam>,
//) -> Result<HttpResponse, ResponseError> {
//let update_status = data
//.delete_documents(path.index_uid.clone(), vec![path.document_id.clone()])
//.await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@ -147,14 +163,14 @@ pub async fn add_documents(
body: Payload,
) -> Result<HttpResponse, ResponseError> {
debug!("called with params: {:?}", params);
let update = Update::DocumentAddition {
payload: Box::new(payload_to_stream(body)),
primary_key: params.primary_key.clone(),
method: IndexDocumentsMethod::ReplaceDocuments,
format: DocumentAdditionFormat::Json,
};
let update_status = data
.add_documents(
path.into_inner().index_uid,
IndexDocumentsMethod::ReplaceDocuments,
UpdateFormat::Json,
body,
params.primary_key.clone(),
)
.register_update(path.index_uid.as_str(), update)
.await?;
debug!("returns: {:?}", update_status);
@ -170,45 +186,45 @@ pub async fn update_documents(
body: Payload,
) -> Result<HttpResponse, ResponseError> {
debug!("called with params: {:?}", params);
let update = data
.add_documents(
path.into_inner().index_uid,
IndexDocumentsMethod::UpdateDocuments,
UpdateFormat::Json,
body,
params.primary_key.clone(),
)
let update = Update::DocumentAddition {
payload: Box::new(payload_to_stream(body)),
primary_key: params.primary_key.clone(),
method: IndexDocumentsMethod::UpdateDocuments,
format: DocumentAdditionFormat::Json,
};
let update_status = data
.register_update(path.index_uid.as_str(), update)
.await?;
debug!("returns: {:?}", update);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() })))
}
pub async fn delete_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
body: web::Json<Vec<Value>>,
) -> Result<HttpResponse, ResponseError> {
debug!("called with params: {:?}", body);
let ids = body
.iter()
.map(|v| {
v.as_str()
.map(String::from)
.unwrap_or_else(|| v.to_string())
})
.collect();
let update_status = data.delete_documents(path.index_uid.clone(), ids).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
pub async fn clear_all_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
let update_status = data.clear_documents(path.index_uid.clone()).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
//pub async fn delete_documents(
//data: GuardedData<Private, Data>,
//path: web::Path<IndexParam>,
//body: web::Json<Vec<Value>>,
//) -> Result<HttpResponse, ResponseError> {
//debug!("called with params: {:?}", body);
//let ids = body
//.iter()
//.map(|v| {
//v.as_str()
//.map(String::from)
//.unwrap_or_else(|| v.to_string())
//})
//.collect();
//let update_status = data.delete_documents(path.index_uid.clone(), ids).await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}
//pub async fn clear_all_documents(
//data: GuardedData<Private, Data>,
//path: web::Path<IndexParam>,
//) -> Result<HttpResponse, ResponseError> {
//let update_status = data.clear_documents(path.index_uid.clone()).await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}

View file

@ -17,7 +17,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("")
.route(web::get().to(list_indexes))
.route(web::post().to(create_index)),
//.route(web::post().to(create_index)),
)
.service(
web::scope("/{index_uid}")
@ -25,13 +25,13 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
web::resource("")
.route(web::get().to(get_index))
.route(web::put().to(update_index))
.route(web::delete().to(delete_index)),
//.route(web::delete().to(delete_index)),
)
.service(web::resource("/stats").route(web::get().to(get_index_stats)))
.service(web::scope("/documents").configure(documents::configure))
.service(web::scope("/search").configure(search::configure))
.service(web::scope("/updates").configure(updates::configure))
.service(web::scope("/settings").configure(settings::configure)),
//.service(web::scope("/settings").configure(settings::configure)),
);
}
@ -48,14 +48,14 @@ pub struct IndexCreateRequest {
primary_key: Option<String>,
}
pub async fn create_index(
data: GuardedData<Private, Data>,
body: web::Json<IndexCreateRequest>,
) -> Result<HttpResponse, ResponseError> {
let body = body.into_inner();
let meta = data.create_index(body.uid, body.primary_key).await?;
Ok(HttpResponse::Created().json(meta))
}
//pub async fn create_index(
//data: GuardedData<Private, Data>,
//body: web::Json<IndexCreateRequest>,
//) -> Result<HttpResponse, ResponseError> {
//let body = body.into_inner();
//let meta = data.create_index(body.uid, body.primary_key).await?;
//Ok(HttpResponse::Created().json(meta))
//}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@ -97,13 +97,13 @@ pub async fn update_index(
Ok(HttpResponse::Ok().json(meta))
}
pub async fn delete_index(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
data.delete_index(path.index_uid.clone()).await?;
Ok(HttpResponse::NoContent().finish())
}
//pub async fn delete_index(
//data: GuardedData<Private, Data>,
//path: web::Path<IndexParam>,
//) -> Result<HttpResponse, ResponseError> {
//data.delete_index(path.index_uid.clone()).await?;
//Ok(HttpResponse::NoContent().finish())
//}
pub async fn get_index_stats(
data: GuardedData<Private, Data>,

View file

@ -1,185 +1,184 @@
use actix_web::{web, HttpResponse};
use log::debug;
//use log::debug;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::index::Settings;
use crate::Data;
use crate::{error::ResponseError, index::Unchecked};
//use crate::extractors::authentication::{policies::*, GuardedData};
//use crate::index::Settings;
//use crate::Data;
//use crate::error::ResponseError;
#[macro_export]
macro_rules! make_setting_route {
($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
pub mod $attr {
use log::debug;
use actix_web::{web, HttpResponse, Resource};
//#[macro_export]
//macro_rules! make_setting_route {
//($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
//pub mod $attr {
//use log::debug;
//use actix_web::{web, HttpResponse, Resource};
use milli::update::Setting;
//use milli::update::Setting;
use crate::data;
use crate::error::ResponseError;
use crate::index::Settings;
use crate::extractors::authentication::{GuardedData, policies::*};
//use crate::data;
//use crate::error::ResponseError;
//use crate::index::Settings;
//use crate::extractors::authentication::{GuardedData, policies::*};
pub async fn delete(
data: GuardedData<Private, data::Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
use crate::index::Settings;
let settings = Settings {
$attr: Setting::Reset,
..Default::default()
};
let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
//pub async fn delete(
//data: GuardedData<Private, data::Data>,
//index_uid: web::Path<String>,
//) -> Result<HttpResponse, ResponseError> {
//use crate::index::Settings;
//let settings = Settings {
//$attr: Setting::Reset,
//..Default::default()
//};
//let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}
pub async fn update(
data: GuardedData<Private, data::Data>,
index_uid: actix_web::web::Path<String>,
body: actix_web::web::Json<Option<$type>>,
) -> std::result::Result<HttpResponse, ResponseError> {
let settings = Settings {
$attr: match body.into_inner() {
Some(inner_body) => Setting::Set(inner_body),
None => Setting::Reset
},
..Default::default()
};
//pub async fn update(
//data: GuardedData<Private, data::Data>,
//index_uid: actix_web::web::Path<String>,
//body: actix_web::web::Json<Option<$type>>,
//) -> std::result::Result<HttpResponse, ResponseError> {
//let settings = Settings {
//$attr: match body.into_inner() {
//Some(inner_body) => Setting::Set(inner_body),
//None => Setting::Reset
//},
//..Default::default()
//};
let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
//let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}
pub async fn get(
data: GuardedData<Private, data::Data>,
index_uid: actix_web::web::Path<String>,
) -> std::result::Result<HttpResponse, ResponseError> {
let settings = data.settings(index_uid.into_inner()).await?;
debug!("returns: {:?}", settings);
let mut json = serde_json::json!(&settings);
let val = json[$camelcase_attr].take();
Ok(HttpResponse::Ok().json(val))
}
//pub async fn get(
//data: GuardedData<Private, data::Data>,
//index_uid: actix_web::web::Path<String>,
//) -> std::result::Result<HttpResponse, ResponseError> {
//let settings = data.settings(index_uid.into_inner()).await?;
//debug!("returns: {:?}", settings);
//let mut json = serde_json::json!(&settings);
//let val = json[$camelcase_attr].take();
//Ok(HttpResponse::Ok().json(val))
//}
pub fn resources() -> Resource {
Resource::new($route)
.route(web::get().to(get))
.route(web::post().to(update))
.route(web::delete().to(delete))
}
}
};
}
//pub fn resources() -> Resource {
//Resource::new($route)
//.route(web::get().to(get))
//.route(web::post().to(update))
//.route(web::delete().to(delete))
//}
//}
//};
//}
make_setting_route!(
"/filterable-attributes",
std::collections::BTreeSet<String>,
filterable_attributes,
"filterableAttributes"
);
//make_setting_route!(
//"/filterable-attributes",
//std::collections::BTreeSet<String>,
//filterable_attributes,
//"filterableAttributes"
//);
make_setting_route!(
"/sortable-attributes",
std::collections::BTreeSet<String>,
sortable_attributes,
"sortableAttributes"
);
//make_setting_route!(
//"/sortable-attributes",
//std::collections::BTreeSet<String>,
//sortable_attributes,
//"sortableAttributes"
//);
make_setting_route!(
"/displayed-attributes",
Vec<String>,
displayed_attributes,
"displayedAttributes"
);
//make_setting_route!(
//"/displayed-attributes",
//Vec<String>,
//displayed_attributes,
//"displayedAttributes"
//);
make_setting_route!(
"/searchable-attributes",
Vec<String>,
searchable_attributes,
"searchableAttributes"
);
//make_setting_route!(
//"/searchable-attributes",
//Vec<String>,
//searchable_attributes,
//"searchableAttributes"
//);
make_setting_route!(
"/stop-words",
std::collections::BTreeSet<String>,
stop_words,
"stopWords"
);
//make_setting_route!(
//"/stop-words",
//std::collections::BTreeSet<String>,
//stop_words,
//"stopWords"
//);
make_setting_route!(
"/synonyms",
std::collections::BTreeMap<String, Vec<String>>,
synonyms,
"synonyms"
);
//make_setting_route!(
//"/synonyms",
//std::collections::BTreeMap<String, Vec<String>>,
//synonyms,
//"synonyms"
//);
make_setting_route!(
"/distinct-attribute",
String,
distinct_attribute,
"distinctAttribute"
);
//make_setting_route!(
//"/distinct-attribute",
//String,
//distinct_attribute,
//"distinctAttribute"
//);
make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules");
//make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules");
macro_rules! generate_configure {
($($mod:ident),*) => {
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("")
.route(web::post().to(update_all))
.route(web::get().to(get_all))
.route(web::delete().to(delete_all)))
$(.service($mod::resources()))*;
}
};
}
//macro_rules! generate_configure {
//($($mod:ident),*) => {
//pub fn configure(cfg: &mut web::ServiceConfig) {
//cfg.service(
//web::resource("")
////.route(web::post().to(update_all))
//.route(web::get().to(get_all))
////.route(web::delete().to(delete_all)))
//$(.service($mod::resources()))*;
//}
//};
//}
generate_configure!(
filterable_attributes,
sortable_attributes,
displayed_attributes,
searchable_attributes,
distinct_attribute,
stop_words,
synonyms,
ranking_rules
);
//generate_configure!(
//filterable_attributes,
//sortable_attributes,
//displayed_attributes,
//searchable_attributes,
//distinct_attribute,
//stop_words,
//synonyms,
//ranking_rules
//);
pub async fn update_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
body: web::Json<Settings<Unchecked>>,
) -> Result<HttpResponse, ResponseError> {
let settings = body.into_inner().check();
let update_result = data
.update_settings(index_uid.into_inner(), settings, true)
.await?;
let json = serde_json::json!({ "updateId": update_result.id() });
debug!("returns: {:?}", json);
Ok(HttpResponse::Accepted().json(json))
}
//pub async fn update_all(
//data: GuardedData<Private, Data>,
//index_uid: web::Path<String>,
//body: web::Json<Settings<Unchecked>>,
//) -> Result<HttpResponse, ResponseError> {
//let settings = body.into_inner().check();
//let update_result = data
//.update_settings(index_uid.into_inner(), settings, true)
//.await?;
//let json = serde_json::json!({ "updateId": update_result.id() });
//debug!("returns: {:?}", json);
//Ok(HttpResponse::Accepted().json(json))
//}
pub async fn get_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
let settings = data.settings(index_uid.into_inner()).await?;
debug!("returns: {:?}", settings);
Ok(HttpResponse::Ok().json(settings))
}
//pub async fn get_all(
//data: GuardedData<Private, Data>,
//index_uid: web::Path<String>,
//) -> Result<HttpResponse, ResponseError> {
//let settings = data.settings(index_uid.into_inner()).await?;
//debug!("returns: {:?}", settings);
//Ok(HttpResponse::Ok().json(settings))
//}
pub async fn delete_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
let settings = Settings::cleared();
let update_result = data
.update_settings(index_uid.into_inner(), settings, false)
.await?;
let json = serde_json::json!({ "updateId": update_result.id() });
debug!("returns: {:?}", json);
Ok(HttpResponse::Accepted().json(json))
}
//pub async fn delete_all(
//data: GuardedData<Private, Data>,
//index_uid: web::Path<String>,
//) -> Result<HttpResponse, ResponseError> {
//let settings = Settings::cleared();
//let update_result = data
//.update_settings(index_uid.into_inner(), settings, false)
//.await?;
//let json = serde_json::json!({ "updateId": update_result.id() });
//debug!("returns: {:?}", json);
//Ok(HttpResponse::Accepted().json(json))
//}