searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-15 13:58:36 +02:00
Code Issues Releases Wiki Activity

Refactor query parameter deserialisation logic

Browse source
This commit is contained in:
Loïc Lecrenier 2023-01-16 16:59:26 +01:00
parent 49ddaaef49
commit 9194508a0f
26 changed files with 1377 additions and 1180 deletions
Download patch file Download diff file Expand all files Collapse all files

38
meilisearch/src/routes/indexes/documents.rs
View file

@ -1,5 +1,4 @@
use std::io::ErrorKind;
use std::num::ParseIntError;
use actix_web::http::header::CONTENT_TYPE;
use actix_web::web::Data;
@ -9,14 +8,15 @@ use deserr::DeserializeFromValue;
use futures::StreamExt;
use index_scheduler::IndexScheduler;
use log::debug;
use meilisearch_types::deserr::query_params::Param;
use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
use meilisearch_types::document_formats::{read_csv, read_json, read_ndjson, PayloadType};
use meilisearch_types::error::{deserr_codes::*, fold_star_or, DeserrQueryParamError};
use meilisearch_types::error::{DeserrJsonError, ResponseError, TakeErrorMessage};
use meilisearch_types::error::deserr_codes::*;
use meilisearch_types::error::ResponseError;
use meilisearch_types::heed::RoTxn;
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::milli::update::IndexDocumentsMethod;
use meilisearch_types::serde_cs::vec::CS;
use meilisearch_types::star_or::StarOr;
use meilisearch_types::star_or::OptionStarOrList;
use meilisearch_types::tasks::KindWithContent;
use meilisearch_types::{milli, Document, Index};
use mime::Mime;
@ -27,7 +27,6 @@ use tempfile::tempfile;
use tokio::fs::File;
use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter};
use super::search::parse_usize_take_error_message;
use crate::analytics::{Analytics, DocumentDeletionKind};
use crate::error::MeilisearchHttpError;
use crate::error::PayloadError::ReceivePayload;
@ -36,7 +35,7 @@ use crate::extractors::authentication::GuardedData;
use crate::extractors::payload::Payload;
use crate::extractors::query_parameters::QueryParameter;
use crate::extractors::sequential_extractor::SeqHandler;
use crate::routes::{PaginationView, SummarizedTaskView};
use crate::routes::{PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
vec!["application/json".to_string(), "application/x-ndjson".to_string(), "text/csv".to_string()]
@ -81,12 +80,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
);
}
#[derive(Deserialize, Debug, DeserializeFromValue)]
#[derive(Debug, DeserializeFromValue)]
#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
pub struct GetDocument {
// TODO: strongly typed argument here
#[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
fields: Option<CS<StarOr<String>>>,
fields: OptionStarOrList<String>,
}
pub async fn get_document(
@ -95,7 +93,7 @@ pub async fn get_document(
params: QueryParameter<GetDocument, DeserrQueryParamError>,
) -> Result<HttpResponse, ResponseError> {
let GetDocument { fields } = params.into_inner();
let attributes_to_retrieve = fields.and_then(fold_star_or);
let attributes_to_retrieve = fields.merge_star_and_none();
let index = index_scheduler.index(&path.index_uid)?;
let document = retrieve_document(&index, &path.document_id, attributes_to_retrieve)?;
@ -119,15 +117,15 @@ pub async fn delete_document(
Ok(HttpResponse::Accepted().json(task))
}
#[derive(Deserialize, Debug, DeserializeFromValue)]
#[derive(Debug, DeserializeFromValue)]
#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
pub struct BrowseQuery {
#[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>, from(&String) = parse_usize_take_error_message -> TakeErrorMessage<ParseIntError>)]
offset: usize,
#[deserr(default = crate::routes::PAGINATION_DEFAULT_LIMIT(), error = DeserrQueryParamError<InvalidDocumentLimit>, from(&String) = parse_usize_take_error_message -> TakeErrorMessage<ParseIntError>)]
limit: usize,
#[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
offset: Param<usize>,
#[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError<InvalidDocumentLimit>)]
limit: Param<usize>,
#[deserr(default, error = DeserrQueryParamError<InvalidDocumentLimit>)]
fields: Option<CS<StarOr<String>>>,
fields: OptionStarOrList<String>,
}
pub async fn get_all_documents(
@ -137,12 +135,12 @@ pub async fn get_all_documents(
) -> Result<HttpResponse, ResponseError> {
debug!("called with params: {:?}", params);
let BrowseQuery { limit, offset, fields } = params.into_inner();
let attributes_to_retrieve = fields.and_then(fold_star_or);
let attributes_to_retrieve = fields.merge_star_and_none();
let index = index_scheduler.index(&index_uid)?;
let (total, documents) = retrieve_documents(&index, offset, limit, attributes_to_retrieve)?;
let (total, documents) = retrieve_documents(&index, offset.0, limit.0, attributes_to_retrieve)?;
let ret = PaginationView::new(offset, limit, total as usize, documents);
let ret = PaginationView::new(offset.0, limit.0, total as usize, documents);
debug!("returns: {:?}", ret);
Ok(HttpResponse::Ok().json(ret))

22
meilisearch/src/routes/indexes/mod.rs
View file

@ -5,8 +5,10 @@ use actix_web::{web, HttpRequest, HttpResponse};
use deserr::{DeserializeError, DeserializeFromValue, ValuePointerRef};
use index_scheduler::IndexScheduler;
use log::debug;
use meilisearch_types::error::{deserr_codes::*, unwrap_any, Code, DeserrQueryParamError};
use meilisearch_types::error::{DeserrJsonError, ResponseError, TakeErrorMessage};
use meilisearch_types::deserr::query_params::Param;
use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
use meilisearch_types::error::ResponseError;
use meilisearch_types::error::{deserr_codes::*, unwrap_any, Code};
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::milli::{self, FieldDistribution, Index};
use meilisearch_types::tasks::KindWithContent;
@ -14,7 +16,6 @@ use serde::{Deserialize, Serialize};
use serde_json::json;
use time::OffsetDateTime;
use self::search::parse_usize_take_error_message;
use super::{Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
use crate::analytics::Analytics;
use crate::extractors::authentication::policies::*;
@ -71,20 +72,17 @@ impl IndexView {
}
}
#[derive(DeserializeFromValue, Deserialize, Debug, Clone, Copy)]
#[derive(DeserializeFromValue, Debug, Clone, Copy)]
#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListIndexes {
#[serde(default)]
#[deserr(default, error = DeserrQueryParamError<InvalidIndexOffset>, from(&String) = parse_usize_take_error_message -> TakeErrorMessage<std::num::ParseIntError>)]
pub offset: usize,
#[serde(default = "PAGINATION_DEFAULT_LIMIT")]
#[deserr(default = PAGINATION_DEFAULT_LIMIT(), error = DeserrQueryParamError<InvalidIndexLimit>, from(&String) = parse_usize_take_error_message -> TakeErrorMessage<std::num::ParseIntError>)]
pub limit: usize,
#[deserr(default, error = DeserrQueryParamError<InvalidIndexOffset>)]
pub offset: Param<usize>,
#[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError<InvalidIndexLimit>)]
pub limit: Param<usize>,
}
impl ListIndexes {
fn as_pagination(self) -> Pagination {
Pagination { offset: self.offset, limit: self.limit }
Pagination { offset: self.offset.0, limit: self.limit.0 }
}
}

59
meilisearch/src/routes/indexes/search.rs
View file

@ -1,15 +1,12 @@
use std::str::FromStr;
use actix_web::web::Data;
use actix_web::{web, HttpRequest, HttpResponse};
use index_scheduler::IndexScheduler;
use log::debug;
use meilisearch_auth::IndexSearchRules;
use meilisearch_types::error::{
deserr_codes::*, parse_option_usize_query_param, parse_usize_query_param,
DeserrQueryParamError, DetailedParseIntError,
};
use meilisearch_types::error::{DeserrJsonError, ResponseError, TakeErrorMessage};
use meilisearch_types::deserr::{DeserrQueryParamError, DeserrJsonError};
use meilisearch_types::deserr::query_params::Param;
use meilisearch_types::error::deserr_codes::*;
use meilisearch_types::error::ResponseError;
use meilisearch_types::serde_cs::vec::CS;
use serde_json::Value;
@ -33,45 +30,33 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
);
}
pub fn parse_usize_take_error_message(
s: &str,
) -> Result<usize, TakeErrorMessage<std::num::ParseIntError>> {
usize::from_str(s).map_err(TakeErrorMessage)
}
pub fn parse_bool_take_error_message(
s: &str,
) -> Result<bool, TakeErrorMessage<std::str::ParseBoolError>> {
s.parse().map_err(TakeErrorMessage)
}
#[derive(Debug, deserr::DeserializeFromValue)]
#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
pub struct SearchQueryGet {
#[deserr(default, error = DeserrQueryParamError<InvalidSearchQ>)]
q: Option<String>,
#[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrQueryParamError<InvalidSearchOffset>, from(String) = parse_usize_query_param -> TakeErrorMessage<DetailedParseIntError>)]
offset: usize,
#[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrQueryParamError<InvalidSearchLimit>, from(String) = parse_usize_query_param -> TakeErrorMessage<DetailedParseIntError>)]
limit: usize,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchPage>, from(Option<String>) = parse_option_usize_query_param -> TakeErrorMessage<std::num::ParseIntError>)]
page: Option<usize>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchHitsPerPage>, from(Option<String>) = parse_option_usize_query_param -> TakeErrorMessage<std::num::ParseIntError>)]
hits_per_page: Option<usize>,
#[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError<InvalidSearchOffset>)]
offset: Param<usize>,
#[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError<InvalidSearchLimit>)]
limit: Param<usize>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchPage>)]
page: Option<Param<usize>>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchHitsPerPage>)]
hits_per_page: Option<Param<usize>>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToRetrieve>)]
attributes_to_retrieve: Option<CS<String>>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToCrop>)]
attributes_to_crop: Option<CS<String>>,
#[deserr(default = DEFAULT_CROP_LENGTH(), error = DeserrQueryParamError<InvalidSearchCropLength>, from(String) = parse_usize_query_param -> TakeErrorMessage<DetailedParseIntError>)]
crop_length: usize,
#[deserr(default = Param(DEFAULT_CROP_LENGTH()), error = DeserrQueryParamError<InvalidSearchCropLength>)]
crop_length: Param<usize>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToHighlight>)]
attributes_to_highlight: Option<CS<String>>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchFilter>)]
filter: Option<String>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchSort>)]
sort: Option<String>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchShowMatchesPosition>, from(&String) = parse_bool_take_error_message -> TakeErrorMessage<std::str::ParseBoolError>)]
show_matches_position: bool,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchShowMatchesPosition>)]
show_matches_position: Param<bool>,
#[deserr(default, error = DeserrQueryParamError<InvalidSearchFacets>)]
facets: Option<CS<String>>,
#[deserr( default = DEFAULT_HIGHLIGHT_PRE_TAG(), error = DeserrQueryParamError<InvalidSearchHighlightPreTag>)]
@ -96,17 +81,17 @@ impl From<SearchQueryGet> for SearchQuery {
Self {
q: other.q,
offset: other.offset,
limit: other.limit,
page: other.page,
hits_per_page: other.hits_per_page,
offset: other.offset.0,
limit: other.limit.0,
page: other.page.as_deref().copied(),
hits_per_page: other.hits_per_page.as_deref().copied(),
attributes_to_retrieve: other.attributes_to_retrieve.map(|o| o.into_iter().collect()),
attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()),
crop_length: other.crop_length,
crop_length: other.crop_length.0,
attributes_to_highlight: other.attributes_to_highlight.map(|o| o.into_iter().collect()),
filter,
sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)),
show_matches_position: other.show_matches_position,
show_matches_position: other.show_matches_position.0,
facets: other.facets.map(|o| o.into_iter().collect()),
highlight_pre_tag: other.highlight_pre_tag,
highlight_post_tag: other.highlight_post_tag,

25
meilisearch/src/routes/indexes/settings.rs
View file

@ -2,7 +2,8 @@ use actix_web::web::Data;
use actix_web::{web, HttpRequest, HttpResponse};
use index_scheduler::IndexScheduler;
use log::debug;
use meilisearch_types::error::{DeserrJsonError, ResponseError};
use meilisearch_types::deserr::DeserrJsonError;
use meilisearch_types::error::ResponseError;
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::settings::{settings, RankingRuleView, Settings, Unchecked};
use meilisearch_types::tasks::KindWithContent;
@ -130,7 +131,7 @@ make_setting_route!(
"/filterable-attributes",
put,
std::collections::BTreeSet<String>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes,
>,
filterable_attributes,
@ -156,7 +157,7 @@ make_setting_route!(
"/sortable-attributes",
put,
std::collections::BTreeSet<String>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsSortableAttributes,
>,
sortable_attributes,
@ -182,7 +183,7 @@ make_setting_route!(
"/displayed-attributes",
put,
Vec<String>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsDisplayedAttributes,
>,
displayed_attributes,
@ -208,7 +209,7 @@ make_setting_route!(
"/typo-tolerance",
patch,
meilisearch_types::settings::TypoSettings,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsTypoTolerance,
>,
typo_tolerance,
@ -253,7 +254,7 @@ make_setting_route!(
"/searchable-attributes",
put,
Vec<String>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsSearchableAttributes,
>,
searchable_attributes,
@ -279,7 +280,7 @@ make_setting_route!(
"/stop-words",
put,
std::collections::BTreeSet<String>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsStopWords,
>,
stop_words,
@ -304,7 +305,7 @@ make_setting_route!(
"/synonyms",
put,
std::collections::BTreeMap<String, Vec<String>>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsSynonyms,
>,
synonyms,
@ -329,7 +330,7 @@ make_setting_route!(
"/distinct-attribute",
put,
String,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsDistinctAttribute,
>,
distinct_attribute,
@ -353,7 +354,7 @@ make_setting_route!(
"/ranking-rules",
put,
Vec<meilisearch_types::settings::RankingRuleView>,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsRankingRules,
>,
ranking_rules,
@ -384,7 +385,7 @@ make_setting_route!(
"/faceting",
patch,
meilisearch_types::settings::FacetingSettings,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsFaceting,
>,
faceting,
@ -409,7 +410,7 @@ make_setting_route!(
"/pagination",
patch,
meilisearch_types::settings::PaginationSettings,
meilisearch_types::error::DeserrJsonError<
meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsPagination,
>,
pagination,