From 742d0ee5312baa0f89abbc41d59274f5b2212aff Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 8 Aug 2024 19:14:19 +0200 Subject: [PATCH 01/27] Implements the get and delete tasks route --- Cargo.lock | 70 +++++ crates/meilisearch-types/Cargo.toml | 1 + .../src/deserr/query_params.rs | 13 + crates/meilisearch-types/src/error.rs | 17 +- .../src/facet_values_sort.rs | 3 +- crates/meilisearch-types/src/index_uid.rs | 4 +- crates/meilisearch-types/src/keys.rs | 24 +- crates/meilisearch-types/src/locales.rs | 5 +- crates/meilisearch-types/src/settings.rs | 63 +++- crates/meilisearch-types/src/task_view.rs | 30 +- crates/meilisearch-types/src/tasks.rs | 37 ++- crates/meilisearch/Cargo.toml | 4 + crates/meilisearch/src/routes/api_key.rs | 263 ++++++++++++++++- crates/meilisearch/src/routes/dump.rs | 50 ++++ .../src/routes/indexes/documents.rs | 238 ++++++++++++++- crates/meilisearch/src/routes/indexes/mod.rs | 259 +++++++++++++++- crates/meilisearch/src/routes/logs.rs | 122 +++++++- crates/meilisearch/src/routes/metrics.rs | 99 ++++++- crates/meilisearch/src/routes/mod.rs | 204 +++++++++++-- .../meilisearch/src/routes/open_api_utils.rs | 24 ++ crates/meilisearch/src/routes/snapshot.rs | 45 +++ crates/meilisearch/src/routes/tasks.rs | 278 +++++++++++++++++- crates/milli/Cargo.toml | 2 + .../milli/src/localized_attributes_rules.rs | 4 +- crates/milli/src/update/settings.rs | 13 + 25 files changed, 1787 insertions(+), 85 deletions(-) create mode 100644 crates/meilisearch/src/routes/open_api_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 91c83fb13..b375a2616 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "zstd", ] [[package]] @@ -92,6 +93,7 @@ dependencies = [ "bytestring", "cfg-if", "http 0.2.11", + "regex", "regex-lite", "serde", "tracing", @@ -197,6 +199,7 @@ dependencies = [ "mime", "once_cell", "pin-project-lite", + "regex", "regex-lite", "serde", "serde_json", @@ -3532,6 +3535,10 @@ dependencies = [ "tracing-trace", "url", "urlencoding", + "utoipa", + "utoipa-rapidoc", + "utoipa-redoc", + "utoipa-scalar", "uuid", "wiremock", "yaup", @@ -3587,6 +3594,7 @@ dependencies = [ "thiserror", "time", "tokio", + "utoipa", "uuid", ] @@ -3698,6 +3706,7 @@ dependencies = [ "uell", "ureq", "url", + "utoipa", "uuid", ] @@ -5953,6 +5962,67 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "utoipa" +version = "5.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf66139459b75afa33caddb62bb2afee3838923b630b9e0ef38c369e543382f" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c136da726bb82a527afa1fdf3f4619eaf104e2982f071f25239cef1c67c79eb" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.60", + "uuid", +] + +[[package]] +name = "utoipa-rapidoc" +version = "4.0.1-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4d3324d5874fb734762214827dd30b47aa78510d12abab674a97f6d7c53688f" +dependencies = [ + "actix-web", + "serde", + "serde_json", + "utoipa", +] + +[[package]] +name = "utoipa-redoc" +version = "4.0.1-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4375bb6b0cb78a240c973f5e99977c482f3e92aeea1907367caa28776b9aaf9" +dependencies = [ + "actix-web", + "serde", + "serde_json", + "utoipa", +] + +[[package]] +name = "utoipa-scalar" +version = "0.2.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dc122c11f9642b20b3be88b60c1a3672319811f139698ac6999e72992ac7c7e" +dependencies = [ + "actix-web", + "serde", + "serde_json", + "utoipa", +] + [[package]] name = "uuid" version = "1.10.0" diff --git a/crates/meilisearch-types/Cargo.toml b/crates/meilisearch-types/Cargo.toml index 76d8d11ca..d0d136399 100644 --- a/crates/meilisearch-types/Cargo.toml +++ b/crates/meilisearch-types/Cargo.toml @@ -40,6 +40,7 @@ time = { version = "0.3.36", features = [ "macros", ] } tokio = "1.38" +utoipa = { version = "5.0.0-rc.0", features = ["macros"] } uuid = { version = "1.10.0", features = ["serde", "v4"] } [dev-dependencies] diff --git a/crates/meilisearch-types/src/deserr/query_params.rs b/crates/meilisearch-types/src/deserr/query_params.rs index dded0ea5c..58113567e 100644 --- a/crates/meilisearch-types/src/deserr/query_params.rs +++ b/crates/meilisearch-types/src/deserr/query_params.rs @@ -16,6 +16,7 @@ use std::ops::Deref; use std::str::FromStr; use deserr::{DeserializeError, Deserr, MergeWithError, ValueKind}; +use utoipa::{PartialSchema, ToSchema}; use super::{DeserrParseBoolError, DeserrParseIntError}; use crate::index_uid::IndexUid; @@ -29,6 +30,18 @@ use crate::tasks::{Kind, Status}; #[derive(Default, Debug, Clone, Copy)] pub struct Param(pub T); +impl ToSchema for Param { + fn name() -> std::borrow::Cow<'static, str> { + T::name() + } +} + +impl PartialSchema for Param { + fn schema() -> utoipa::openapi::RefOr { + T::schema() + } +} + impl Deref for Param { type Target = T; diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 0c4027899..a864f8aae 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -7,17 +7,25 @@ use aweb::rt::task::JoinError; use convert_case::Casing; use milli::heed::{Error as HeedError, MdbError}; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct ResponseError { #[serde(skip)] pub code: StatusCode, + /// The error message. pub message: String, + /// The error code. + #[schema(value_type = Code)] #[serde(rename = "code")] error_code: String, + /// The error type. + #[schema(value_type = ErrorType)] #[serde(rename = "type")] error_type: String, + /// A link to the documentation about this specific error. #[serde(rename = "link")] error_link: String, } @@ -97,7 +105,9 @@ pub trait ErrorCode { } #[allow(clippy::enum_variant_names)] -enum ErrorType { +#[derive(ToSchema)] +#[schema(rename_all = "snake_case")] +pub enum ErrorType { Internal, InvalidRequest, Auth, @@ -129,7 +139,8 @@ impl fmt::Display for ErrorType { /// `MyErrorCode::default().error_code()`. macro_rules! make_error_codes { ($($code_ident:ident, $err_type:ident, $status:ident);*) => { - #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, ToSchema)] + #[schema(rename_all = "snake_case")] pub enum Code { $($code_ident),* } diff --git a/crates/meilisearch-types/src/facet_values_sort.rs b/crates/meilisearch-types/src/facet_values_sort.rs index 278061f19..8e0dd2ca4 100644 --- a/crates/meilisearch-types/src/facet_values_sort.rs +++ b/crates/meilisearch-types/src/facet_values_sort.rs @@ -1,8 +1,9 @@ use deserr::Deserr; use milli::OrderBy; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr)] +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr, ToSchema)] #[serde(rename_all = "camelCase")] #[deserr(rename_all = camelCase)] pub enum FacetValuesSort { diff --git a/crates/meilisearch-types/src/index_uid.rs b/crates/meilisearch-types/src/index_uid.rs index 03a31a82f..4bf126794 100644 --- a/crates/meilisearch-types/src/index_uid.rs +++ b/crates/meilisearch-types/src/index_uid.rs @@ -4,13 +4,15 @@ use std::fmt; use std::str::FromStr; use deserr::Deserr; +use utoipa::ToSchema; use crate::error::{Code, ErrorCode}; /// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400 /// bytes long -#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord, ToSchema)] #[deserr(try_from(String) = IndexUid::try_from -> IndexUidFormatError)] +#[schema(value_type = String, example = "movies")] pub struct IndexUid(String); impl IndexUid { diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index f7d80bbcb..8fcbab14d 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use time::format_description::well_known::Rfc3339; use time::macros::{format_description, time}; use time::{Date, OffsetDateTime, PrimitiveDateTime}; +use utoipa::ToSchema; use uuid::Uuid; use crate::deserr::{immutable_field_error, DeserrError, DeserrJsonError}; @@ -32,19 +33,31 @@ impl MergeWithError for Dese } } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct CreateApiKey { + /// A description for the key. `null` if empty. + #[schema(example = json!(null))] #[deserr(default, error = DeserrJsonError)] pub description: Option, + /// A human-readable name for the key. `null` if empty. + #[schema(example = "Indexing Products API key")] #[deserr(default, error = DeserrJsonError)] pub name: Option, + /// A uuid v4 to identify the API Key. If not specified, it's generated by Meilisearch. + #[schema(value_type = Uuid, example = json!(null))] #[deserr(default = Uuid::new_v4(), error = DeserrJsonError, try_from(&String) = Uuid::from_str -> uuid::Error)] pub uid: KeyId, + /// A list of actions permitted for the key. `["*"]` for all actions. The `*` character can be used as a wildcard when located at the last position. e.g. `documents.*` to authorize access on all documents endpoints. + #[schema(example = json!(["documents.add"]))] #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_api_key_actions)] pub actions: Vec, + /// A list of accesible indexes permitted for the key. `["*"]` for all indexes. The `*` character can be used as a wildcard when located at the last position. e.g. `products_*` to allow access to all indexes whose names start with `products_`. #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_api_key_indexes)] + #[schema(value_type = Vec, example = json!(["products"]))] pub indexes: Vec, + /// Represent the expiration date and time as RFC 3339 format. `null` equals to no expiration time. #[deserr(error = DeserrJsonError, try_from(Option) = parse_expiration_date -> ParseOffsetDateTimeError, missing_field_error = DeserrJsonError::missing_api_key_expires_at)] pub expires_at: Option, } @@ -86,12 +99,15 @@ fn deny_immutable_fields_api_key( } } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_api_key)] +#[schema(rename_all = "camelCase")] pub struct PatchApiKey { #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = "This key is used to update documents in the products index")] pub description: Setting, #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = "Indexing Products API key")] pub name: Setting, } @@ -179,7 +195,9 @@ fn parse_expiration_date( } } -#[derive(Copy, Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Sequence, Deserr)] +#[derive( + Copy, Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Sequence, Deserr, ToSchema, +)] #[repr(u8)] pub enum Action { #[serde(rename = "*")] diff --git a/crates/meilisearch-types/src/locales.rs b/crates/meilisearch-types/src/locales.rs index 8d746779e..945c38cc3 100644 --- a/crates/meilisearch-types/src/locales.rs +++ b/crates/meilisearch-types/src/locales.rs @@ -1,8 +1,9 @@ use deserr::Deserr; use milli::LocalizedAttributesRule; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; -#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)] #[deserr(rename_all = camelCase)] #[serde(rename_all = "camelCase")] pub struct LocalizedAttributesRuleView { @@ -33,7 +34,7 @@ impl From for LocalizedAttributesRule { /// this enum implements `Deserr` in order to be used in the API. macro_rules! make_locale { ($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)] + #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd, ToSchema)] #[deserr(rename_all = camelCase)] #[serde(rename_all = "camelCase")] pub enum Locale { diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index b12dfc9a2..92d61e28f 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -13,6 +13,7 @@ use milli::proximity::ProximityPrecision; use milli::update::Setting; use milli::{Criterion, CriterionError, Index, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize, Serializer}; +use utoipa::ToSchema; use crate::deserr::DeserrJsonError; use crate::error::deserr_codes::*; @@ -39,10 +40,10 @@ where .serialize(s) } -#[derive(Clone, Default, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Default, Debug, Serialize, PartialEq, Eq, ToSchema)] pub struct Checked; -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, ToSchema)] pub struct Unchecked; impl Deserr for Unchecked @@ -69,54 +70,63 @@ fn validate_min_word_size_for_typo_setting( Ok(s) } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(deny_unknown_fields, rename_all = camelCase, validate = validate_min_word_size_for_typo_setting -> DeserrJsonError)] pub struct MinWordSizeTyposSetting { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option, example = json!(5))] pub one_typo: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option, example = json!(9))] pub two_typos: Setting, } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(deny_unknown_fields, rename_all = camelCase, where_predicate = __Deserr_E: deserr::MergeWithError>)] pub struct TypoSettings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option, example = json!(true))] pub enabled: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!({ "oneTypo": 5, "twoTypo": 9 }))] pub min_word_size_for_typos: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option>, example = json!(["iPhone", "phone"]))] pub disable_on_words: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option>, example = json!(["uuid", "url"]))] pub disable_on_attributes: Setting>, } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(rename_all = camelCase, deny_unknown_fields)] pub struct FacetingSettings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option, example = json!(10))] pub max_values_per_facet: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option>, example = json!({ "genre": FacetValuesSort::Count }))] pub sort_facet_values_by: Setting>, } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(rename_all = camelCase, deny_unknown_fields)] pub struct PaginationSettings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option, example = json!(250))] pub max_total_hits: Setting, } @@ -137,70 +147,105 @@ impl MergeWithError for DeserrJsonError` from a `Settings`. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde( deny_unknown_fields, rename_all = "camelCase", bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>") )] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct Settings { + /// Fields displayed in the returned documents. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["id", "title", "description", "url"]))] pub displayed_attributes: WildcardSetting, - + /// Fields in which to search for matching query words sorted by order of importance. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["title", "description"]))] pub searchable_attributes: WildcardSetting, - + /// Attributes to use for faceting and filtering. See [Filtering and Faceted Search](https://www.meilisearch.com/docs/learn/filtering_and_sorting/search_with_facet_filters). #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["release_date", "genre"]))] pub filterable_attributes: Setting>, + /// Attributes to use when sorting search results. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["release_date"]))] pub sortable_attributes: Setting>, + /// List of ranking rules sorted by order of importance. The order is customizable. + /// [A list of ordered built-in ranking rules](https://www.meilisearch.com/docs/learn/relevancy/relevancy). #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!([RankingRuleView::Words, RankingRuleView::Typo, RankingRuleView::Proximity, RankingRuleView::Attribute, RankingRuleView::Exactness]))] pub ranking_rules: Setting>, + /// List of words ignored when present in search queries. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["the", "a", "them", "their"]))] pub stop_words: Setting>, + /// List of characters not delimiting where one term begins and ends. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!([" ", "\n"]))] pub non_separator_tokens: Setting>, + /// List of characters delimiting where one term begins and ends. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["S"]))] pub separator_tokens: Setting>, + /// List of strings Meilisearch should parse as a single term. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(["iPhone pro"]))] pub dictionary: Setting>, + /// List of associated words treated similarly. A word associated to an array of word as synonyms. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>>, example = json!({ "he": ["she", "they", "them"], "phone": ["iPhone", "android"]}))] pub synonyms: Setting>>, + /// Search returns documents with distinct (different) values of the given field. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!("sku"))] pub distinct_attribute: Setting, + /// Precision level when calculating the proximity ranking rule. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!(ProximityPrecisionView::ByAttribute))] pub proximity_precision: Setting, + /// Customize typo tolerance feature. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!({ "enabled": true, "disableOnAttributes": ["title"]}))] pub typo_tolerance: Setting, + /// Faceting settings. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))] pub faceting: Setting, + /// Pagination settings. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))] pub pagination: Setting, + /// Embedder required for performing meaning-based search queries. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = String)] // TODO: TAMO pub embedders: Setting>>, + /// Maximum duration of a search query. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!(50))] pub search_cutoff_ms: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option>, example = json!(50))] pub localized_attributes: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 64dbd58f7..8a6720cc2 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -7,26 +7,42 @@ use crate::error::ResponseError; use crate::settings::{Settings, Unchecked}; use crate::tasks::{serialize_duration, Details, IndexSwap, Kind, Status, Task, TaskId}; -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct TaskView { + /// The unique sequential identifier of the task. + #[schema(value_type = u32, example = 4312)] pub uid: TaskId, + /// The unique identifier of the index where this task is operated. + #[schema(example = json!("movies"))] pub batch_uid: Option, #[serde(default)] pub index_uid: Option, pub status: Status, + /// The type of the task. #[serde(rename = "type")] pub kind: Kind, + /// The uid of the task that performed the taskCancelation if the task has been canceled. + #[schema(value_type = Option, example = json!(4326))] pub canceled_by: Option, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option, pub error: Option, + /// Total elasped time the engine was in processing state expressed as a `ISO-8601` duration format. + #[schema(value_type = Option, example = json!(null))] #[serde(serialize_with = "serialize_duration", default)] pub duration: Option, + /// An `RFC 3339` format for date/time/duration. + #[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))] #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, + /// An `RFC 3339` format for date/time/duration. + #[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))] #[serde(with = "time::serde::rfc3339::option", default)] pub started_at: Option, + /// An `RFC 3339` format for date/time/duration. + #[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))] #[serde(with = "time::serde::rfc3339::option", default)] pub finished_at: Option, } @@ -53,32 +69,44 @@ impl TaskView { #[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct DetailsView { + /// Number of documents received for documentAdditionOrUpdate task. #[serde(skip_serializing_if = "Option::is_none")] pub received_documents: Option, + /// Number of documents finally indexed for documentAdditionOrUpdate task or a documentAdditionOrUpdate batch of tasks. #[serde(skip_serializing_if = "Option::is_none")] pub indexed_documents: Option>, + /// Number of documents edited for editDocumentByFunction task. #[serde(skip_serializing_if = "Option::is_none")] pub edited_documents: Option>, + /// Value for the primaryKey field encountered if any for indexCreation or indexUpdate task. #[serde(skip_serializing_if = "Option::is_none")] pub primary_key: Option>, + /// Number of provided document ids for the documentDeletion task. #[serde(skip_serializing_if = "Option::is_none")] pub provided_ids: Option, + /// Number of documents finally deleted for documentDeletion and indexDeletion tasks. #[serde(skip_serializing_if = "Option::is_none")] pub deleted_documents: Option>, + /// Number of tasks that match the request for taskCancelation or taskDeletion tasks. #[serde(skip_serializing_if = "Option::is_none")] pub matched_tasks: Option, + /// Number of tasks canceled for taskCancelation. #[serde(skip_serializing_if = "Option::is_none")] pub canceled_tasks: Option>, + /// Number of tasks deleted for taskDeletion. #[serde(skip_serializing_if = "Option::is_none")] pub deleted_tasks: Option>, + /// Original filter query for taskCancelation or taskDeletion tasks. #[serde(skip_serializing_if = "Option::is_none")] pub original_filter: Option>, + /// Identifier generated for the dump for dumpCreation task. #[serde(skip_serializing_if = "Option::is_none")] pub dump_uid: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub context: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub function: Option, + /// [Learn more about the settings in this guide](https://www.meilisearch.com/docs/reference/api/settings). #[serde(skip_serializing_if = "Option::is_none")] #[serde(flatten)] pub settings: Option>>, diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index c62f550ae..7960951ed 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -9,6 +9,7 @@ use milli::Object; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize, Serializer}; use time::{Duration, OffsetDateTime}; +use utoipa::ToSchema; use uuid::Uuid; use crate::batches::BatchId; @@ -151,7 +152,7 @@ pub enum KindWithContent { SnapshotCreation, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), @@ -363,9 +364,22 @@ impl From<&KindWithContent> for Option
{ } } +/// The status of a task. #[derive( - Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Sequence, PartialOrd, Ord, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + Sequence, + PartialOrd, + Ord, + ToSchema, )] +#[schema(example = json!(Status::Processing))] #[serde(rename_all = "camelCase")] pub enum Status { Enqueued, @@ -424,10 +438,23 @@ impl fmt::Display for ParseTaskStatusError { } impl std::error::Error for ParseTaskStatusError {} +/// The type of the task. #[derive( - Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Sequence, PartialOrd, Ord, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + Sequence, + PartialOrd, + Ord, + ToSchema, )] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase", example = json!(enum_iterator::all::().collect::>()))] pub enum Kind { DocumentAdditionOrUpdate, DocumentEdition, @@ -444,6 +471,10 @@ pub enum Kind { } impl Kind { + pub fn all_variants() -> Vec { + enum_iterator::all::().collect() + } + pub fn related_to_one_index(&self) -> bool { match self { Kind::DocumentAdditionOrUpdate diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 68ca8e136..8a2f0c6c0 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -105,6 +105,10 @@ tracing-actix-web = "0.7.11" build-info = { version = "1.7.0", path = "../build-info" } roaring = "0.10.7" mopa-maintained = "0.2.3" +utoipa = { version = "5.0.0-rc.0", features = ["actix_extras", "macros", "non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } +utoipa-scalar = { version = "0.2.0-rc.0", features = ["actix-web"] } +utoipa-rapidoc = { version = "4.0.1-rc.0", features = ["actix-web"] } +utoipa-redoc = { version = "4.0.1-rc.0", features = ["actix-web"] } [dev-dependencies] actix-rt = "2.10.0" diff --git a/crates/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs index 0bd4b9d59..3309c1f6e 100644 --- a/crates/meilisearch/src/routes/api_key.rs +++ b/crates/meilisearch/src/routes/api_key.rs @@ -13,6 +13,7 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::keys::{CreateApiKey, Key, PatchApiKey}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; +use utoipa::{IntoParams, OpenApi, ToSchema}; use uuid::Uuid; use super::PAGINATION_DEFAULT_LIMIT; @@ -21,6 +22,20 @@ use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; use crate::routes::Pagination; +#[derive(OpenApi)] +#[openapi( + paths(create_api_key, list_api_keys, get_api_key, patch_api_key, delete_api_key), + tags(( + name = "Keys", + description = "Manage API `keys` for a Meilisearch instance. Each key has a given set of permissions. +You must have the master key or the default admin key to access the keys route. More information about the keys and their rights. +Accessing any route under `/keys` without having set a master key will result in an error.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/keys"), + + )), +)] +pub struct ApiKeyApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -35,6 +50,52 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } + +/// Create an API Key +/// +/// Create an API Key. +#[utoipa::path( + post, + path = "/", + tag = "Keys", + security(("Bearer" = ["keys.create", "keys.*", "*"])), + request_body = CreateApiKey, + responses( + (status = 202, description = "Key has been created", body = KeyView, content_type = "application/json", example = json!( + { + "uid": "01b4bc42-eb33-4041-b481-254d00cce834", + "key": "d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4", + "name": "Indexing Products API key", + "description": null, + "actions": [ + "documents.add" + ], + "indexes": [ + "products" + ], + "expiresAt": "2021-11-13T00:00:00Z", + "createdAt": "2021-11-12T10:00:00Z", + "updatedAt": "2021-11-12T10:00:00Z" + } + )), + (status = 401, description = "The route has been hit on an unprotected instance", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Meilisearch is running without a master key. To access this API endpoint, you must have set a master key at launch.", + "code": "missing_master_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_master_key" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn create_api_key( auth_controller: GuardedData, Data>, body: AwebJson, @@ -51,11 +112,14 @@ pub async fn create_api_key( Ok(HttpResponse::Created().json(res)) } -#[derive(Deserr, Debug, Clone, Copy)] +#[derive(Deserr, Debug, Clone, Copy, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct ListApiKeys { + #[into_params(value_type = usize, default = 0)] #[deserr(default, error = DeserrQueryParamError)] pub offset: Param, + #[into_params(value_type = usize, default = PAGINATION_DEFAULT_LIMIT)] #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError)] pub limit: Param, } @@ -66,6 +130,60 @@ impl ListApiKeys { } } + +/// Get API Keys +/// +/// List all API Keys +/// TODO: Tamo fix the return type +#[utoipa::path( + get, + path = "/", + tag = "Keys", + security(("Bearer" = ["keys.get", "keys.*", "*"])), + params(ListApiKeys), + responses( + (status = 202, description = "List of keys", body = serde_json::Value, content_type = "application/json", example = json!( + { + "results": [ + { + "uid": "01b4bc42-eb33-4041-b481-254d00cce834", + "key": "d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4", + "name": "An API Key", + "description": null, + "actions": [ + "documents.add" + ], + "indexes": [ + "movies" + ], + "expiresAt": "2022-11-12T10:00:00Z", + "createdAt": "2021-11-12T10:00:00Z", + "updatedAt": "2021-11-12T10:00:00Z" + } + ], + "limit": 20, + "offset": 0, + "total": 1 + } + )), + (status = 401, description = "The route has been hit on an unprotected instance", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Meilisearch is running without a master key. To access this API endpoint, you must have set a master key at launch.", + "code": "missing_master_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_master_key" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn list_api_keys( auth_controller: GuardedData, Data>, list_api_keys: AwebQueryParameter, @@ -84,6 +202,52 @@ pub async fn list_api_keys( Ok(HttpResponse::Ok().json(page_view)) } + +/// Get an API Key +/// +/// Get an API key from its `uid` or its `key` field. +#[utoipa::path( + get, + path = "/{key}", + tag = "Keys", + security(("Bearer" = ["keys.get", "keys.*", "*"])), + params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), + responses( + (status = 200, description = "The key is returned", body = KeyView, content_type = "application/json", example = json!( + { + "uid": "01b4bc42-eb33-4041-b481-254d00cce834", + "key": "d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4", + "name": "An API Key", + "description": null, + "actions": [ + "documents.add" + ], + "indexes": [ + "movies" + ], + "expiresAt": "2022-11-12T10:00:00Z", + "createdAt": "2021-11-12T10:00:00Z", + "updatedAt": "2021-11-12T10:00:00Z" + } + )), + (status = 401, description = "The route has been hit on an unprotected instance", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Meilisearch is running without a master key. To access this API endpoint, you must have set a master key at launch.", + "code": "missing_master_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_master_key" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_api_key( auth_controller: GuardedData, Data>, path: web::Path, @@ -103,6 +267,55 @@ pub async fn get_api_key( Ok(HttpResponse::Ok().json(res)) } + +/// Update an API Key +/// +/// Update an API key from its `uid` or its `key` field. +/// Only the `name` and `description` of the api key can be updated. +/// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. +#[utoipa::path( + patch, + path = "/{key}", + tag = "Keys", + security(("Bearer" = ["keys.update", "keys.*", "*"])), + params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), + request_body = PatchApiKey, + responses( + (status = 200, description = "The key have been updated", body = KeyView, content_type = "application/json", example = json!( + { + "uid": "01b4bc42-eb33-4041-b481-254d00cce834", + "key": "d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4", + "name": "An API Key", + "description": null, + "actions": [ + "documents.add" + ], + "indexes": [ + "movies" + ], + "expiresAt": "2022-11-12T10:00:00Z", + "createdAt": "2021-11-12T10:00:00Z", + "updatedAt": "2021-11-12T10:00:00Z" + } + )), + (status = 401, description = "The route has been hit on an unprotected instance", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Meilisearch is running without a master key. To access this API endpoint, you must have set a master key at launch.", + "code": "missing_master_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_master_key" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn patch_api_key( auth_controller: GuardedData, Data>, body: AwebJson, @@ -123,6 +336,39 @@ pub async fn patch_api_key( Ok(HttpResponse::Ok().json(res)) } + + +/// Update an API Key +/// +/// Update an API key from its `uid` or its `key` field. +/// Only the `name` and `description` of the api key can be updated. +/// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. +#[utoipa::path( + delete, + path = "/{key}", + tag = "Keys", + security(("Bearer" = ["keys.delete", "keys.*", "*"])), + params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), + responses( + (status = NO_CONTENT, description = "The key have been removed"), + (status = 401, description = "The route has been hit on an unprotected instance", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Meilisearch is running without a master key. To access this API endpoint, you must have set a master key at launch.", + "code": "missing_master_key", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_master_key" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn delete_api_key( auth_controller: GuardedData, Data>, path: web::Path, @@ -144,19 +390,30 @@ pub struct AuthParam { key: String, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] -struct KeyView { +pub(super) struct KeyView { + /// The name of the API Key if any name: Option, + /// The description of the API Key if any description: Option, + /// The actual API Key you can send to Meilisearch key: String, + /// The `Uuid` specified while creating the key or autogenerated by Meilisearch. uid: Uuid, + /// The actions accessible with this key. actions: Vec, + /// The indexes accessible with this key. indexes: Vec, + /// The expiration date of the key. Once this timestamp is exceeded the key is not deleted but cannot be used anymore. #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] expires_at: Option, + /// The date of creation of this API Key. + #[schema(read_only)] #[serde(serialize_with = "time::serde::rfc3339::serialize")] created_at: OffsetDateTime, + /// The date of the last update made on this key. + #[schema(read_only)] #[serde(serialize_with = "time::serde::rfc3339::serialize")] updated_at: OffsetDateTime, } diff --git a/crates/meilisearch/src/routes/dump.rs b/crates/meilisearch/src/routes/dump.rs index c78dc4dad..37f06d4c6 100644 --- a/crates/meilisearch/src/routes/dump.rs +++ b/crates/meilisearch/src/routes/dump.rs @@ -5,6 +5,7 @@ use meilisearch_auth::AuthController; use meilisearch_types::error::ResponseError; use meilisearch_types::tasks::KindWithContent; use tracing::debug; +use utoipa::OpenApi; use crate::analytics::Analytics; use crate::extractors::authentication::policies::*; @@ -13,12 +14,61 @@ use crate::extractors::sequential_extractor::SeqHandler; use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView}; use crate::Opt; +#[derive(OpenApi)] +#[openapi( + paths(create_dump), + tags(( + name = "Dumps", + description = "The `dumps` route allows the creation of database dumps. +Dumps are `.dump` files that can be used to launch Meilisearch. Dumps are compatible between Meilisearch versions. +Creating a dump is also referred to as exporting it, whereas launching Meilisearch with a dump is referred to as importing it. +During a [dump export](https://www.meilisearch.com/docs/reference/api/dump#create-a-dump), all indexes of the current instance are +exported—together with their documents and settings—and saved as a single `.dump` file. During a dump import, +all indexes contained in the indicated `.dump` file are imported along with their associated documents and settings. +Any existing index with the same uid as an index in the dump file will be overwritten. +Dump imports are [performed at launch](https://www.meilisearch.com/docs/learn/advanced/dumps#importing-a-dump) using an option.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/dump"), + + )), +)] +pub struct DumpApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))); } crate::empty_analytics!(DumpAnalytics, "Dump Created"); +/// Create a dump +/// +/// Triggers a dump creation process. Once the process is complete, a dump is created in the +/// [dump directory](https://www.meilisearch.com/docs/learn/self_hosted/configure_meilisearch_at_launch#dump-directory). +/// If the dump directory does not exist yet, it will be created. +#[utoipa::path( + post, + path = "/", + tag = "Dumps", + security(("Bearer" = ["dumps.create", "dumps.*", "*"])), + responses( + (status = 202, description = "Dump is being created", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 0, + "indexUid": null, + "status": "enqueued", + "type": "DumpCreation", + "enqueuedAt": "2021-01-01T09:39:00.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn create_dump( index_scheduler: GuardedData, Data>, auth_controller: GuardedData, Data>, diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 3b9a89885..205b71420 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -31,6 +31,7 @@ use tempfile::tempfile; use tokio::fs::File; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter}; use tracing::debug; +use utoipa::{IntoParams, OpenApi, ToSchema}; use crate::analytics::{Aggregate, AggregateMethod, Analytics}; use crate::error::MeilisearchHttpError; @@ -71,6 +72,19 @@ pub struct DocumentParam { document_id: String, } +#[derive(OpenApi)] +#[openapi( + paths(get_documents, replace_documents, update_documents, clear_all_documents, delete_documents_batch), + tags( + ( + name = "Documents", + description = "Documents are objects composed of fields that can store any type of data. Each field contains an attribute and its associated value. Documents are stored inside [indexes](https://www.meilisearch.com/docs/learn/getting_started/indexes).", + external_docs(url = "https://www.meilisearch.com/docs/learn/getting_started/documents"), + ), + ), +)] +pub struct DocumentsApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -286,17 +300,23 @@ pub struct BrowseQueryGet { filter: Option, } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct BrowseQuery { + #[schema(default, example = 150)] #[deserr(default, error = DeserrJsonError)] offset: usize, + #[schema(default = 20, example = 1)] #[deserr(default = PAGINATION_DEFAULT_LIMIT, error = DeserrJsonError)] limit: usize, + #[schema(example = json!(["title, description"]))] #[deserr(default, error = DeserrJsonError)] fields: Option>, + #[schema(default, example = true)] #[deserr(default, error = DeserrJsonError)] retrieve_vectors: bool, + #[schema(default, example = "popularity > 1000")] #[deserr(default, error = DeserrJsonError)] filter: Option, } @@ -326,6 +346,62 @@ pub async fn documents_by_query_post( documents_by_query(&index_scheduler, index_uid, body) } +/// Get documents +/// +/// Get documents by batches. +#[utoipa::path( + get, + path = "/{indexUid}/documents", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.get", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + // Here we can use the post version of the browse query since it contains the exact same parameter + BrowseQuery + ), + responses( + // body = PaginationView + (status = 200, description = "The documents are returned", body = serde_json::Value, content_type = "application/json", example = json!( + { + "results": [ + { + "id": 25684, + "title": "American Ninja 5", + "poster": "https://image.tmdb.org/t/p/w1280/iuAQVI4mvjI83wnirpD8GVNRVuY.jpg", + "overview": "When a scientists daughter is kidnapped, American Ninja, attempts to find her, but this time he teams up with a youngster he has trained in the ways of the ninja.", + "release_date": 725846400 + }, + { + "id": 45881, + "title": "The Bridge of San Luis Rey", + "poster": "https://image.tmdb.org/t/p/w500/4X7quIcdkc24Cveg5XdpfRqxtYA.jpg", + "overview": "The Bridge of San Luis Rey is American author Thornton Wilder's second novel, first published in 1927 to worldwide acclaim. It tells the story of several interrelated people who die in the collapse of an Inca rope-fiber suspension bridge in Peru, and the events that lead up to their being on the bridge.[ A friar who has witnessed the tragic accident then goes about inquiring into the lives of the victims, seeking some sort of cosmic answer to the question of why each had to die. The novel won the Pulitzer Prize in 1928.", + "release_date": 1072915200 + } + ], + "limit": 20, + "offset": 0, + "total": 2 + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_documents( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -396,11 +472,17 @@ fn documents_by_query( Ok(HttpResponse::Ok().json(ret)) } -#[derive(Deserialize, Debug, Deserr)] +#[derive(Deserialize, Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase")] pub struct UpdateDocumentsQuery { + /// The primary key of the documents. primaryKey is optional. If you want to set the primary key of your index through this route, + /// it only has to be done the first time you add documents to the index. After which it will be ignored if given. + #[param(example = "id")] #[deserr(default, error = DeserrQueryParamError)] pub primary_key: Option, + /// Customize the csv delimiter when importing CSV documents. + #[param(value_type = char, default = ",", example = ";")] #[deserr(default, try_from(char) = from_char_csv_delimiter -> DeserrQueryParamError, error = DeserrQueryParamError)] pub csv_delimiter: Option, } @@ -451,6 +533,51 @@ impl Aggregate for DocumentsAggregator { } } +/// Add or replace documents +/// +/// Add a list of documents or replace them if they already exist. +/// +/// If you send an already existing document (same id) the whole existing document will be overwritten by the new document. Fields previously in the document not present in the new document are removed. +/// +/// For a partial update of the document see Add or update documents route. +/// > info +/// > If the provided index does not exist, it will be created. +/// > info +/// > Use the reserved `_geo` object to add geo coordinates to a document. `_geo` is an object made of `lat` and `lng` field. +/// > +/// > When the vectorStore feature is enabled you can use the reserved `_vectors` field in your documents. +/// > It can accept an array of floats, multiple arrays of floats in an outer array or an object. +/// > This object accepts keys corresponding to the different embedders defined your index settings. +#[utoipa::path( + post, + path = "/{indexUid}/documents", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.add", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + // Here we can use the post version of the browse query since it contains the exact same parameter + UpdateDocumentsQuery, + ), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentAdditionOrUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn replace_documents( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -508,6 +635,49 @@ pub async fn replace_documents( Ok(HttpResponse::Accepted().json(task)) } +/// Add or update documents +/// +/// Add a list of documents or update them if they already exist. +/// If you send an already existing document (same id) the old document will be only partially updated according to the fields of the new document. Thus, any fields not present in the new document are kept and remained unchanged. +/// To completely overwrite a document, see Add or replace documents route. +/// > info +/// > If the provided index does not exist, it will be created. +/// > info +/// > Use the reserved `_geo` object to add geo coordinates to a document. `_geo` is an object made of `lat` and `lng` field. +/// > +/// > When the vectorStore feature is enabled you can use the reserved `_vectors` field in your documents. +/// > It can accept an array of floats, multiple arrays of floats in an outer array or an object. +/// > This object accepts keys corresponding to the different embedders defined your index settings. +#[utoipa::path( + put, + path = "/{indexUid}/documents", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.add", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + // Here we can use the post version of the browse query since it contains the exact same parameter + UpdateDocumentsQuery, + ), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentAdditionOrUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn update_documents( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -742,6 +912,38 @@ async fn copy_body_to_file( Ok(read_file) } +/// Delete documents +/// +/// Delete a selection of documents based on array of document id's. +#[utoipa::path( + delete, + path = "/{indexUid}/documents", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.delete", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + ), + // TODO: how to return an array of strings + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentAdditionOrUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn delete_documents_batch( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -936,6 +1138,38 @@ pub async fn edit_documents_by_function( Ok(HttpResponse::Accepted().json(task)) } +/// Delete all documents +/// +/// Delete all documents in the specified index. +#[utoipa::path( + delete, + path = "/{indexUid}/documents", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.delete", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + UpdateDocumentsQuery, + ), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentDeletion", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn clear_all_documents( index_scheduler: GuardedData, Data>, index_uid: web::Path, diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 26a6569e7..f6881f70c 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -16,6 +16,7 @@ use meilisearch_types::tasks::KindWithContent; use serde::Serialize; use time::OffsetDateTime; use tracing::debug; +use utoipa::{IntoParams, OpenApi, ToSchema}; use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT}; use crate::analytics::{Aggregate, Analytics}; @@ -36,6 +37,22 @@ mod settings_analytics; pub mod similar; mod similar_analytics; +#[derive(OpenApi)] +#[openapi( + nest( + (path = "/", api = documents::DocumentsApi), + ), + paths(list_indexes, create_index, get_index, update_index, delete_index, get_index_stats), + tags( + ( + name = "Indexes", + description = "An index is an entity that gathers a set of [documents](https://www.meilisearch.com/docs/learn/getting_started/documents) with its own [settings](https://www.meilisearch.com/docs/reference/api/settings). Learn more about indexes.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/indexes"), + ), + ), +)] +pub struct IndexesApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -59,14 +76,18 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] pub struct IndexView { + /// Unique identifier for the index pub uid: String, + /// An `RFC 3339` format for date/time/duration. #[serde(with = "time::serde::rfc3339")] pub created_at: OffsetDateTime, + /// An `RFC 3339` format for date/time/duration. #[serde(with = "time::serde::rfc3339")] pub updated_at: OffsetDateTime, + /// Custom primaryKey for documents pub primary_key: Option, } @@ -84,20 +105,61 @@ impl IndexView { } } -#[derive(Deserr, Debug, Clone, Copy)] +#[derive(Deserr, Debug, Clone, Copy, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase")] pub struct ListIndexes { + /// The number of indexes to skip before starting to retrieve anything + #[param(value_type = Option, default, example = 100)] #[deserr(default, error = DeserrQueryParamError)] pub offset: Param, + /// The number of indexes to retrieve + #[param(value_type = Option, default = 20, example = 1)] #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError)] pub limit: Param, } + impl ListIndexes { fn as_pagination(self) -> Pagination { Pagination { offset: self.offset.0, limit: self.limit.0 } } } +/// List indexes +/// +/// List all indexes. +#[utoipa::path( + get, + path = "/", + tag = "Indexes", + security(("Bearer" = ["indexes.get", "indexes.*", "*"])), + params(ListIndexes), + responses( + (status = 200, description = "Indexes are returned", body = serde_json::Value, content_type = "application/json", example = json!( + { + "results": [ + { + "uid": "movies", + "primaryKey": "movie_id", + "createdAt": "2019-11-20T09:40:33.711324Z", + "updatedAt": "2019-11-20T09:40:33.711324Z" + } + ], + "limit": 1, + "offset": 0, + "total": 1 + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn list_indexes( index_scheduler: GuardedData, Data>, paginate: AwebQueryParameter, @@ -121,11 +183,16 @@ pub async fn list_indexes( Ok(HttpResponse::Ok().json(ret)) } -#[derive(Deserr, Debug)] +#[derive(Deserr, Debug, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct IndexCreateRequest { + /// The name of the index + #[schema(example = "movies")] #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_index_uid)] uid: IndexUid, + /// The primary key of the index + #[schema(example = "id")] #[deserr(default, error = DeserrJsonError)] primary_key: Option, } @@ -149,6 +216,35 @@ impl Aggregate for IndexCreatedAggregate { } } +/// Create index +/// +/// Create an index. +#[utoipa::path( + post, + path = "/", + tag = "Indexes", + security(("Bearer" = ["indexes.create", "indexes.*", "*"])), + request_body = IndexCreateRequest, + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "indexCreation", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn create_index( index_scheduler: GuardedData, Data>, body: AwebJson, @@ -198,13 +294,42 @@ fn deny_immutable_fields_index( } } -#[derive(Deserr, Debug)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_index)] -pub struct UpdateIndexRequest { - #[deserr(default, error = DeserrJsonError)] - primary_key: Option, -} - +/// Get index +/// +/// Get information about an index. +#[utoipa::path( + get, + path = "/{indexUid}", + tag = "Indexes", + security(("Bearer" = ["indexes.get", "indexes.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + responses( + (status = 200, description = "The index is returned", body = IndexView, content_type = "application/json", example = json!( + { + "uid": "movies", + "primaryKey": "movie_id", + "createdAt": "2019-11-20T09:40:33.711324Z", + "updatedAt": "2019-11-20T09:40:33.711324Z" + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_index( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -237,6 +362,48 @@ impl Aggregate for IndexUpdatedAggregate { serde_json::to_value(*self).unwrap_or_default() } } + +#[derive(Deserr, Debug, ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_index)] +#[schema(rename_all = "camelCase")] +pub struct UpdateIndexRequest { + /// The new primary key of the index + #[deserr(default, error = DeserrJsonError)] + primary_key: Option, +} + +/// Update index +/// +/// Update the `primaryKey` of an index. +/// Return an error if the index doesn't exists yet or if it contains documents. +#[utoipa::path( + patch, + path = "/{indexUid}", + tag = "Indexes", + security(("Bearer" = ["indexes.update", "indexes.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + request_body = UpdateIndexRequest, + responses( + (status = ACCEPTED, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 0, + "indexUid": "movies", + "status": "enqueued", + "type": "indexUpdate", + "enqueuedAt": "2021-01-01T09:39:00.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +>>>>>>> 0f289a437 (Implements the get and delete tasks route):meilisearch/src/routes/indexes/mod.rs pub async fn update_index( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -269,6 +436,35 @@ pub async fn update_index( Ok(HttpResponse::Accepted().json(task)) } +/// Delete index +/// +/// Delete an index. +#[utoipa::path( + delete, + path = "/{indexUid}", + tag = "Indexes", + security(("Bearer" = ["indexes.delete", "indexes.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + responses( + (status = ACCEPTED, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 0, + "indexUid": "movies", + "status": "enqueued", + "type": "indexDeletion", + "enqueuedAt": "2021-01-01T09:39:00.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn delete_index( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -289,14 +485,15 @@ pub async fn delete_index( } /// Stats of an `Index`, as known to the `stats` route. -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, ToSchema)] #[serde(rename_all = "camelCase")] pub struct IndexStats { /// Number of documents in the index pub number_of_documents: u64, - /// Whether the index is currently performing indexation, according to the scheduler. + /// Whether or not the index is currently ingesting document pub is_indexing: bool, /// Association of every field name with the number of times it occurs in the documents. + #[schema(value_type = HashMap)] pub field_distribution: FieldDistribution, } @@ -310,6 +507,44 @@ impl From for IndexStats { } } +/// Get stats of index +/// +/// Get the stats of an index. +#[utoipa::path( + get, + path = "/{indexUid}/stats", + tags = ["Indexes", "Stats"], + security(("Bearer" = ["stats.get", "stats.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + responses( + (status = OK, description = "The stats of the index", body = IndexStats, content_type = "application/json", example = json!( + { + "numberOfDocuments": 10, + "isIndexing": true, + "fieldDistribution": { + "genre": 10, + "author": 9 + } + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_index_stats( index_scheduler: GuardedData, Data>, index_uid: web::Path, diff --git a/crates/meilisearch/src/routes/logs.rs b/crates/meilisearch/src/routes/logs.rs index 57e2cbd22..dc6b6d14c 100644 --- a/crates/meilisearch/src/routes/logs.rs +++ b/crates/meilisearch/src/routes/logs.rs @@ -14,9 +14,11 @@ use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::deserr_codes::*; use meilisearch_types::error::{Code, ResponseError}; +use serde::Serialize; use tokio::sync::mpsc; use tracing_subscriber::filter::Targets; use tracing_subscriber::Layer; +use utoipa::{OpenApi, ToSchema}; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::*; @@ -24,6 +26,20 @@ use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; use crate::{LogRouteHandle, LogStderrHandle}; + +#[derive(OpenApi)] +#[openapi( + paths(get_logs, cancel_logs, update_stderr_target), + tags(( + name = "Logs", + description = "Everything about retrieving or customizing logs. +Currently [experimental](https://www.meilisearch.com/docs/learn/experimental/overview).", + external_docs(url = "https://www.meilisearch.com/docs/learn/experimental/log_customization"), + + )), +)] +pub struct LogsApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("stream") @@ -33,12 +49,16 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("stderr").route(web::post().to(SeqHandler(update_stderr_target)))); } -#[derive(Debug, Default, Clone, Copy, Deserr, PartialEq, Eq)] +#[derive(Debug, Default, Clone, Copy, Deserr, Serialize, PartialEq, Eq, ToSchema)] #[deserr(rename_all = camelCase)] +#[schema(rename_all = "camelCase")] pub enum LogMode { + /// Output the logs in a human readable form. #[default] Human, + /// Output the logs in json. Json, + /// Output the logs in the firefox profiler format. They can then be loaded and visualized at https://profiler.firefox.com/ Profile, } @@ -83,16 +103,26 @@ impl MergeWithError for DeserrJsonError { } } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields, validate = validate_get_logs -> DeserrJsonError)] +#[schema(rename_all = "camelCase")] pub struct GetLogs { + /// Lets you specify which parts of the code you want to inspect and is formatted like that: code_part=log_level,code_part=log_level + /// - If the `code_part` is missing, then the `log_level` will be applied to everything. + /// - If the `log_level` is missing, then the `code_part` will be selected in `info` log level. #[deserr(default = "info".parse().unwrap(), try_from(&String) = MyTargets::from_str -> DeserrJsonError)] + #[schema(value_type = String, default = "info", example = json!("milli=trace,index_scheduler,actix_web=off"))] target: MyTargets, + /// Lets you customize the format of the logs. #[deserr(default, error = DeserrJsonError)] + #[schema(default = LogMode::default)] mode: LogMode, + /// A boolean to indicate if you want to profile the memory as well. This is only useful while using the `profile` mode. + /// Be cautious, though; it slows down the engine a lot. #[deserr(default = false, error = DeserrJsonError)] + #[schema(default = false)] profile_memory: bool, } @@ -248,6 +278,46 @@ fn entry_stream( ) } +/// Retrieve logs +/// +/// Stream logs over HTTP. The format of the logs depends on the configuration specified in the payload. +/// The logs are sent as multi-part, and the stream never stops, so make sure your clients correctly handle that. +/// To make the server stop sending you logs, you can call the `DELETE /logs/stream` route. +/// +/// There can only be one listener at a timeand an error will be returned if you call this route while it's being used by another client. +#[utoipa::path( + post, + path = "/stream", + tag = "Logs", + security(("Bearer" = ["metrics.get", "metrics.*", "*"])), + request_body = GetLogs, + responses( + (status = OK, description = "Logs are being returned", body = String, content_type = "application/json", example = json!( + r#" +2024-10-08T13:35:02.643750Z WARN HTTP request{method=GET host="localhost:7700" route=/metrics query_parameters= user_agent=HTTPie/3.2.3 status_code=400 error=Getting metrics requires enabling the `metrics` experimental feature. See https://github.com/meilisearch/product/discussions/625}: tracing_actix_web::middleware: Error encountered while processing the incoming HTTP request: ResponseError { code: 400, message: "Getting metrics requires enabling the `metrics` experimental feature. See https://github.com/meilisearch/product/discussions/625", error_code: "feature_not_enabled", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#feature_not_enabled" } +2024-10-08T13:35:02.644191Z INFO HTTP request{method=GET host="localhost:7700" route=/metrics query_parameters= user_agent=HTTPie/3.2.3 status_code=400 error=Getting metrics requires enabling the `metrics` experimental feature. See https://github.com/meilisearch/product/discussions/625}: meilisearch: close time.busy=1.66ms time.idle=658µs +2024-10-08T13:35:18.564152Z INFO HTTP request{method=PATCH host="localhost:7700" route=/experimental-features query_parameters= user_agent=curl/8.6.0 status_code=200}: meilisearch: close time.busy=1.17ms time.idle=127µs +2024-10-08T13:35:23.094987Z INFO HTTP request{method=GET host="localhost:7700" route=/metrics query_parameters= user_agent=HTTPie/3.2.3 status_code=200}: meilisearch: close time.busy=2.12ms time.idle=595µs +"# + )), + (status = 400, description = "The route is already being used", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The `/logs/stream` route is currently in use by someone else.", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_logs( index_scheduler: GuardedData, Data>, logs: Data, @@ -280,6 +350,27 @@ pub async fn get_logs( } } + +/// Stop retrieving logs +/// +/// Call this route to make the engine stops sending logs through the `POST /logs/stream` route. +#[utoipa::path( + delete, + path = "/stream", + tag = "Logs", + security(("Bearer" = ["metrics.get", "metrics.*", "*"])), + responses( + (status = NO_CONTENT, description = "Logs are being returned"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn cancel_logs( index_scheduler: GuardedData, Data>, logs: Data, @@ -293,13 +384,38 @@ pub async fn cancel_logs( Ok(HttpResponse::NoContent().finish()) } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct UpdateStderrLogs { + /// Lets you specify which parts of the code you want to inspect and is formatted like that: code_part=log_level,code_part=log_level + /// - If the `code_part` is missing, then the `log_level` will be applied to everything. + /// - If the `log_level` is missing, then the `code_part` will be selected in `info` log level. #[deserr(default = "info".parse().unwrap(), try_from(&String) = MyTargets::from_str -> DeserrJsonError)] + #[schema(value_type = String, default = "info", example = json!("milli=trace,index_scheduler,actix_web=off"))] target: MyTargets, } +/// Update target of the console logs +/// +/// This route lets you specify at runtime the level of the console logs outputted on stderr. +#[utoipa::path( + post, + path = "/stderr", + tag = "Logs", + request_body = UpdateStderrLogs, + security(("Bearer" = ["metrics.get", "metrics.*", "*"])), + responses( + (status = NO_CONTENT, description = "The console logs have been updated"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn update_stderr_target( index_scheduler: GuardedData, Data>, logs: Data, diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 191beba8c..bf5d6741c 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -1,12 +1,17 @@ +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::{AuthenticationError, GuardedData}; +use crate::routes::create_all_stats; +use crate::search_queue::SearchQueue; use actix_web::http::header; use actix_web::web::{self, Data}; use actix_web::HttpResponse; -use index_scheduler::{IndexScheduler, Query}; +use index_scheduler::IndexScheduler; use meilisearch_auth::AuthController; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; -use meilisearch_types::tasks::Status; use prometheus::{Encoder, TextEncoder}; +use utoipa::OpenApi; + use time::OffsetDateTime; use crate::extractors::authentication::policies::ActionPolicy; @@ -14,10 +19,100 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::routes::create_all_stats; use crate::search_queue::SearchQueue; +#[derive(OpenApi)] +#[openapi(paths(get_metrics))] +pub struct MetricApi; + pub fn configure(config: &mut web::ServiceConfig) { config.service(web::resource("").route(web::get().to(get_metrics))); } +/// Get prometheus metrics +/// +/// Retrieve metrics on the engine. See https://www.meilisearch.com/docs/learn/experimental/metrics +/// Currently, [the feature is experimental](https://www.meilisearch.com/docs/learn/experimental/overview) +/// which means it must be enabled. +#[utoipa::path( + get, + path = "/", + tag = "Stats", + security(("Bearer" = ["metrics.get", "metrics.*", "*"])), + responses( + (status = 200, description = "The metrics of the instance", body = String, content_type = "text/plain", example = json!( + r#" +# HELP meilisearch_db_size_bytes Meilisearch DB Size In Bytes +# TYPE meilisearch_db_size_bytes gauge +meilisearch_db_size_bytes 1130496 +# HELP meilisearch_http_requests_total Meilisearch HTTP requests total +# TYPE meilisearch_http_requests_total counter +meilisearch_http_requests_total{method="GET",path="/metrics",status="400"} 1 +meilisearch_http_requests_total{method="PATCH",path="/experimental-features",status="200"} 1 +# HELP meilisearch_http_response_time_seconds Meilisearch HTTP response times +# TYPE meilisearch_http_response_time_seconds histogram +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.005"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.01"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.025"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.05"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.075"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.1"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.25"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.5"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="0.75"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="1"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="2.5"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="5"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="7.5"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="10"} 0 +meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="+Inf"} 0 +meilisearch_http_response_time_seconds_sum{method="GET",path="/metrics"} 0 +meilisearch_http_response_time_seconds_count{method="GET",path="/metrics"} 0 +# HELP meilisearch_index_count Meilisearch Index Count +# TYPE meilisearch_index_count gauge +meilisearch_index_count 1 +# HELP meilisearch_index_docs_count Meilisearch Index Docs Count +# TYPE meilisearch_index_docs_count gauge +meilisearch_index_docs_count{index="mieli"} 2 +# HELP meilisearch_is_indexing Meilisearch Is Indexing +# TYPE meilisearch_is_indexing gauge +meilisearch_is_indexing 0 +# HELP meilisearch_last_update Meilisearch Last Update +# TYPE meilisearch_last_update gauge +meilisearch_last_update 1726675964 +# HELP meilisearch_nb_tasks Meilisearch Number of tasks +# TYPE meilisearch_nb_tasks gauge +meilisearch_nb_tasks{kind="indexes",value="mieli"} 39 +meilisearch_nb_tasks{kind="statuses",value="canceled"} 0 +meilisearch_nb_tasks{kind="statuses",value="enqueued"} 0 +meilisearch_nb_tasks{kind="statuses",value="failed"} 4 +meilisearch_nb_tasks{kind="statuses",value="processing"} 0 +meilisearch_nb_tasks{kind="statuses",value="succeeded"} 35 +meilisearch_nb_tasks{kind="types",value="documentAdditionOrUpdate"} 9 +meilisearch_nb_tasks{kind="types",value="documentDeletion"} 0 +meilisearch_nb_tasks{kind="types",value="documentEdition"} 0 +meilisearch_nb_tasks{kind="types",value="dumpCreation"} 0 +meilisearch_nb_tasks{kind="types",value="indexCreation"} 0 +meilisearch_nb_tasks{kind="types",value="indexDeletion"} 8 +meilisearch_nb_tasks{kind="types",value="indexSwap"} 0 +meilisearch_nb_tasks{kind="types",value="indexUpdate"} 0 +meilisearch_nb_tasks{kind="types",value="settingsUpdate"} 22 +meilisearch_nb_tasks{kind="types",value="snapshotCreation"} 0 +meilisearch_nb_tasks{kind="types",value="taskCancelation"} 0 +meilisearch_nb_tasks{kind="types",value="taskDeletion"} 0 +# HELP meilisearch_used_db_size_bytes Meilisearch Used DB Size In Bytes +# TYPE meilisearch_used_db_size_bytes gauge +meilisearch_used_db_size_bytes 409600 +"# + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_metrics( index_scheduler: GuardedData, Data>, auth_controller: Data, diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 91237b707..5ee8498d2 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -1,20 +1,38 @@ use std::collections::BTreeMap; -use actix_web::web::Data; -use actix_web::{web, HttpRequest, HttpResponse}; -use index_scheduler::IndexScheduler; -use meilisearch_auth::AuthController; -use meilisearch_types::error::{Code, ResponseError}; -use meilisearch_types::settings::{Settings, Unchecked}; -use meilisearch_types::tasks::{Kind, Status, Task, TaskId}; -use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; -use tracing::debug; - use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::search_queue::SearchQueue; use crate::Opt; +use actix_web::web::Data; +use actix_web::{web, HttpRequest, HttpResponse}; +use index_scheduler::IndexScheduler; +use meilisearch_auth::AuthController; +use meilisearch_types::error::{Code, ErrorType, ResponseError}; +use meilisearch_types::index_uid::IndexUid; +use meilisearch_types::keys::CreateApiKey; +use meilisearch_types::settings::{ + Checked, FacetingSettings, MinWordSizeTyposSetting, PaginationSettings, Settings, TypoSettings, + Unchecked, +}; +use meilisearch_types::task_view::{DetailsView, TaskView}; +use meilisearch_types::tasks::{Kind, Status, Task, TaskId}; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use tracing::debug; +use utoipa::{OpenApi, ToSchema}; +use utoipa_rapidoc::RapiDoc; +use utoipa_redoc::{Redoc, Servable}; +use utoipa_scalar::{Scalar, Servable as ScalarServable}; + +use self::api_key::KeyView; +use self::indexes::documents::BrowseQuery; +use self::indexes::{IndexCreateRequest, IndexStats, UpdateIndexRequest}; +use self::logs::GetLogs; +use self::logs::LogMode; +use self::logs::UpdateStderrLogs; +use self::open_api_utils::OpenApiAuth; +use self::tasks::AllTasks; const PAGINATION_DEFAULT_LIMIT: usize = 20; @@ -27,24 +45,50 @@ mod logs; mod metrics; mod multi_search; mod multi_search_analytics; +mod open_api_utils; mod snapshot; mod swap_indexes; pub mod tasks; +#[derive(OpenApi)] +#[openapi( + nest( + (path = "/tasks", api = tasks::TaskApi), + (path = "/indexes", api = indexes::IndexesApi), + (path = "/snapshots", api = snapshot::SnapshotApi), + (path = "/dumps", api = dump::DumpApi), + (path = "/keys", api = api_key::ApiKeyApi), + (path = "/metrics", api = metrics::MetricApi), + (path = "/logs", api = logs::LogsApi), + ), + paths(get_health, get_version, get_stats), + tags( + (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), + ), + modifiers(&OpenApiAuth), + components(schemas(BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) +)] +pub struct MeilisearchApi; + pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::scope("/tasks").configure(tasks::configure)) - .service(web::scope("/batches").configure(batches::configure)) - .service(web::resource("/health").route(web::get().to(get_health))) - .service(web::scope("/logs").configure(logs::configure)) - .service(web::scope("/keys").configure(api_key::configure)) - .service(web::scope("/dumps").configure(dump::configure)) - .service(web::scope("/snapshots").configure(snapshot::configure)) - .service(web::resource("/stats").route(web::get().to(get_stats))) - .service(web::resource("/version").route(web::get().to(get_version))) - .service(web::scope("/indexes").configure(indexes::configure)) - .service(web::scope("/multi-search").configure(multi_search::configure)) - .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) - .service(web::scope("/metrics").configure(metrics::configure)) + let openapi = MeilisearchApi::openapi(); + + cfg.service(web::scope("/tasks").configure(tasks::configure)) // done + .service(web::scope("/batches").configure(batches::configure)) // TODO + .service(Scalar::with_url("/scalar", openapi.clone())) // done + .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) // done + .service(Redoc::with_url("/redoc", openapi)) // done + .service(web::resource("/health").route(web::get().to(get_health))) // done + .service(web::scope("/logs").configure(logs::configure)) // done + .service(web::scope("/keys").configure(api_key::configure)) // done + .service(web::scope("/dumps").configure(dump::configure)) // done + .service(web::scope("/snapshots").configure(snapshot::configure)) // done + .service(web::resource("/stats").route(web::get().to(get_stats))) // done + .service(web::resource("/version").route(web::get().to(get_version))) // done + .service(web::scope("/indexes").configure(indexes::configure)) // WIP + .service(web::scope("/multi-search").configure(multi_search::configure)) // TODO + .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // TODO + .service(web::scope("/metrics").configure(metrics::configure)) // done .service(web::scope("/experimental-features").configure(features::configure)); } @@ -98,14 +142,20 @@ pub fn is_dry_run(req: &HttpRequest, opt: &Opt) -> Result { .map_or(false, |s| s.to_lowercase() == "true")) } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct SummarizedTaskView { + /// The task unique identifier. + #[schema(value_type = u32)] task_uid: TaskId, + /// The index affected by this task. May be `null` if the task is not linked to any index. index_uid: Option, + /// The status of the task. status: Status, + /// The type of the task. #[serde(rename = "type")] kind: Kind, + /// The date on which the task was enqueued. #[serde(serialize_with = "time::serde::rfc3339::serialize")] enqueued_at: OffsetDateTime, } @@ -128,6 +178,7 @@ pub struct Pagination { } #[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] pub struct PaginationView { pub results: Vec, pub offset: usize, @@ -283,17 +334,56 @@ pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "Meilisearch is running" })) } -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Stats { + /// The size of the database, in bytes. pub database_size: u64, #[serde(skip)] pub used_database_size: u64, + /// The date of the last update in the RFC 3339 formats. Can be `null` if no update has ever been processed. #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] pub last_update: Option, + /// The stats of every individual index your API key lets you access. + #[schema(value_type = HashMap)] pub indexes: BTreeMap, } +/// Get stats of all indexes. +/// +/// Get stats of all indexes. +#[utoipa::path( + get, + path = "/stats", + tag = "Stats", + security(("Bearer" = ["stats.get", "stats.*", "*"])), + responses( + (status = 200, description = "The stats of the instance", body = Stats, content_type = "application/json", example = json!( + { + "databaseSize": 567, + "lastUpdate": "2019-11-20T09:40:33.711324Z", + "indexes": { + "movies": { + "numberOfDocuments": 10, + "isIndexing": true, + "fieldDistribution": { + "genre": 10, + "author": 9 + } + } + } + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_stats( index_scheduler: GuardedData, Data>, auth_controller: GuardedData, Data>, @@ -343,14 +433,43 @@ pub fn create_all_stats( Ok(stats) } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] #[serde(rename_all = "camelCase")] struct VersionResponse { + /// The commit used to compile this build of Meilisearch. commit_sha: String, + /// The date of this build. commit_date: String, + /// The version of Meilisearch. pkg_version: String, } +/// Get version +/// +/// Current version of Meilisearch. +#[utoipa::path( + get, + path = "/version", + tag = "Version", + security(("Bearer" = ["version", "*"])), + responses( + (status = 200, description = "Instance is healthy", body = VersionResponse, content_type = "application/json", example = json!( + { + "commitSha": "b46889b5f0f2f8b91438a08a358ba8f05fc09fc1", + "commitDate": "2021-07-08", + "pkgVersion": "0.23.0" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_version( _index_scheduler: GuardedData, Data>, ) -> HttpResponse { @@ -370,6 +489,35 @@ async fn get_version( }) } +#[derive(Default, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +struct HealthResponse { + /// The status of the instance. + status: HealthStatus, +} + +#[derive(Default, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +enum HealthStatus { + #[default] + Available, +} + +/// Get Health +/// +/// The health check endpoint enables you to periodically test the health of your Meilisearch instance. +#[utoipa::path( + get, + path = "/health", + tag = "Health", + responses( + (status = 200, description = "Instance is healthy", body = HealthResponse, content_type = "application/json", example = json!( + { + "status": "available" + } + )), + ) +)] pub async fn get_health( index_scheduler: Data, auth_controller: Data, @@ -379,5 +527,5 @@ pub async fn get_health( index_scheduler.health().unwrap(); auth_controller.health().unwrap(); - Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) + Ok(HttpResponse::Ok().json(&HealthResponse::default())) } diff --git a/crates/meilisearch/src/routes/open_api_utils.rs b/crates/meilisearch/src/routes/open_api_utils.rs new file mode 100644 index 000000000..89a3ef76a --- /dev/null +++ b/crates/meilisearch/src/routes/open_api_utils.rs @@ -0,0 +1,24 @@ +use serde::Serialize; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; + +#[derive(Debug, Serialize)] +pub struct OpenApiAuth; + +impl utoipa::Modify for OpenApiAuth { + fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) { + if let Some(schema) = openapi.components.as_mut() { + schema.add_security_scheme( + "Bearer", + SecurityScheme::Http( + HttpBuilder::new() + .scheme(HttpAuthScheme::Bearer) + .bearer_format("Uuidv4, string or JWT") + .description(Some( +"An API key is a token that you provide when making API calls. Include the token in a header parameter called `Authorization`. +Example: `Authorization: Bearer 8fece4405662dd830e4cb265e7e047aab2e79672a760a12712d2a263c9003509`")) + .build(), + ), + ); + } + } +} diff --git a/crates/meilisearch/src/routes/snapshot.rs b/crates/meilisearch/src/routes/snapshot.rs index cacbc41af..b619d7411 100644 --- a/crates/meilisearch/src/routes/snapshot.rs +++ b/crates/meilisearch/src/routes/snapshot.rs @@ -4,6 +4,7 @@ use index_scheduler::IndexScheduler; use meilisearch_types::error::ResponseError; use meilisearch_types::tasks::KindWithContent; use tracing::debug; +use utoipa::OpenApi; use crate::analytics::Analytics; use crate::extractors::authentication::policies::*; @@ -12,12 +13,56 @@ use crate::extractors::sequential_extractor::SeqHandler; use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView}; use crate::Opt; +#[derive(OpenApi)] +#[openapi( + paths(create_snapshot), + tags(( + name = "Snapshots", + description = "The snapshots route allows the creation of database snapshots. Snapshots are .snapshot files that can be used to launch Meilisearch. +Creating a snapshot is also referred to as exporting it, whereas launching Meilisearch with a snapshot is referred to as importing it. +During a snapshot export, all indexes of the current instance are exported—together with their documents and settings—and saved as a single .snapshot file. +During a snapshot import, all indexes contained in the indicated .snapshot file are imported along with their associated documents and settings. +Snapshot imports are performed at launch using an option.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/snapshots"), + + )), +)] +pub struct SnapshotApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot)))); } crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created"); +/// Create a snapshot +/// +/// Triggers a snapshot creation process. Once the process is complete, a snapshot is created in the snapshot directory. If the snapshot directory does not exist yet, it will be created. +#[utoipa::path( + post, + path = "/", + tag = "Snapshots", + security(("Bearer" = ["snapshots.create", "snapshots.*", "*"])), + responses( + (status = 202, description = "Snapshot is being created", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 0, + "indexUid": null, + "status": "enqueued", + "type": "snapshotCreation", + "enqueuedAt": "2021-01-01T09:39:00.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn create_snapshot( index_scheduler: GuardedData, Data>, req: HttpRequest, diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index 71c45eb1d..ffcc01698 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -17,14 +17,29 @@ use time::format_description::well_known::Rfc3339; use time::macros::format_description; use time::{Date, Duration, OffsetDateTime, Time}; use tokio::task; +use utoipa::{IntoParams, OpenApi, ToSchema}; use super::{get_task_id, is_dry_run, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT}; -use crate::analytics::{Aggregate, AggregateMethod, Analytics}; +use crate::analytics::Analytics; +use super::{get_task_id, is_dry_run, SummarizedTaskView}; +use crate::analytics::Analytics; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; use crate::{aggregate_methods, Opt}; +#[derive(OpenApi)] +#[openapi( + paths(get_tasks, delete_tasks, cancel_tasks, get_task), + tags(( + name = "Tasks", + description = "The tasks route gives information about the progress of the [asynchronous operations](https://docs.meilisearch.com/learn/advanced/asynchronous_operations.html).", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/tasks"), + + )), +)] +pub struct TaskApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -35,41 +50,66 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task)))); } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct TasksFilterQuery { - #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT as u32), error = DeserrQueryParamError)] + /// Maximum number of results to return. + #[deserr(default = Param(DEFAULT_LIMIT), error = DeserrQueryParamError)] + #[param(required = false, value_type = u32, example = 12, default = json!(DEFAULT_LIMIT))] pub limit: Param, + /// Fetch the next set of results from the given uid. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option, example = 12421)] pub from: Option>, + /// The order you want to retrieve the objects. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option, example = true)] pub reverse: Option>, - - #[deserr(default, error = DeserrQueryParamError)] - pub batch_uids: OptionStarOrList, - + /// Permits to filter tasks by their uid. By default, when the uids query parameter is not set, all task uids are returned. It's possible to specify several uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([231, 423, 598, "*"]))] pub uids: OptionStarOrList, + /// Permits to filter tasks using the uid of the task that canceled them. It's possible to specify several task uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([374, "*"]))] pub canceled_by: OptionStarOrList, + /// Permits to filter tasks by their related type. By default, when `types` query parameter is not set, all task types are returned. It's possible to specify several types by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([Kind::DocumentAdditionOrUpdate, "*"]))] pub types: OptionStarOrList, + /// Permits to filter tasks by their status. By default, when `statuses` query parameter is not set, all task statuses are returned. It's possible to specify several statuses by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([Status::Succeeded, Status::Failed, Status::Canceled, Status::Enqueued, Status::Processing, "*"]))] pub statuses: OptionStarOrList, + /// Permits to filter tasks by their related index. By default, when `indexUids` query parameter is not set, the tasks of all the indexes are returned. It is possible to specify several indexes by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!(["movies", "theater", "*"]))] pub index_uids: OptionStarOrList, + /// Permits to filter tasks based on their enqueuedAt time. Matches tasks enqueued after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_enqueued_at: OptionStarOr, + /// Permits to filter tasks based on their enqueuedAt time. Matches tasks enqueued before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_enqueued_at: OptionStarOr, + /// Permits to filter tasks based on their startedAt time. Matches tasks started after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_started_at: OptionStarOr, + /// Permits to filter tasks based on their startedAt time. Matches tasks started before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_started_at: OptionStarOr, + /// Permits to filter tasks based on their finishedAt time. Matches tasks finished after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_finished_at: OptionStarOr, + /// Permits to filter tasks based on their finishedAt time. Matches tasks finished before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_finished_at: OptionStarOr, } @@ -117,33 +157,58 @@ impl TaskDeletionOrCancelationQuery { } } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct TaskDeletionOrCancelationQuery { + /// Permits to filter tasks by their uid. By default, when the `uids` query parameter is not set, all task uids are returned. It's possible to specify several uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] - pub uids: OptionStarOrList, + #[param(required = false, value_type = Option>, example = json!([231, 423, 598, "*"]))] + pub uids: OptionStarOrList, + /// Lets you filter tasks by their `batchUid`. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([231, 423, 598, "*"]))] pub batch_uids: OptionStarOrList, + /// Permits to filter tasks using the uid of the task that canceled them. It's possible to specify several task uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] - pub canceled_by: OptionStarOrList, + #[param(required = false, value_type = Option>, example = json!([374, "*"]))] + pub canceled_by: OptionStarOrList, + /// Permits to filter tasks by their related type. By default, when `types` query parameter is not set, all task types are returned. It's possible to specify several types by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([Kind::DocumentDeletion, "*"]))] pub types: OptionStarOrList, + /// Permits to filter tasks by their status. By default, when `statuses` query parameter is not set, all task statuses are returned. It's possible to specify several statuses by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!([Status::Succeeded, Status::Failed, Status::Canceled, "*"]))] pub statuses: OptionStarOrList, + /// Permits to filter tasks by their related index. By default, when `indexUids` query parameter is not set, the tasks of all the indexes are returned. It is possible to specify several indexes by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option>, example = json!(["movies", "theater", "*"]))] pub index_uids: OptionStarOrList, + /// Permits to filter tasks based on their enqueuedAt time. Matches tasks enqueued after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_enqueued_at: OptionStarOr, + /// Permits to filter tasks based on their enqueuedAt time. Matches tasks enqueued before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_enqueued_at: OptionStarOr, + /// Permits to filter tasks based on their startedAt time. Matches tasks started after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_started_at: OptionStarOr, + /// Permits to filter tasks based on their startedAt time. Matches tasks started before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_started_at: OptionStarOr, + /// Permits to filter tasks based on their finishedAt time. Matches tasks finished after the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_after -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub after_finished_at: OptionStarOr, + /// Permits to filter tasks based on their finishedAt time. Matches tasks finished before the given date. Supports RFC 3339 date format. #[deserr(default, error = DeserrQueryParamError, try_from(OptionStarOr) = deserialize_date_before -> InvalidTaskDateError)] + #[param(required = false, value_type = Option, example = json!(["2024-08-08T16:37:09.971Z", "*"]))] pub before_finished_at: OptionStarOr, } @@ -226,6 +291,51 @@ impl Aggregate for TaskFilterAnalytics, Data>, params: AwebQueryParameter, @@ -275,6 +385,51 @@ async fn cancel_tasks( Ok(HttpResponse::Ok().json(task)) } +/// Delete tasks +/// +/// Delete [tasks](https://docs.meilisearch.com/learn/advanced/asynchronous_operations.html) on filter +#[utoipa::path( + delete, + path = "", + tag = "Tasks", + security(("Bearer" = ["tasks.delete", "tasks.*", "*"])), + params(TaskDeletionOrCancelationQuery), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "taskDeletion", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 400, description = "A filter is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Query parameters to filter the tasks to delete are missing. Available query parameters are: `uids`, `indexUids`, `statuses`, `types`, `canceledBy`, `beforeEnqueuedAt`, `afterEnqueuedAt`, `beforeStartedAt`, `afterStartedAt`, `beforeFinishedAt`, `afterFinishedAt`.", + "code": "missing_task_filters", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#missing_task_filters" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + (status = 404, description = "The task uid does not exists", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Task :taskUid not found.", + "code": "task_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors/#task_not_found" + } + )) + ) +)] async fn delete_tasks( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, @@ -323,15 +478,70 @@ async fn delete_tasks( Ok(HttpResponse::Ok().json(task)) } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct AllTasks { + /// The list of tasks that matched the filter. results: Vec, + /// Total number of browsable results using offset/limit parameters for the given resource. total: u64, + /// Limit given for the query. If limit is not provided as a query parameter, this parameter displays the default limit value. limit: u32, + /// The first task uid returned. from: Option, + /// Represents the value to send in from to fetch the next slice of the results. The first item for the next slice starts at this exact number. When the returned value is null, it means that all the data have been browsed in the given order. next: Option, } + +/// Get all tasks +/// +/// Get all [tasks](https://docs.meilisearch.com/learn/advanced/asynchronous_operations.html) +#[utoipa::path( + get, + path = "", + tag = "Tasks", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params(TasksFilterQuery), + responses( + (status = 200, description = "Get all tasks", body = AllTasks, content_type = "application/json", example = json!( + { + "results": [ + { + "uid": 144, + "indexUid": "mieli", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "settings": { + "filterableAttributes": [ + "play_count" + ] + } + }, + "error": null, + "duration": "PT0.009330S", + "enqueuedAt": "2024-08-08T09:01:13.348471Z", + "startedAt": "2024-08-08T09:01:13.349442Z", + "finishedAt": "2024-08-08T09:01:13.358772Z" + } + ], + "total": 1, + "limit": 1, + "from": 144, + "next": null + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_tasks( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, @@ -356,6 +566,52 @@ async fn get_tasks( Ok(HttpResponse::Ok().json(tasks)) } +/// Get a task +/// +/// Get a [task](https://www.meilisearch.com/docs/learn/async/asynchronous_operations) +#[utoipa::path( + get, + path = "/{taskUid}", + tag = "Tasks", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params(("taskUid", format = UInt32, example = 0, description = "The task identifier", nullable = false)), + responses( + (status = 200, description = "Task successfully retrieved", body = TaskView, content_type = "application/json", example = json!( + { + "uid": 1, + "indexUid": "movies", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 79000, + "indexedDocuments": 79000 + }, + "error": null, + "duration": "PT1S", + "enqueuedAt": "2021-01-01T09:39:00.000000Z", + "startedAt": "2021-01-01T09:39:01.000000Z", + "finishedAt": "2021-01-01T09:39:02.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + (status = 404, description = "The task uid does not exists", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Task :taskUid not found.", + "code": "task_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors/#task_not_found" + } + )) + ) +)] async fn get_task( index_scheduler: GuardedData, Data>, task_uid: web::Path, diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 9f113e013..01432ee21 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -90,6 +90,7 @@ tracing = "0.1.40" ureq = { version = "2.10.0", features = ["json"] } url = "2.5.2" rayon-par-bridge = "0.1.0" +<<<<<<< HEAD:crates/milli/Cargo.toml hashbrown = "0.15.0" bumpalo = "3.16.0" bumparaw-collections = "0.1.2" @@ -100,6 +101,7 @@ uell = "0.1.0" enum-iterator = "2.1.0" bbqueue = { git = "https://github.com/meilisearch/bbqueue" } flume = { version = "0.11.1", default-features = false } +utoipa = { version = "5.0.0-rc.0", features = ["non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } [dev-dependencies] mimalloc = { version = "0.1.43", default-features = false } diff --git a/crates/milli/src/localized_attributes_rules.rs b/crates/milli/src/localized_attributes_rules.rs index 3c421ca6b..2b9bf099c 100644 --- a/crates/milli/src/localized_attributes_rules.rs +++ b/crates/milli/src/localized_attributes_rules.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use charabia::Language; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use crate::fields_ids_map::FieldsIdsMap; use crate::FieldId; @@ -14,9 +15,10 @@ use crate::FieldId; /// The pattern `attribute_name*` matches any attribute name that starts with `attribute_name`. /// The pattern `*attribute_name` matches any attribute name that ends with `attribute_name`. /// The pattern `*attribute_name*` matches any attribute name that contains `attribute_name`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct LocalizedAttributesRule { pub attribute_patterns: Vec, + #[schema(value_type = Vec)] pub locales: Vec, } diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 85259c2d0..3592e74e3 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -10,6 +10,7 @@ use itertools::{EitherOrBoth, Itertools}; use roaring::RoaringBitmap; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use time::OffsetDateTime; +use utoipa::{PartialSchema, ToSchema}; use super::del_add::DelAddOperation; use super::index_documents::{IndexDocumentsConfig, Transform}; @@ -40,6 +41,18 @@ pub enum Setting { NotSet, } +impl ToSchema for Setting { + fn name() -> std::borrow::Cow<'static, str> { + T::name() + } +} + +impl PartialSchema for Setting { + fn schema() -> utoipa::openapi::RefOr { + T::schema() + } +} + impl Deserr for Setting where T: Deserr, From 13afdaf393ce735e0c48fd45dd1347b906b108b8 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 18 Dec 2024 17:13:57 +0100 Subject: [PATCH 02/27] finish rebase and update utoipa to the latest version --- Cargo.lock | 22 ++++++++++---------- crates/meilisearch-types/Cargo.toml | 2 +- crates/meilisearch-types/src/settings.rs | 6 +++++- crates/meilisearch-types/src/task_view.rs | 3 ++- crates/meilisearch/Cargo.toml | 8 +++---- crates/meilisearch/src/routes/batches.rs | 2 +- crates/meilisearch/src/routes/indexes/mod.rs | 1 - crates/meilisearch/src/routes/tasks.rs | 14 ++++++++----- crates/milli/Cargo.toml | 3 +-- 9 files changed, 34 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b375a2616..fdb799787 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5964,9 +5964,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "5.0.0-rc.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf66139459b75afa33caddb62bb2afee3838923b630b9e0ef38c369e543382f" +checksum = "514a48569e4e21c86d0b84b5612b5e73c0b2cf09db63260134ba426d4e8ea714" dependencies = [ "indexmap", "serde", @@ -5976,22 +5976,22 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "5.0.0-rc.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c136da726bb82a527afa1fdf3f4619eaf104e2982f071f25239cef1c67c79eb" +checksum = "5629efe65599d0ccd5d493688cbf6e03aa7c1da07fe59ff97cf5977ed0637f66" dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.60", + "syn 2.0.87", "uuid", ] [[package]] name = "utoipa-rapidoc" -version = "4.0.1-rc.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d3324d5874fb734762214827dd30b47aa78510d12abab674a97f6d7c53688f" +checksum = "2f5e784e313457e79d65c378bdfc2f74275f0db91a72252be7d34360ec2afbe1" dependencies = [ "actix-web", "serde", @@ -6001,9 +6001,9 @@ dependencies = [ [[package]] name = "utoipa-redoc" -version = "4.0.1-rc.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4375bb6b0cb78a240c973f5e99977c482f3e92aeea1907367caa28776b9aaf9" +checksum = "9218304bba9a0ea5e92085b0a427ccce5fd56eaaf6436d245b7578e6a95787e1" dependencies = [ "actix-web", "serde", @@ -6013,9 +6013,9 @@ dependencies = [ [[package]] name = "utoipa-scalar" -version = "0.2.0-rc.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dc122c11f9642b20b3be88b60c1a3672319811f139698ac6999e72992ac7c7e" +checksum = "c1291aa7a2223c2f8399d1c6627ca0ba57ca0d7ecac762a2094a9dfd6376445a" dependencies = [ "actix-web", "serde", diff --git a/crates/meilisearch-types/Cargo.toml b/crates/meilisearch-types/Cargo.toml index d0d136399..55db187b5 100644 --- a/crates/meilisearch-types/Cargo.toml +++ b/crates/meilisearch-types/Cargo.toml @@ -40,7 +40,7 @@ time = { version = "0.3.36", features = [ "macros", ] } tokio = "1.38" -utoipa = { version = "5.0.0-rc.0", features = ["macros"] } +utoipa = { version = "5.2.0", features = ["macros"] } uuid = { version = "1.10.0", features = ["serde", "v4"] } [dev-dependencies] diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 92d61e28f..f7216a0cf 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -249,9 +249,12 @@ pub struct Settings { pub localized_attributes: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!(true))] pub facet_search: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!("Hemlo"))] + // TODO: TAMO pub prefix_search: Setting, #[serde(skip)] @@ -1046,8 +1049,9 @@ impl std::ops::Deref for WildcardSetting { } } -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub enum PrefixSearchSettings { #[default] diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 8a6720cc2..467408097 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -1,6 +1,7 @@ use milli::Object; use serde::{Deserialize, Serialize}; use time::{Duration, OffsetDateTime}; +use utoipa::ToSchema; use crate::batches::BatchId; use crate::error::ResponseError; @@ -66,7 +67,7 @@ impl TaskView { } } -#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct DetailsView { /// Number of documents received for documentAdditionOrUpdate task. diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 8a2f0c6c0..5094e6807 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -105,10 +105,10 @@ tracing-actix-web = "0.7.11" build-info = { version = "1.7.0", path = "../build-info" } roaring = "0.10.7" mopa-maintained = "0.2.3" -utoipa = { version = "5.0.0-rc.0", features = ["actix_extras", "macros", "non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } -utoipa-scalar = { version = "0.2.0-rc.0", features = ["actix-web"] } -utoipa-rapidoc = { version = "4.0.1-rc.0", features = ["actix-web"] } -utoipa-redoc = { version = "4.0.1-rc.0", features = ["actix-web"] } +utoipa = { version = "5.2.0", features = ["actix_extras", "macros", "non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } +utoipa-scalar = { version = "0.2.0", features = ["actix-web"] } +utoipa-rapidoc = { version = "5.0.0", features = ["actix-web"] } +utoipa-redoc = { version = "5.0.0", features = ["actix-web"] } [dev-dependencies] actix-rt = "2.10.0" diff --git a/crates/meilisearch/src/routes/batches.rs b/crates/meilisearch/src/routes/batches.rs index 36bf31605..7cdca0bf7 100644 --- a/crates/meilisearch/src/routes/batches.rs +++ b/crates/meilisearch/src/routes/batches.rs @@ -74,7 +74,7 @@ async fn get_batches( let next = if results.len() == limit as usize { results.pop().map(|t| t.uid) } else { None }; let from = results.first().map(|t| t.uid); - let tasks = AllBatches { results, limit: limit.saturating_sub(1), total, from, next }; + let tasks = AllBatches { results, limit: limit.saturating_sub(1) as u32, total, from, next }; Ok(HttpResponse::Ok().json(tasks)) } diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index f6881f70c..86e08ee2b 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -403,7 +403,6 @@ pub struct UpdateIndexRequest { )), ) )] ->>>>>>> 0f289a437 (Implements the get and delete tasks route):meilisearch/src/routes/indexes/mod.rs pub async fn update_index( index_scheduler: GuardedData, Data>, index_uid: web::Path, diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index ffcc01698..c01e0139c 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -20,9 +20,7 @@ use tokio::task; use utoipa::{IntoParams, OpenApi, ToSchema}; use super::{get_task_id, is_dry_run, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT}; -use crate::analytics::Analytics; -use super::{get_task_id, is_dry_run, SummarizedTaskView}; -use crate::analytics::Analytics; +use crate::analytics::{Aggregate, AggregateMethod, Analytics}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; @@ -55,8 +53,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct TasksFilterQuery { /// Maximum number of results to return. - #[deserr(default = Param(DEFAULT_LIMIT), error = DeserrQueryParamError)] - #[param(required = false, value_type = u32, example = 12, default = json!(DEFAULT_LIMIT))] + #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT as u32), error = DeserrQueryParamError)] + #[param(required = false, value_type = u32, example = 12, default = json!(PAGINATION_DEFAULT_LIMIT))] pub limit: Param, /// Fetch the next set of results from the given uid. #[deserr(default, error = DeserrQueryParamError)] @@ -66,6 +64,12 @@ pub struct TasksFilterQuery { #[deserr(default, error = DeserrQueryParamError)] #[param(required = false, value_type = Option, example = true)] pub reverse: Option>, + + /// Permits to filter tasks by their batch uid. By default, when the `batchUids` query parameter is not set, all task uids are returned. It's possible to specify several batch uids by separating them with the `,` character. + #[deserr(default, error = DeserrQueryParamError)] + #[param(required = false, value_type = Option, example = 12421)] + pub batch_uids: OptionStarOrList, + /// Permits to filter tasks by their uid. By default, when the uids query parameter is not set, all task uids are returned. It's possible to specify several uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] #[param(required = false, value_type = Option>, example = json!([231, 423, 598, "*"]))] diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 01432ee21..27b6ae876 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -90,7 +90,6 @@ tracing = "0.1.40" ureq = { version = "2.10.0", features = ["json"] } url = "2.5.2" rayon-par-bridge = "0.1.0" -<<<<<<< HEAD:crates/milli/Cargo.toml hashbrown = "0.15.0" bumpalo = "3.16.0" bumparaw-collections = "0.1.2" @@ -101,7 +100,7 @@ uell = "0.1.0" enum-iterator = "2.1.0" bbqueue = { git = "https://github.com/meilisearch/bbqueue" } flume = { version = "0.11.1", default-features = false } -utoipa = { version = "5.0.0-rc.0", features = ["non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } +utoipa = { version = "5.0.2", features = ["non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } [dev-dependencies] mimalloc = { version = "0.1.43", default-features = false } From 78f6f22a808a83be7ebefbc71016b7212eedfbb7 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 19 Dec 2024 15:54:10 +0100 Subject: [PATCH 03/27] implement all the /indexes/documents route --- crates/meilisearch-types/src/star_or.rs | 3 +- crates/meilisearch/src/routes/batches.rs | 2 +- .../src/routes/indexes/documents.rs | 215 +++++++++++++++++- crates/meilisearch/src/routes/mod.rs | 4 +- crates/meilisearch/src/routes/tasks.rs | 2 +- 5 files changed, 213 insertions(+), 13 deletions(-) diff --git a/crates/meilisearch-types/src/star_or.rs b/crates/meilisearch-types/src/star_or.rs index cd26a1fb0..6af833ed8 100644 --- a/crates/meilisearch-types/src/star_or.rs +++ b/crates/meilisearch-types/src/star_or.rs @@ -6,6 +6,7 @@ use std::str::FromStr; use deserr::{DeserializeError, Deserr, MergeWithError, ValueKind}; use serde::de::Visitor; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use utoipa::{IntoParams, PartialSchema, ToSchema}; use crate::deserr::query_params::FromQueryParameter; @@ -229,7 +230,7 @@ pub enum OptionStarOrList { List(Vec), } -impl OptionStarOrList { +impl OptionStarOrList { pub fn is_some(&self) -> bool { match self { Self::None => false, diff --git a/crates/meilisearch/src/routes/batches.rs b/crates/meilisearch/src/routes/batches.rs index 7cdca0bf7..36bf31605 100644 --- a/crates/meilisearch/src/routes/batches.rs +++ b/crates/meilisearch/src/routes/batches.rs @@ -74,7 +74,7 @@ async fn get_batches( let next = if results.len() == limit as usize { results.pop().map(|t| t.uid) } else { None }; let from = results.first().map(|t| t.uid); - let tasks = AllBatches { results, limit: limit.saturating_sub(1) as u32, total, from, next }; + let tasks = AllBatches { results, limit: limit.saturating_sub(1), total, from, next }; Ok(HttpResponse::Ok().json(tasks)) } diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 205b71420..ca73ac8b3 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -74,7 +74,7 @@ pub struct DocumentParam { #[derive(OpenApi)] #[openapi( - paths(get_documents, replace_documents, update_documents, clear_all_documents, delete_documents_batch), + paths(get_document, get_documents, delete_document, replace_documents, update_documents, clear_all_documents, delete_documents_batch, delete_documents_by_filter, edit_documents_by_function, documents_by_query_post), tags( ( name = "Documents", @@ -107,12 +107,14 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] pub struct GetDocument { #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Option>)] fields: OptionStarOrList, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Option)] retrieve_vectors: Param, } @@ -188,6 +190,56 @@ impl Aggregate for DocumentsFetchAggregator { } } + +/// Get one document +/// +/// Get one document from its primary key. +#[utoipa::path( + get, + path = "/{indexUid}/documents/{documentId}", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.get", "documents.*", "*"])), + params( + ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), + ("documentId" = String, Path, example = "85087", description = "The document identifier", nullable = false), + GetDocument, + ), + responses( + (status = 200, description = "The documents are returned", body = serde_json::Value, content_type = "application/json", example = json!( + { + "id": 25684, + "title": "American Ninja 5", + "poster": "https://image.tmdb.org/t/p/w1280/iuAQVI4mvjI83wnirpD8GVNRVuY.jpg", + "overview": "When a scientists daughter is kidnapped, American Ninja, attempts to find her, but this time he teams up with a youngster he has trained in the ways of the ninja.", + "release_date": 725846400 + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 404, description = "Document not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Document `a` not found.", + "code": "document_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#document_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn get_document( index_scheduler: GuardedData, Data>, document_param: web::Path, @@ -251,6 +303,39 @@ impl Aggregate for DocumentsDeletionAggregator { } } + +/// Delete a document +/// +/// Delete a single document by id. +#[utoipa::path( + delete, + path = "/{indexUid}/documents/{documentsId}", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.delete", "documents.*", "*"])), + params( + ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), + ("documentsId" = String, Path, example = "movies", description = "Document Identifier", nullable = false), + ), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentAdditionOrUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn delete_document( index_scheduler: GuardedData, Data>, path: web::Path, @@ -321,6 +406,58 @@ pub struct BrowseQuery { filter: Option, } +/// Get documents with POST +/// +/// Get a set of documents. +#[utoipa::path( + post, + path = "/{indexUid}/documents/fetch", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.delete", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + BrowseQuery, + ), + responses( + (status = 200, description = "Task successfully enqueued", body = PaginationView, content_type = "application/json", example = json!( + { + "results":[ + { + "title":"The Travels of Ibn Battuta", + "genres":[ + "Travel", + "Adventure" + ], + "language":"English", + "rating":4.5 + }, + { + "title":"Pride and Prejudice", + "genres":[ + "Classics", + "Fiction", + "Romance", + "Literature" + ], + "language":"English", + "rating":4 + }, + ], + "offset":0, + "limit":2, + "total":5 + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn documents_by_query_post( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -356,12 +493,10 @@ pub async fn documents_by_query_post( security(("Bearer" = ["documents.get", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - // Here we can use the post version of the browse query since it contains the exact same parameter BrowseQuery ), responses( - // body = PaginationView - (status = 200, description = "The documents are returned", body = serde_json::Value, content_type = "application/json", example = json!( + (status = 200, description = "The documents are returned", body = PaginationView, content_type = "application/json", example = json!( { "results": [ { @@ -922,8 +1057,8 @@ async fn copy_body_to_file( security(("Bearer" = ["documents.delete", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + // TODO: how to task an array of strings in parameter ), - // TODO: how to return an array of strings responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { @@ -983,13 +1118,45 @@ pub async fn delete_documents_batch( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct DocumentDeletionByFilter { #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_document_filter)] filter: Value, } +/// Delete documents by filter +/// +/// Delete a set of documents based on a filter. +#[utoipa::path( + post, + path = "/{indexUid}/documents/delete", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.delete", "documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + DocumentDeletionByFilter, + ), + responses( + (status = 202, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentDeletion", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn delete_documents_by_filter( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -1030,7 +1197,7 @@ pub async fn delete_documents_by_filter( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct DocumentEditionByFunction { #[deserr(default, error = DeserrJsonError)] @@ -1069,6 +1236,38 @@ impl Aggregate for EditDocumentsByFunctionAggregator { } } +/// Edit documents by function. +/// +/// Use a [RHAI function](https://rhai.rs/book/engine/hello-world.html) to edit one or more documents directly in Meilisearch. +#[utoipa::path( + post, + path = "/{indexUid}/documents/edit", + tags = ["Indexes", "Documents"], + security(("Bearer" = ["documents.*", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + DocumentEditionByFunction, + ), + responses( + (status = 202, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": null, + "status": "enqueued", + "type": "documentDeletion", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn edit_documents_by_function( index_scheduler: GuardedData, Data>, index_uid: web::Path, diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 5ee8498d2..ddd6b6139 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -66,7 +66,7 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; @@ -177,7 +177,7 @@ pub struct Pagination { pub limit: usize, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct PaginationView { pub results: Vec, diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index c01e0139c..2f3871c1a 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -67,7 +67,7 @@ pub struct TasksFilterQuery { /// Permits to filter tasks by their batch uid. By default, when the `batchUids` query parameter is not set, all task uids are returned. It's possible to specify several batch uids by separating them with the `,` character. #[deserr(default, error = DeserrQueryParamError)] - #[param(required = false, value_type = Option, example = 12421)] + #[param(required = false, value_type = Option, example = 12421)] pub batch_uids: OptionStarOrList, /// Permits to filter tasks by their uid. By default, when the uids query parameter is not set, all task uids are returned. It's possible to specify several uids by separating them with the `,` character. From 04e4586fb3aeba3a0e07cd7760fb1b8390cf78f2 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 23 Dec 2024 15:58:52 +0100 Subject: [PATCH 04/27] add the searches route and fix a few broken things --- .../src/deserr/query_params.rs | 13 -- crates/meilisearch-types/src/star_or.rs | 2 +- crates/meilisearch/src/routes/api_key.rs | 6 +- .../src/routes/indexes/documents.rs | 2 +- .../meilisearch/src/routes/indexes/search.rs | 159 +++++++++++++++++- crates/meilisearch/src/routes/mod.rs | 3 + crates/meilisearch/src/search/mod.rs | 51 +++--- crates/milli/src/search/new/matches/mod.rs | 3 +- 8 files changed, 196 insertions(+), 43 deletions(-) diff --git a/crates/meilisearch-types/src/deserr/query_params.rs b/crates/meilisearch-types/src/deserr/query_params.rs index 58113567e..dded0ea5c 100644 --- a/crates/meilisearch-types/src/deserr/query_params.rs +++ b/crates/meilisearch-types/src/deserr/query_params.rs @@ -16,7 +16,6 @@ use std::ops::Deref; use std::str::FromStr; use deserr::{DeserializeError, Deserr, MergeWithError, ValueKind}; -use utoipa::{PartialSchema, ToSchema}; use super::{DeserrParseBoolError, DeserrParseIntError}; use crate::index_uid::IndexUid; @@ -30,18 +29,6 @@ use crate::tasks::{Kind, Status}; #[derive(Default, Debug, Clone, Copy)] pub struct Param(pub T); -impl ToSchema for Param { - fn name() -> std::borrow::Cow<'static, str> { - T::name() - } -} - -impl PartialSchema for Param { - fn schema() -> utoipa::openapi::RefOr { - T::schema() - } -} - impl Deref for Param { type Target = T; diff --git a/crates/meilisearch-types/src/star_or.rs b/crates/meilisearch-types/src/star_or.rs index 6af833ed8..1070b99ff 100644 --- a/crates/meilisearch-types/src/star_or.rs +++ b/crates/meilisearch-types/src/star_or.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use deserr::{DeserializeError, Deserr, MergeWithError, ValueKind}; use serde::de::Visitor; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use utoipa::{IntoParams, PartialSchema, ToSchema}; +use utoipa::PartialSchema; use crate::deserr::query_params::FromQueryParameter; diff --git a/crates/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs index 3309c1f6e..2a08448db 100644 --- a/crates/meilisearch/src/routes/api_key.rs +++ b/crates/meilisearch/src/routes/api_key.rs @@ -16,7 +16,7 @@ use time::OffsetDateTime; use utoipa::{IntoParams, OpenApi, ToSchema}; use uuid::Uuid; -use super::PAGINATION_DEFAULT_LIMIT; +use super::{PAGINATION_DEFAULT_LIMIT, PAGINATION_DEFAULT_LIMIT_FN}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; @@ -116,11 +116,11 @@ pub async fn create_api_key( #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] #[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct ListApiKeys { - #[into_params(value_type = usize, default = 0)] #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = usize, default = 0)] pub offset: Param, - #[into_params(value_type = usize, default = PAGINATION_DEFAULT_LIMIT)] #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError)] + #[param(value_type = usize, default = PAGINATION_DEFAULT_LIMIT_FN)] pub limit: Param, } diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index ca73ac8b3..dee46f2be 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -205,7 +205,7 @@ impl Aggregate for DocumentsFetchAggregator { GetDocument, ), responses( - (status = 200, description = "The documents are returned", body = serde_json::Value, content_type = "application/json", example = json!( + (status = 200, description = "The document is returned", body = serde_json::Value, content_type = "application/json", example = json!( { "id": 25684, "title": "American Ninja 5", diff --git a/crates/meilisearch/src/routes/indexes/search.rs b/crates/meilisearch/src/routes/indexes/search.rs index 291193c4e..ca3c61753 100644 --- a/crates/meilisearch/src/routes/indexes/search.rs +++ b/crates/meilisearch/src/routes/indexes/search.rs @@ -12,6 +12,7 @@ use meilisearch_types::milli; use meilisearch_types::serde_cs::vec::CS; use serde_json::Value; use tracing::debug; +use utoipa::{IntoParams, OpenApi}; use crate::analytics::Analytics; use crate::error::MeilisearchHttpError; @@ -22,12 +23,28 @@ use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS; use crate::routes::indexes::search_analytics::{SearchAggregator, SearchGET, SearchPOST}; use crate::search::{ add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold, - RetrieveVectors, SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH, + RetrieveVectors, SearchKind, SearchQuery, SearchResult, SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO, }; use crate::search_queue::SearchQueue; +#[derive(OpenApi)] +#[openapi( + paths(search_with_url_query, search_with_post), + tags( + ( + name = "Search", + description = "Meilisearch exposes two routes to perform searches: + +- A POST route: this is the preferred route when using API authentication, as it allows [preflight request](https://developer.mozilla.org/en-US/docs/Glossary/Preflight_request) caching and better performance. +- A GET route: the usage of this route is discouraged, unless you have good reason to do otherwise (specific caching abilities for example)", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/search"), + ), + ), +)] +pub struct SearchApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -36,30 +53,41 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -#[derive(Debug, deserr::Deserr)] +#[derive(Debug, deserr::Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct SearchQueryGet { #[deserr(default, error = DeserrQueryParamError)] q: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] vector: Option>, #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError)] + #[param(value_type = usize, default = DEFAULT_SEARCH_OFFSET)] offset: Param, #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError)] + #[param(value_type = usize, default = DEFAULT_SEARCH_LIMIT)] limit: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Option)] page: Option>, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Option)] hits_per_page: Option>, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] attributes_to_retrieve: Option>, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool, default)] retrieve_vectors: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] attributes_to_crop: Option>, #[deserr(default = Param(DEFAULT_CROP_LENGTH()), error = DeserrQueryParamError)] + #[param(value_type = usize, default = DEFAULT_CROP_LENGTH)] crop_length: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] attributes_to_highlight: Option>, #[deserr(default, error = DeserrQueryParamError)] filter: Option, @@ -68,30 +96,41 @@ pub struct SearchQueryGet { #[deserr(default, error = DeserrQueryParamError)] distinct: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool)] show_matches_position: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool)] show_ranking_score: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool)] show_ranking_score_details: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] facets: Option>, - #[deserr( default = DEFAULT_HIGHLIGHT_PRE_TAG(), error = DeserrQueryParamError)] + #[deserr(default = DEFAULT_HIGHLIGHT_PRE_TAG(), error = DeserrQueryParamError)] + #[param(default = DEFAULT_HIGHLIGHT_PRE_TAG)] highlight_pre_tag: String, - #[deserr( default = DEFAULT_HIGHLIGHT_POST_TAG(), error = DeserrQueryParamError)] + #[deserr(default = DEFAULT_HIGHLIGHT_POST_TAG(), error = DeserrQueryParamError)] + #[param(default = DEFAULT_HIGHLIGHT_POST_TAG)] highlight_post_tag: String, #[deserr(default = DEFAULT_CROP_MARKER(), error = DeserrQueryParamError)] + #[param(default = DEFAULT_CROP_MARKER)] crop_marker: String, #[deserr(default, error = DeserrQueryParamError)] matching_strategy: MatchingStrategy, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] pub attributes_to_search_on: Option>, #[deserr(default, error = DeserrQueryParamError)] pub hybrid_embedder: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = f32)] pub hybrid_semantic_ratio: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = f32)] pub ranking_score_threshold: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec, explode = false)] pub locales: Option>, } @@ -220,6 +259,62 @@ pub fn fix_sort_query_parameters(sort_query: &str) -> Vec { sort_parameters } +/// Search an index with GET +/// +/// Search for documents matching a specific query in the given index. +#[utoipa::path( + get, + path = "/{indexUid}/search", + tags = ["Indexes", "Search"], + security(("Bearer" = ["search", "*"])), + params( + ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), + SearchQueryGet + ), + responses( + (status = 200, description = "The documents are returned", body = SearchResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 2770, + "title": "American Pie 2", + "poster": "https://image.tmdb.org/t/p/w1280/q4LNgUnRfltxzp3gf1MAGiK5LhV.jpg", + "overview": "The whole gang are back and as close as ever. They decide to get even closer by spending the summer together at a beach house. They decide to hold the biggest…", + "release_date": 997405200 + }, + { + "id": 190859, + "title": "American Sniper", + "poster": "https://image.tmdb.org/t/p/w1280/svPHnYE7N5NAGO49dBmRhq0vDQ3.jpg", + "overview": "U.S. Navy SEAL Chris Kyle takes his sole mission—protect his comrades—to heart and becomes one of the most lethal snipers in American history. His pinpoint accuracy not only saves countless lives but also makes him a prime…", + "release_date": 1418256000 + } + ], + "offset": 0, + "limit": 2, + "estimatedTotalHits": 976, + "processingTimeMs": 35, + "query": "american " + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn search_with_url_query( index_scheduler: GuardedData, Data>, search_queue: web::Data, @@ -271,6 +366,62 @@ pub async fn search_with_url_query( Ok(HttpResponse::Ok().json(search_result)) } +/// Search with POST +/// +/// Search for documents matching a specific query in the given index. +#[utoipa::path( + post, + path = "/{indexUid}/search", + tags = ["Indexes", "Search"], + security(("Bearer" = ["search", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + ), + request_body = SearchQuery, + responses( + (status = 200, description = "The documents are returned", body = SearchResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 2770, + "title": "American Pie 2", + "poster": "https://image.tmdb.org/t/p/w1280/q4LNgUnRfltxzp3gf1MAGiK5LhV.jpg", + "overview": "The whole gang are back and as close as ever. They decide to get even closer by spending the summer together at a beach house. They decide to hold the biggest…", + "release_date": 997405200 + }, + { + "id": 190859, + "title": "American Sniper", + "poster": "https://image.tmdb.org/t/p/w1280/svPHnYE7N5NAGO49dBmRhq0vDQ3.jpg", + "overview": "U.S. Navy SEAL Chris Kyle takes his sole mission—protect his comrades—to heart and becomes one of the most lethal snipers in American history. His pinpoint accuracy not only saves countless lives but also makes him a prime…", + "release_date": 1418256000 + } + ], + "offset": 0, + "limit": 2, + "estimatedTotalHits": 976, + "processingTimeMs": 35, + "query": "american " + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn search_with_post( index_scheduler: GuardedData, Data>, search_queue: web::Data, diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index ddd6b6139..9a40f216a 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -35,6 +35,7 @@ use self::open_api_utils::OpenApiAuth; use self::tasks::AllTasks; const PAGINATION_DEFAULT_LIMIT: usize = 20; +const PAGINATION_DEFAULT_LIMIT_FN: fn() -> usize = || 20; mod api_key; pub mod batches; @@ -55,6 +56,8 @@ pub mod tasks; nest( (path = "/tasks", api = tasks::TaskApi), (path = "/indexes", api = indexes::IndexesApi), + // We must stop the search path here because the rest must be configured by each route individually + (path = "/indexes", api = indexes::search::SearchApi), (path = "/snapshots", api = snapshot::SnapshotApi), (path = "/dumps", api = dump::DumpApi), (path = "/keys", api = api_key::ApiKeyApi), diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index b48266b6a..6e73e794c 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -34,6 +34,7 @@ use serde::Serialize; use serde_json::{json, Value}; #[cfg(test)] mod mod_test; +use utoipa::ToSchema; use crate::error::MeilisearchHttpError; @@ -52,7 +53,7 @@ pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "".to_string(); pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "".to_string(); pub const DEFAULT_SEMANTIC_RATIO: fn() -> SemanticRatio = || SemanticRatio(0.5); -#[derive(Clone, Default, PartialEq, Deserr)] +#[derive(Clone, Default, PartialEq, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct SearchQuery { #[deserr(default, error = DeserrJsonError)] @@ -62,8 +63,10 @@ pub struct SearchQuery { #[deserr(default, error = DeserrJsonError)] pub hybrid: Option, #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError)] + #[schema(default = DEFAULT_SEARCH_OFFSET)] pub offset: usize, #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError)] + #[schema(default = DEFAULT_SEARCH_LIMIT)] pub limit: usize, #[deserr(default, error = DeserrJsonError)] pub page: Option, @@ -75,15 +78,16 @@ pub struct SearchQuery { pub retrieve_vectors: bool, #[deserr(default, error = DeserrJsonError)] pub attributes_to_crop: Option>, - #[deserr(default, error = DeserrJsonError, default = DEFAULT_CROP_LENGTH())] + #[deserr(error = DeserrJsonError, default = DEFAULT_CROP_LENGTH())] + #[schema(default = DEFAULT_CROP_LENGTH)] pub crop_length: usize, #[deserr(default, error = DeserrJsonError)] pub attributes_to_highlight: Option>, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub show_matches_position: bool, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub show_ranking_score: bool, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub show_ranking_score_details: bool, #[deserr(default, error = DeserrJsonError)] pub filter: Option, @@ -93,26 +97,28 @@ pub struct SearchQuery { pub distinct: Option, #[deserr(default, error = DeserrJsonError)] pub facets: Option>, - #[deserr(default, error = DeserrJsonError, default = DEFAULT_HIGHLIGHT_PRE_TAG())] + #[deserr(error = DeserrJsonError, default = DEFAULT_HIGHLIGHT_PRE_TAG())] + #[schema(default = DEFAULT_HIGHLIGHT_PRE_TAG)] pub highlight_pre_tag: String, - #[deserr(default, error = DeserrJsonError, default = DEFAULT_HIGHLIGHT_POST_TAG())] + #[deserr(error = DeserrJsonError, default = DEFAULT_HIGHLIGHT_POST_TAG())] + #[schema(default = DEFAULT_HIGHLIGHT_POST_TAG)] pub highlight_post_tag: String, - #[deserr(default, error = DeserrJsonError, default = DEFAULT_CROP_MARKER())] + #[deserr(error = DeserrJsonError, default = DEFAULT_CROP_MARKER())] + #[schema(default = DEFAULT_CROP_MARKER)] pub crop_marker: String, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub matching_strategy: MatchingStrategy, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub attributes_to_search_on: Option>, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub ranking_score_threshold: Option, - #[deserr(default, error = DeserrJsonError, default)] + #[deserr(default, error = DeserrJsonError)] pub locales: Option>, } -#[derive(Debug, Clone, Copy, PartialEq, Deserr)] +#[derive(Debug, Clone, Copy, PartialEq, Deserr, ToSchema)] #[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)] pub struct RankingScoreThreshold(f64); - impl std::convert::TryFrom for RankingScoreThreshold { type Error = InvalidSearchRankingScoreThreshold; @@ -266,10 +272,11 @@ impl fmt::Debug for SearchQuery { } } -#[derive(Debug, Clone, Default, PartialEq, Deserr)] +#[derive(Debug, Clone, Default, PartialEq, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct HybridQuery { #[deserr(default, error = DeserrJsonError, default)] + #[schema(value_type = f32, default)] pub semantic_ratio: SemanticRatio, #[deserr(error = DeserrJsonError)] pub embedder: String, @@ -587,7 +594,7 @@ impl TryFrom for ExternalDocumentId { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, ToSchema)] #[deserr(rename_all = camelCase)] pub enum MatchingStrategy { /// Remove query words from last to first @@ -634,11 +641,13 @@ impl From for OrderBy { } } -#[derive(Debug, Clone, Serialize, PartialEq)] +#[derive(Debug, Clone, Serialize, PartialEq, ToSchema)] pub struct SearchHit { #[serde(flatten)] + #[schema(additional_properties, inline, value_type = HashMap)] pub document: Document, #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] + #[schema(additional_properties, value_type = HashMap)] pub formatted: Document, #[serde(rename = "_matchesPosition", skip_serializing_if = "Option::is_none")] pub matches_position: Option, @@ -648,8 +657,9 @@ pub struct SearchHit { pub ranking_score_details: Option>, } -#[derive(Serialize, Clone, PartialEq)] +#[derive(Serialize, Clone, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct SearchResult { pub hits: Vec, pub query: String, @@ -657,6 +667,7 @@ pub struct SearchResult { #[serde(flatten)] pub hits_info: HitsInfo, #[serde(skip_serializing_if = "Option::is_none")] + #[schema(value_type = HashMap)] pub facet_distribution: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub facet_stats: Option>, @@ -729,7 +740,7 @@ pub struct SearchResultWithIndex { pub result: SearchResult, } -#[derive(Serialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Debug, Clone, PartialEq, Eq, ToSchema)] #[serde(untagged)] pub enum HitsInfo { #[serde(rename_all = "camelCase")] @@ -738,7 +749,7 @@ pub enum HitsInfo { OffsetLimit { limit: usize, offset: usize, estimated_total_hits: usize }, } -#[derive(Serialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Debug, Clone, PartialEq, ToSchema)] pub struct FacetStats { pub min: f64, pub max: f64, diff --git a/crates/milli/src/search/new/matches/mod.rs b/crates/milli/src/search/new/matches/mod.rs index 19c1127cd..83d00caf0 100644 --- a/crates/milli/src/search/new/matches/mod.rs +++ b/crates/milli/src/search/new/matches/mod.rs @@ -13,6 +13,7 @@ use matching_words::{MatchType, PartialMatch}; use r#match::{Match, MatchPosition}; use serde::Serialize; use simple_token_kind::SimpleTokenKind; +use utoipa::ToSchema; const DEFAULT_CROP_MARKER: &str = "…"; const DEFAULT_HIGHLIGHT_PREFIX: &str = ""; @@ -100,7 +101,7 @@ impl FormatOptions { } } -#[derive(Serialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Debug, Clone, PartialEq, Eq, ToSchema)] pub struct MatchBounds { pub start: usize, pub length: usize, From 668b26b6415417f0ba76b42896538b5814c136e3 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 23 Dec 2024 16:11:22 +0100 Subject: [PATCH 05/27] add the facet search --- .../src/routes/indexes/facet_search.rs | 80 +++++++++++++++++-- crates/meilisearch/src/routes/indexes/mod.rs | 1 + 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/facet_search.rs b/crates/meilisearch/src/routes/indexes/facet_search.rs index ff11f1305..ab335c528 100644 --- a/crates/meilisearch/src/routes/indexes/facet_search.rs +++ b/crates/meilisearch/src/routes/indexes/facet_search.rs @@ -11,6 +11,7 @@ use meilisearch_types::index_uid::IndexUid; use meilisearch_types::locales::Locale; use serde_json::Value; use tracing::debug; +use utoipa::{OpenApi, ToSchema}; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::*; @@ -18,20 +19,33 @@ use crate::extractors::authentication::GuardedData; use crate::routes::indexes::search::search_kind; use crate::search::{ add_search_rules, perform_facet_search, FacetSearchResult, HybridQuery, MatchingStrategy, - RankingScoreThreshold, SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, + RankingScoreThreshold, SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, }; use crate::search_queue::SearchQueue; +#[derive(OpenApi)] +#[openapi( + paths(search), + tags( + ( + name = "Facet Search", + description = "The `/facet-search` route allows you to search for facet values. Facet search supports prefix search and typo tolerance. The returned hits are sorted lexicographically in ascending order. You can configure how facets are sorted using the sortFacetValuesBy property of the faceting index settings.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/facet_search"), + ), + ), +)] +pub struct FacetSearchApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(search))); } -/// # Important -/// -/// Intentionally don't use `deny_unknown_fields` to ignore search parameters sent by user -#[derive(Debug, Clone, Default, PartialEq, deserr::Deserr)] +// # Important +// +// Intentionally don't use `deny_unknown_fields` to ignore search parameters sent by user +#[derive(Debug, Clone, Default, PartialEq, deserr::Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase)] pub struct FacetSearchQuery { #[deserr(default, error = DeserrJsonError)] @@ -158,6 +172,62 @@ impl Aggregate for FacetSearchAggregator { } } +/// Perform a facet search +/// +/// Search for a facet value within a given facet. +#[utoipa::path( + post, + path = "/{indexUid}/facet-search", + tags = ["Indexes", "Facet Search"], + security(("Bearer" = ["search", "*"])), + params( + ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), + ), + request_body = FacetSearchQuery, + responses( + (status = 200, description = "The documents are returned", body = SearchResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 2770, + "title": "American Pie 2", + "poster": "https://image.tmdb.org/t/p/w1280/q4LNgUnRfltxzp3gf1MAGiK5LhV.jpg", + "overview": "The whole gang are back and as close as ever. They decide to get even closer by spending the summer together at a beach house. They decide to hold the biggest…", + "release_date": 997405200 + }, + { + "id": 190859, + "title": "American Sniper", + "poster": "https://image.tmdb.org/t/p/w1280/svPHnYE7N5NAGO49dBmRhq0vDQ3.jpg", + "overview": "U.S. Navy SEAL Chris Kyle takes his sole mission—protect his comrades—to heart and becomes one of the most lethal snipers in American history. His pinpoint accuracy not only saves countless lives but also makes him a prime…", + "release_date": 1418256000 + } + ], + "offset": 0, + "limit": 2, + "estimatedTotalHits": 976, + "processingTimeMs": 35, + "query": "american " + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn search( index_scheduler: GuardedData, Data>, search_queue: Data, diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 86e08ee2b..76bb68d11 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -41,6 +41,7 @@ mod similar_analytics; #[openapi( nest( (path = "/", api = documents::DocumentsApi), + (path = "/", api = facet_search::FacetSearchApi), ), paths(list_indexes, create_index, get_index, update_index, delete_index, get_index_stats), tags( From 4eaa626bca55e8e564db317024646dd277a39d51 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 23 Dec 2024 16:32:24 +0100 Subject: [PATCH 06/27] add the similar route --- crates/meilisearch/src/routes/indexes/mod.rs | 1 + .../meilisearch/src/routes/indexes/similar.rs | 139 +++++++++++++++++- crates/meilisearch/src/routes/mod.rs | 3 +- crates/meilisearch/src/search/mod.rs | 6 +- 4 files changed, 145 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 76bb68d11..b2f949da9 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -42,6 +42,7 @@ mod similar_analytics; nest( (path = "/", api = documents::DocumentsApi), (path = "/", api = facet_search::FacetSearchApi), + (path = "/", api = similar::SimilarApi), ), paths(list_indexes, create_index, get_index, update_index, delete_index, get_index_stats), tags( diff --git a/crates/meilisearch/src/routes/indexes/similar.rs b/crates/meilisearch/src/routes/indexes/similar.rs index f47771061..63022b28f 100644 --- a/crates/meilisearch/src/routes/indexes/similar.rs +++ b/crates/meilisearch/src/routes/indexes/similar.rs @@ -11,6 +11,7 @@ use meilisearch_types::keys::actions; use meilisearch_types::serde_cs::vec::CS; use serde_json::Value; use tracing::debug; +use utoipa::{IntoParams, OpenApi}; use super::ActionPolicy; use crate::analytics::Analytics; @@ -22,6 +23,21 @@ use crate::search::{ SimilarQuery, SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, }; +#[derive(OpenApi)] +#[openapi( + paths(similar_get, similar_post), + tags( + ( + name = "Similar documents", + description = "The /similar route uses AI-powered search to return a number of documents similar to a target document. + +Meilisearch exposes two routes for retrieving similar documents: POST and GET. In the majority of cases, POST will offer better performance and ease of use.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/similar"), + ), + ), +)] +pub struct SimilarApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -30,6 +46,62 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } +/// Get similar documents with GET +/// +/// Retrieve documents similar to a specific search result. +#[utoipa::path( + get, + path = "/{indexUid}/similar", + tags = ["Indexes", "Similar documents"], + security(("Bearer" = ["search", "*"])), + params( + ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), + SimilarQueryGet + ), + responses( + (status = 200, description = "The documents are returned", body = SimilarResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 2770, + "title": "American Pie 2", + "poster": "https://image.tmdb.org/t/p/w1280/q4LNgUnRfltxzp3gf1MAGiK5LhV.jpg", + "overview": "The whole gang are back and as close as ever. They decide to get even closer by spending the summer together at a beach house. They decide to hold the biggest…", + "release_date": 997405200 + }, + { + "id": 190859, + "title": "American Sniper", + "poster": "https://image.tmdb.org/t/p/w1280/svPHnYE7N5NAGO49dBmRhq0vDQ3.jpg", + "overview": "U.S. Navy SEAL Chris Kyle takes his sole mission—protect his comrades—to heart and becomes one of the most lethal snipers in American history. His pinpoint accuracy not only saves countless lives but also makes him a prime…", + "release_date": 1418256000 + } + ], + "offset": 0, + "limit": 2, + "estimatedTotalHits": 976, + "processingTimeMs": 35, + "query": "american " + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn similar_get( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -58,6 +130,62 @@ pub async fn similar_get( Ok(HttpResponse::Ok().json(similar)) } +/// Get similar documents with POST +/// +/// Retrieve documents similar to a specific search result. +#[utoipa::path( + post, + path = "/{indexUid}/similar", + tags = ["Indexes", "Similar documents"], + security(("Bearer" = ["search", "*"])), + params( + ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), + ), + request_body = SimilarQuery, + responses( + (status = 200, description = "The documents are returned", body = SimilarResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 2770, + "title": "American Pie 2", + "poster": "https://image.tmdb.org/t/p/w1280/q4LNgUnRfltxzp3gf1MAGiK5LhV.jpg", + "overview": "The whole gang are back and as close as ever. They decide to get even closer by spending the summer together at a beach house. They decide to hold the biggest…", + "release_date": 997405200 + }, + { + "id": 190859, + "title": "American Sniper", + "poster": "https://image.tmdb.org/t/p/w1280/svPHnYE7N5NAGO49dBmRhq0vDQ3.jpg", + "overview": "U.S. Navy SEAL Chris Kyle takes his sole mission—protect his comrades—to heart and becomes one of the most lethal snipers in American history. His pinpoint accuracy not only saves countless lives but also makes him a prime…", + "release_date": 1418256000 + } + ], + "offset": 0, + "limit": 2, + "estimatedTotalHits": 976, + "processingTimeMs": 35, + "query": "american " + } + )), + (status = 404, description = "Index not found", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "Index `movies` not found.", + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn similar_post( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -125,26 +253,35 @@ async fn similar( .await? } -#[derive(Debug, deserr::Deserr)] +#[derive(Debug, deserr::Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(parameter_in = Query)] pub struct SimilarQueryGet { #[deserr(error = DeserrQueryParamError)] + #[param(value_type = String)] id: Param, #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError)] + #[param(value_type = usize, default = DEFAULT_SEARCH_OFFSET)] offset: Param, #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError)] + #[param(value_type = usize, default = DEFAULT_SEARCH_LIMIT)] limit: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = Vec)] attributes_to_retrieve: Option>, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool, default)] retrieve_vectors: Param, #[deserr(default, error = DeserrQueryParamError)] filter: Option, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool, default)] show_ranking_score: Param, #[deserr(default, error = DeserrQueryParamError)] + #[param(value_type = bool, default)] show_ranking_score_details: Param, #[deserr(default, error = DeserrQueryParamError, default)] + #[param(value_type = Option)] pub ranking_score_threshold: Option, #[deserr(error = DeserrQueryParamError)] pub embedder: String, diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 9a40f216a..838335204 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; +use crate::search::{SimilarQuery, SimilarResult}; use crate::search_queue::SearchQueue; use crate::Opt; use actix_web::web::Data; @@ -69,7 +70,7 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 6e73e794c..cada265dd 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -537,10 +537,11 @@ impl SearchQueryWithIndex { } } -#[derive(Debug, Clone, PartialEq, Deserr)] +#[derive(Debug, Clone, PartialEq, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct SimilarQuery { #[deserr(error = DeserrJsonError)] + #[schema(value_type = String)] pub id: ExternalDocumentId, #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError)] pub offset: usize, @@ -559,6 +560,7 @@ pub struct SimilarQuery { #[deserr(default, error = DeserrJsonError, default)] pub show_ranking_score_details: bool, #[deserr(default, error = DeserrJsonError, default)] + #[schema(value_type = f64)] pub ranking_score_threshold: Option, } @@ -722,7 +724,7 @@ impl fmt::Debug for SearchResult { } } -#[derive(Serialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Debug, Clone, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct SimilarResult { pub hits: Vec, From 0bf4157a75466f13f13365d051ee6d630bc37a74 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 23 Dec 2024 20:52:47 +0100 Subject: [PATCH 07/27] try my best to make the sub-settings routes works, it doesn't --- Cargo.lock | 1 + crates/meilisearch-types/src/settings.rs | 41 +++++-- crates/meilisearch/Cargo.toml | 1 + crates/meilisearch/src/routes/indexes/mod.rs | 1 + .../src/routes/indexes/settings.rs | 112 +++++++++++++++++- .../src/routes/indexes/settings_analytics.rs | 13 +- crates/milli/src/update/settings.rs | 13 -- crates/milli/src/vector/mod.rs | 5 +- crates/milli/src/vector/settings.rs | 18 ++- 9 files changed, 171 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdb799787..fb405d891 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3471,6 +3471,7 @@ dependencies = [ "clap", "crossbeam-channel", "deserr", + "doc-comment", "dump", "either", "file-store", diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index f7216a0cf..a5416583b 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -144,6 +144,25 @@ impl MergeWithError for DeserrJsonError)] + pub inner: Setting, +} + +impl Deserr for SettingEmbeddingSettings { + fn deserialize_from_value( + value: deserr::Value, + location: ValuePointerRef, + ) -> Result { + Setting::::deserialize_from_value( + value, location, + ) + .map(|inner| Self { inner }) + } +} + /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings /// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a /// call to `check` will return a `Settings` from a `Settings`. @@ -237,7 +256,7 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] #[schema(value_type = String)] // TODO: TAMO - pub embedders: Setting>>, + pub embedders: Setting>, /// Maximum duration of a search query. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] @@ -254,7 +273,6 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] #[schema(value_type = Option, example = json!("Hemlo"))] - // TODO: TAMO pub prefix_search: Setting, #[serde(skip)] @@ -269,7 +287,7 @@ impl Settings { }; for mut embedder in embedders.values_mut() { - let Setting::Set(embedder) = &mut embedder else { + let SettingEmbeddingSettings { inner: Setting::Set(embedder) } = &mut embedder else { continue; }; @@ -434,8 +452,9 @@ impl Settings { let Setting::Set(mut configs) = self.embedders else { return Ok(self) }; for (name, config) in configs.iter_mut() { let config_to_check = std::mem::take(config); - let checked_config = milli::update::validate_embedding_settings(config_to_check, name)?; - *config = checked_config + let checked_config = + milli::update::validate_embedding_settings(config_to_check.inner, name)?; + *config = SettingEmbeddingSettings { inner: checked_config }; } self.embedders = Setting::Set(configs); Ok(self) @@ -713,7 +732,9 @@ pub fn apply_settings_to_builder( } match embedders { - Setting::Set(value) => builder.set_embedder_settings(value.clone()), + Setting::Set(value) => builder.set_embedder_settings( + value.iter().map(|(k, v)| (k.clone(), v.inner.clone())).collect(), + ), Setting::Reset => builder.reset_embedder_settings(), Setting::NotSet => (), } @@ -827,7 +848,9 @@ pub fn settings( let embedders: BTreeMap<_, _> = index .embedding_configs(rtxn)? .into_iter() - .map(|IndexEmbeddingConfig { name, config, .. }| (name, Setting::Set(config.into()))) + .map(|IndexEmbeddingConfig { name, config, .. }| { + (name, SettingEmbeddingSettings { inner: Setting::Set(config.into()) }) + }) .collect(); let embedders = if embedders.is_empty() { Setting::NotSet } else { Setting::Set(embedders) }; @@ -886,7 +909,7 @@ pub fn settings( Ok(settings) } -#[derive(Debug, Clone, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, PartialEq, Eq, Deserr, ToSchema)] #[deserr(try_from(&String) = FromStr::from_str -> CriterionError)] pub enum RankingRuleView { /// Sorted by decreasing number of matched query terms. @@ -982,7 +1005,7 @@ impl From for Criterion { } } -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserr, Serialize, Deserialize, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub enum ProximityPrecisionView { diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 5094e6807..2b6583526 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -109,6 +109,7 @@ utoipa = { version = "5.2.0", features = ["actix_extras", "macros", "non_strict_ utoipa-scalar = { version = "0.2.0", features = ["actix-web"] } utoipa-rapidoc = { version = "5.0.0", features = ["actix-web"] } utoipa-redoc = { version = "5.0.0", features = ["actix-web"] } +doc-comment = "0.3.3" [dev-dependencies] actix-rt = "2.10.0" diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index b2f949da9..c6f2a9397 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -43,6 +43,7 @@ mod similar_analytics; (path = "/", api = documents::DocumentsApi), (path = "/", api = facet_search::FacetSearchApi), (path = "/", api = similar::SimilarApi), + (path = "/", api = settings::SettingsApi), ), paths(list_indexes, create_index, get_index, update_index, delete_index, get_index_stats), tags( diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index b2922e5ff..cf104ee99 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -6,9 +6,12 @@ use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::update::Setting; -use meilisearch_types::settings::{settings, SecretPolicy, Settings, Unchecked}; +use meilisearch_types::settings::{ + settings, SecretPolicy, SettingEmbeddingSettings, Settings, Unchecked, +}; use meilisearch_types::tasks::KindWithContent; use tracing::debug; +use utoipa::OpenApi; use super::settings_analytics::*; use crate::analytics::Analytics; @@ -29,6 +32,20 @@ macro_rules! make_setting_routes { make_setting_route!($route, $update_verb, $type, $err_ty, $attr, $camelcase_attr, $analytics); )* + #[derive(OpenApi)] + #[openapi( + nest($((path = "/", api = $attr::$attr),)*), + // paths(/* update_all, get_all, delete_all,*/ $( $attr::get, $attr::update, $attr::delete,)*), + tags( + ( + name = "Settings", + description = "Use the /settings route to customize search settings for a given index. You can either modify all index settings at once using the update settings endpoint, or use a child route to configure a single setting.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/settings"), + ), + ), + )] + pub struct SettingsApi; + pub fn configure(cfg: &mut web::ServiceConfig) { use crate::extractors::sequential_extractor::SeqHandler; cfg.service( @@ -62,7 +79,42 @@ macro_rules! make_setting_route { use $crate::extractors::sequential_extractor::SeqHandler; use $crate::Opt; use $crate::routes::{is_dry_run, get_task_id, SummarizedTaskView}; + #[allow(unused_imports)] + use super::*; + #[derive(OpenApi)] + #[openapi( + paths(get, update, delete,), + )] + pub struct $attr; + + #[doc = $camelcase_attr] + #[utoipa::path( + delete, + path = "/", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.update", "settings.*", "*"])), + request_body = $type, + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) + )] pub async fn delete( index_scheduler: GuardedData< ActionPolicy<{ actions::SETTINGS_UPDATE }>, @@ -96,6 +148,34 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(task)) } + + #[doc = $camelcase_attr] + #[utoipa::path( + $update_verb, + path = "/", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.update", "settings.*", "*"])), + request_body = $type, + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) + )] pub async fn update( index_scheduler: GuardedData< ActionPolicy<{ actions::SETTINGS_UPDATE }>, @@ -151,6 +231,34 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(task)) } + + #[doc = $camelcase_attr] + #[utoipa::path( + get, + path = "/", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.get", "settings.*", "*"])), + request_body = $type, + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) + )] pub async fn get( index_scheduler: GuardedData< ActionPolicy<{ actions::SETTINGS_GET }>, @@ -359,7 +467,7 @@ make_setting_routes!( { route: "/embedders", update_verb: patch, - value_type: std::collections::BTreeMap>, + value_type: std::collections::BTreeMap, err_type: meilisearch_types::deserr::DeserrJsonError< meilisearch_types::error::deserr_codes::InvalidSettingsEmbedders, >, diff --git a/crates/meilisearch/src/routes/indexes/settings_analytics.rs b/crates/meilisearch/src/routes/indexes/settings_analytics.rs index ddca2c00a..ffeadcab6 100644 --- a/crates/meilisearch/src/routes/indexes/settings_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/settings_analytics.rs @@ -8,10 +8,9 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::locales::{Locale, LocalizedAttributesRuleView}; use meilisearch_types::milli::update::Setting; -use meilisearch_types::milli::vector::settings::EmbeddingSettings; use meilisearch_types::settings::{ FacetingSettings, PaginationSettings, PrefixSearchSettings, ProximityPrecisionView, - RankingRuleView, TypoSettings, + RankingRuleView, SettingEmbeddingSettings, TypoSettings, }; use serde::Serialize; @@ -497,13 +496,13 @@ pub struct EmbeddersAnalytics { } impl EmbeddersAnalytics { - pub fn new(setting: Option<&BTreeMap>>) -> Self { + pub fn new(setting: Option<&BTreeMap>) -> Self { let mut sources = std::collections::HashSet::new(); if let Some(s) = &setting { for source in s .values() - .filter_map(|config| config.clone().set()) + .filter_map(|config| config.inner.clone().set()) .filter_map(|config| config.source.set()) { use meilisearch_types::milli::vector::settings::EmbedderSource; @@ -522,18 +521,18 @@ impl EmbeddersAnalytics { sources: Some(sources), document_template_used: setting.as_ref().map(|map| { map.values() - .filter_map(|config| config.clone().set()) + .filter_map(|config| config.inner.clone().set()) .any(|config| config.document_template.set().is_some()) }), document_template_max_bytes: setting.as_ref().and_then(|map| { map.values() - .filter_map(|config| config.clone().set()) + .filter_map(|config| config.inner.clone().set()) .filter_map(|config| config.document_template_max_bytes.set()) .max() }), binary_quantization_used: setting.as_ref().map(|map| { map.values() - .filter_map(|config| config.clone().set()) + .filter_map(|config| config.inner.clone().set()) .any(|config| config.binary_quantized.set().is_some()) }), } diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 3592e74e3..85259c2d0 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -10,7 +10,6 @@ use itertools::{EitherOrBoth, Itertools}; use roaring::RoaringBitmap; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use time::OffsetDateTime; -use utoipa::{PartialSchema, ToSchema}; use super::del_add::DelAddOperation; use super::index_documents::{IndexDocumentsConfig, Transform}; @@ -41,18 +40,6 @@ pub enum Setting { NotSet, } -impl ToSchema for Setting { - fn name() -> std::borrow::Cow<'static, str> { - T::name() - } -} - -impl PartialSchema for Setting { - fn schema() -> utoipa::openapi::RefOr { - T::schema() - } -} - impl Deserr for Setting where T: Deserr, diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index a1d71ef93..0be698027 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -9,6 +9,7 @@ use heed::{RoTxn, RwTxn, Unspecified}; use ordered_float::OrderedFloat; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use self::error::{EmbedError, NewEmbedderError}; use crate::prompt::{Prompt, PromptData}; @@ -710,18 +711,20 @@ impl Embedder { /// /// The intended use is to make the similarity score more comparable to the regular ranking score. /// This allows to correct effects where results are too "packed" around a certain value. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize, ToSchema)] #[serde(from = "DistributionShiftSerializable")] #[serde(into = "DistributionShiftSerializable")] pub struct DistributionShift { /// Value where the results are "packed". /// /// Similarity scores are translated so that they are packed around 0.5 instead + #[schema(value_type = f32)] pub current_mean: OrderedFloat, /// standard deviation of a similarity score. /// /// Set below 0.4 to make the results less packed around the mean, and above 0.4 to make them more packed. + #[schema(value_type = f32)] pub current_sigma: OrderedFloat, } diff --git a/crates/milli/src/vector/settings.rs b/crates/milli/src/vector/settings.rs index d1cf364a2..4a1b1882c 100644 --- a/crates/milli/src/vector/settings.rs +++ b/crates/milli/src/vector/settings.rs @@ -4,6 +4,7 @@ use std::num::NonZeroUsize; use deserr::Deserr; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use super::{ollama, openai, DistributionShift}; use crate::prompt::{default_max_bytes, PromptData}; @@ -11,48 +12,61 @@ use crate::update::Setting; use crate::vector::EmbeddingConfig; use crate::UserError; -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(rename_all = camelCase, deny_unknown_fields)] pub struct EmbeddingSettings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub source: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub model: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub revision: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub api_key: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub dimensions: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub binary_quantized: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub document_template: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub document_template_max_bytes: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub url: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub request: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub response: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option>)] pub headers: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] + #[schema(value_type = Option)] pub distribution: Setting, } @@ -539,7 +553,7 @@ impl EmbeddingSettings { } } -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)] +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(rename_all = camelCase, deny_unknown_fields)] pub enum EmbedderSource { From 11ce3b9636cd5cd4aba3cdeef3babf885e1a9785 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 23 Dec 2024 22:00:44 +0100 Subject: [PATCH 08/27] fix the settings --- .../src/routes/indexes/settings.rs | 107 ++++++++++++++---- 1 file changed, 88 insertions(+), 19 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index cf104ee99..10d11b11b 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -34,8 +34,7 @@ macro_rules! make_setting_routes { #[derive(OpenApi)] #[openapi( - nest($((path = "/", api = $attr::$attr),)*), - // paths(/* update_all, get_all, delete_all,*/ $( $attr::get, $attr::update, $attr::delete,)*), + paths(update_all, get_all, delete_all, $( $attr::get, $attr::update, $attr::delete,)*), tags( ( name = "Settings", @@ -82,16 +81,10 @@ macro_rules! make_setting_route { #[allow(unused_imports)] use super::*; - #[derive(OpenApi)] - #[openapi( - paths(get, update, delete,), - )] - pub struct $attr; - #[doc = $camelcase_attr] #[utoipa::path( delete, - path = "/", + path = concat!("{indexUid}/settings", $route), tags = ["Indexes", "Settings"], security(("Bearer" = ["settings.update", "settings.*", "*"])), request_body = $type, @@ -152,7 +145,7 @@ macro_rules! make_setting_route { #[doc = $camelcase_attr] #[utoipa::path( $update_verb, - path = "/", + path = concat!("{indexUid}/settings", $route), tags = ["Indexes", "Settings"], security(("Bearer" = ["settings.update", "settings.*", "*"])), request_body = $type, @@ -235,19 +228,13 @@ macro_rules! make_setting_route { #[doc = $camelcase_attr] #[utoipa::path( get, - path = "/", + path = concat!("{indexUid}/settings", $route), tags = ["Indexes", "Settings"], security(("Bearer" = ["settings.get", "settings.*", "*"])), request_body = $type, responses( - (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( - { - "taskUid": 147, - "indexUid": "movies", - "status": "enqueued", - "type": "settingsUpdate", - "enqueuedAt": "2024-08-08T17:05:55.791772Z" - } + (status = 200, description = concat!($camelcase_attr, " is returned"), body = SummarizedTaskView, content_type = "application/json", example = json!( + <$type>::default() )), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( { @@ -510,6 +497,38 @@ make_setting_routes!( }, ); +#[utoipa::path( + patch, + path = "{indexUid}/settings", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.update", "settings.*", "*"])), + request_body = Settings, + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +/// Update settings +/// +/// Update the settings of an index. +/// Passing null to an index setting will reset it to its default value. +/// Updates in the settings route are partial. This means that any parameters not provided in the body will be left unchanged. +/// If the provided index does not exist, it will be created. pub async fn update_all( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -587,6 +606,28 @@ pub async fn update_all( Ok(HttpResponse::Accepted().json(task)) } +#[utoipa::path( + get, + path = "{indexUid}/settings", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.update", "settings.*", "*"])), + responses( + (status = 200, description = "Settings are returned", body = Settings, content_type = "application/json", example = json!( + Settings::::default() + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +/// All settings +/// +/// This route allows you to retrieve, configure, or reset all of an index's settings at once. pub async fn get_all( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -600,6 +641,34 @@ pub async fn get_all( Ok(HttpResponse::Ok().json(new_settings)) } +#[utoipa::path( + delete, + path = "{indexUid}/settings", + tags = ["Indexes", "Settings"], + security(("Bearer" = ["settings.update", "settings.*", "*"])), + responses( + (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 147, + "indexUid": "movies", + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2024-08-08T17:05:55.791772Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +/// Reset settings +/// +/// Reset all the settings of an index to their default value. pub async fn delete_all( index_scheduler: GuardedData, Data>, index_uid: web::Path, From 9473a2a6ca2921fd81d464c42ebe8142a73481ff Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 15:56:44 +0100 Subject: [PATCH 09/27] add the multi-search --- .../src/routes/indexes/documents.rs | 9 +- crates/meilisearch/src/routes/mod.rs | 12 +- crates/meilisearch/src/routes/multi_search.rs | 116 +++++++++++++++++- crates/meilisearch/src/search/federated.rs | 20 ++- crates/meilisearch/src/search/mod.rs | 16 ++- 5 files changed, 153 insertions(+), 20 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index dee46f2be..bbe312089 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -190,7 +190,6 @@ impl Aggregate for DocumentsFetchAggregator { } } - /// Get one document /// /// Get one document from its primary key. @@ -303,7 +302,6 @@ impl Aggregate for DocumentsDeletionAggregator { } } - /// Delete a document /// /// Delete a single document by id. @@ -1197,13 +1195,16 @@ pub async fn delete_documents_by_filter( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, IntoParams)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct DocumentEditionByFunction { + /// A string containing a RHAI function. #[deserr(default, error = DeserrJsonError)] pub filter: Option, + /// A string containing a filter expression. #[deserr(default, error = DeserrJsonError)] pub context: Option, + /// An object with data Meilisearch should make available for the editing function. #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_document_edition_function)] pub function: String, } @@ -1246,8 +1247,8 @@ impl Aggregate for EditDocumentsByFunctionAggregator { security(("Bearer" = ["documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - DocumentEditionByFunction, ), + request_body = DocumentEditionByFunction, responses( (status = 202, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 838335204..0211f2151 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -2,7 +2,12 @@ use std::collections::BTreeMap; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; -use crate::search::{SimilarQuery, SimilarResult}; +use crate::routes::indexes::documents::DocumentEditionByFunction; +use crate::routes::multi_search::SearchResults; +use crate::search::{ + FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, + SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, +}; use crate::search_queue::SearchQueue; use crate::Opt; use actix_web::web::Data; @@ -64,13 +69,14 @@ pub mod tasks; (path = "/keys", api = api_key::ApiKeyApi), (path = "/metrics", api = metrics::MetricApi), (path = "/logs", api = logs::LogsApi), + (path = "/multi-search", api = multi_search::MultiSearchApi), ), paths(get_health, get_version, get_stats), tags( (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; @@ -89,7 +95,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/snapshots").configure(snapshot::configure)) // done .service(web::resource("/stats").route(web::get().to(get_stats))) // done .service(web::resource("/version").route(web::get().to(get_version))) // done - .service(web::scope("/indexes").configure(indexes::configure)) // WIP + .service(web::scope("/indexes").configure(indexes::configure)) // done .service(web::scope("/multi-search").configure(multi_search::configure)) // TODO .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // TODO .service(web::scope("/metrics").configure(metrics::configure)) // done diff --git a/crates/meilisearch/src/routes/multi_search.rs b/crates/meilisearch/src/routes/multi_search.rs index a2db0b22b..1515dd707 100644 --- a/crates/meilisearch/src/routes/multi_search.rs +++ b/crates/meilisearch/src/routes/multi_search.rs @@ -8,6 +8,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use serde::Serialize; use tracing::debug; +use utoipa::{OpenApi, ToSchema}; use super::multi_search_analytics::MultiSearchAggregator; use crate::analytics::Analytics; @@ -17,20 +18,129 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; use crate::routes::indexes::search::search_kind; use crate::search::{ - add_search_rules, perform_federated_search, perform_search, FederatedSearch, RetrieveVectors, + add_search_rules, perform_federated_search, perform_search, FederatedSearch, FederatedSearchResult, RetrieveVectors, SearchQueryWithIndex, SearchResultWithIndex, }; use crate::search_queue::SearchQueue; + +#[derive(OpenApi)] +#[openapi( + paths(multi_search_with_post), + tags(( + name = "Multi-search", + description = "The `/multi-search` route allows you to perform multiple search queries on one or more indexes by bundling them into a single HTTP request. Multi-search is also known as federated search.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/multi_search"), + + )), +)] +pub struct MultiSearchApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(multi_search_with_post)))); } -#[derive(Serialize)] -struct SearchResults { +#[derive(Serialize, ToSchema)] +pub struct SearchResults { results: Vec, } +/// Perform a multi-search +/// +/// Bundle multiple search queries in a single API request. Use this endpoint to search through multiple indexes at once. +#[utoipa::path( + post, + path = "/", + tag = "Multi-search", + security(("Bearer" = ["search", "*"])), + responses( + (status = OK, description = "Non federated multi-search", body = SearchResults, content_type = "application/json", example = json!( + { + "results":[ + { + "indexUid":"movies", + "hits":[ + { + "id":13682, + "title":"Pooh's Heffalump Movie", + }, + ], + "query":"pooh", + "processingTimeMs":26, + "limit":1, + "offset":0, + "estimatedTotalHits":22 + }, + { + "indexUid":"movies", + "hits":[ + { + "id":12, + "title":"Finding Nemo", + }, + ], + "query":"nemo", + "processingTimeMs":5, + "limit":1, + "offset":0, + "estimatedTotalHits":11 + }, + { + "indexUid":"movie_ratings", + "hits":[ + { + "id":"Us", + "director": "Jordan Peele", + } + ], + "query":"Us", + "processingTimeMs":0, + "limit":1, + "offset":0, + "estimatedTotalHits":1 + } + ] + } + )), + (status = OK, description = "Federated multi-search", body = FederatedSearchResult, content_type = "application/json", example = json!( + { + "hits": [ + { + "id": 42, + "title": "Batman returns", + "overview": "The overview of batman returns", + "_federation": { + "indexUid": "movies", + "queriesPosition": 0 + } + }, + { + "comicsId": "batman-killing-joke", + "description": "This comic is really awesome", + "title": "Batman: the killing joke", + "_federation": { + "indexUid": "comics", + "queriesPosition": 1 + } + }, + ], + "processingTimeMs": 0, + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + "semanticHitCount": 0 + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn multi_search_with_post( index_scheduler: GuardedData, Data>, search_queue: Data, diff --git a/crates/meilisearch/src/search/federated.rs b/crates/meilisearch/src/search/federated.rs index c1c6bb7d7..dec3927e3 100644 --- a/crates/meilisearch/src/search/federated.rs +++ b/crates/meilisearch/src/search/federated.rs @@ -22,6 +22,7 @@ use meilisearch_types::milli::score_details::{ScoreDetails, ScoreValue}; use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget}; use roaring::RoaringBitmap; use serde::Serialize; +use utoipa::ToSchema; use super::ranking_rules::{self, RankingRules}; use super::{ @@ -33,10 +34,11 @@ use crate::routes::indexes::search::search_kind; pub const DEFAULT_FEDERATED_WEIGHT: f64 = 1.0; -#[derive(Debug, Default, Clone, Copy, PartialEq, deserr::Deserr)] +#[derive(Debug, Default, Clone, Copy, PartialEq, deserr::Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct FederationOptions { #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = f64)] pub weight: Weight, } @@ -70,8 +72,9 @@ impl std::ops::Deref for Weight { } } -#[derive(Debug, deserr::Deserr)] +#[derive(Debug, deserr::Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct Federation { #[deserr(default = super::DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError)] pub limit: usize, @@ -83,22 +86,26 @@ pub struct Federation { pub merge_facets: Option, } -#[derive(Copy, Clone, Debug, deserr::Deserr, Default)] +#[derive(Copy, Clone, Debug, deserr::Deserr, Default, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct MergeFacets { #[deserr(default, error = DeserrJsonError)] pub max_values_per_facet: Option, } -#[derive(Debug, deserr::Deserr)] +#[derive(Debug, deserr::Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct FederatedSearch { pub queries: Vec, #[deserr(default)] pub federation: Option, } -#[derive(Serialize, Clone)] + +#[derive(Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct FederatedSearchResult { pub hits: Vec, pub processing_time_ms: u128, @@ -109,6 +116,7 @@ pub struct FederatedSearchResult { pub semantic_hit_count: Option, #[serde(skip_serializing_if = "Option::is_none")] + #[schema(value_type = Option>>)] pub facet_distribution: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub facet_stats: Option>, @@ -355,7 +363,7 @@ struct SearchResultByIndex { facets: Option, } -#[derive(Debug, Clone, Default, Serialize)] +#[derive(Debug, Clone, Default, Serialize, ToSchema)] pub struct FederatedFacets(pub BTreeMap); impl FederatedFacets { diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index cada265dd..7cefb57b6 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -39,7 +39,10 @@ use utoipa::ToSchema; use crate::error::MeilisearchHttpError; mod federated; -pub use federated::{perform_federated_search, FederatedSearch, Federation, FederationOptions}; +pub use federated::{ + perform_federated_search, FederatedSearch, FederatedSearchResult, Federation, + FederationOptions, MergeFacets, +}; mod ranking_rules; @@ -388,8 +391,9 @@ impl SearchQuery { // This struct contains the fields of `SearchQuery` inline. // This is because neither deserr nor serde support `flatten` when using `deny_unknown_fields. // The `From` implementation ensures both structs remain up to date. -#[derive(Debug, Clone, PartialEq, Deserr)] +#[derive(Debug, Clone, PartialEq, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct SearchQueryWithIndex { #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_index_uid)] pub index_uid: IndexUid, @@ -734,8 +738,9 @@ pub struct SimilarResult { pub hits_info: HitsInfo, } -#[derive(Serialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Debug, Clone, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct SearchResultWithIndex { pub index_uid: String, #[serde(flatten)] @@ -746,8 +751,10 @@ pub struct SearchResultWithIndex { #[serde(untagged)] pub enum HitsInfo { #[serde(rename_all = "camelCase")] + #[schema(rename_all = "camelCase")] Pagination { hits_per_page: usize, page: usize, total_pages: usize, total_hits: usize }, #[serde(rename_all = "camelCase")] + #[schema(rename_all = "camelCase")] OffsetLimit { limit: usize, offset: usize, estimated_total_hits: usize }, } @@ -1034,8 +1041,9 @@ pub fn perform_search( Ok(result) } -#[derive(Debug, Clone, Default, Serialize)] +#[derive(Debug, Clone, Default, Serialize, ToSchema)] pub struct ComputedFacets { + #[schema(value_type = Option>>)] pub distribution: BTreeMap>, pub stats: BTreeMap, } From e2686c0fce4b325fe78f669f04e843822a8439e5 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 16:16:56 +0100 Subject: [PATCH 10/27] add the swap indexes --- crates/meilisearch/src/routes/mod.rs | 6 ++- crates/meilisearch/src/routes/multi_search.rs | 1 - crates/meilisearch/src/routes/swap_indexes.rs | 39 ++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 0211f2151..da4cb5528 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -4,6 +4,7 @@ use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::routes::indexes::documents::DocumentEditionByFunction; use crate::routes::multi_search::SearchResults; +use crate::routes::swap_indexes::SwapIndexesPayload; use crate::search::{ FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, @@ -70,13 +71,14 @@ pub mod tasks; (path = "/metrics", api = metrics::MetricApi), (path = "/logs", api = logs::LogsApi), (path = "/multi-search", api = multi_search::MultiSearchApi), + (path = "/swap-indexes", api = swap_indexes::SwapIndexesApi), ), paths(get_health, get_version, get_stats), tags( (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; @@ -96,7 +98,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("/stats").route(web::get().to(get_stats))) // done .service(web::resource("/version").route(web::get().to(get_version))) // done .service(web::scope("/indexes").configure(indexes::configure)) // done - .service(web::scope("/multi-search").configure(multi_search::configure)) // TODO + .service(web::scope("/multi-search").configure(multi_search::configure)) // done .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // TODO .service(web::scope("/metrics").configure(metrics::configure)) // done .service(web::scope("/experimental-features").configure(features::configure)); diff --git a/crates/meilisearch/src/routes/multi_search.rs b/crates/meilisearch/src/routes/multi_search.rs index 1515dd707..711bdd03c 100644 --- a/crates/meilisearch/src/routes/multi_search.rs +++ b/crates/meilisearch/src/routes/multi_search.rs @@ -23,7 +23,6 @@ use crate::search::{ }; use crate::search_queue::SearchQueue; - #[derive(OpenApi)] #[openapi( paths(multi_search_with_post), diff --git a/crates/meilisearch/src/routes/swap_indexes.rs b/crates/meilisearch/src/routes/swap_indexes.rs index 9b8b67e63..2d46642c0 100644 --- a/crates/meilisearch/src/routes/swap_indexes.rs +++ b/crates/meilisearch/src/routes/swap_indexes.rs @@ -9,6 +9,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; use serde::Serialize; +use utoipa::{OpenApi, ToSchema}; use super::{get_task_id, is_dry_run, SummarizedTaskView}; use crate::analytics::{Aggregate, Analytics}; @@ -18,13 +19,18 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; use crate::Opt; +#[derive(OpenApi)] +#[openapi(paths(swap_indexes))] +pub struct SwapIndexesApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(swap_indexes)))); } -#[derive(Deserr, Debug, Clone, PartialEq, Eq)] +#[derive(Deserr, Debug, Clone, PartialEq, Eq, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct SwapIndexesPayload { + /// Array of the two indexUids to be swapped #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_swap_indexes)] indexes: Vec, } @@ -50,6 +56,37 @@ impl Aggregate for IndexSwappedAnalytics { } } +/// Swap indexes +/// +/// Swap the documents, settings, and task history of two or more indexes. You can only swap indexes in pairs. However, a single request can swap as many index pairs as you wish. +/// Swapping indexes is an atomic transaction: either all indexes are successfully swapped, or none are. +/// Swapping indexA and indexB will also replace every mention of indexA by indexB and vice-versa in the task history. enqueued tasks are left unmodified. +#[utoipa::path( + post, + path = "/", + tag = "Indexes", + security(("Bearer" = ["search", "*"])), + request_body = Vec, + responses( + (status = OK, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + { + "taskUid": 3, + "indexUid": null, + "status": "enqueued", + "type": "indexSwap", + "enqueuedAt": "2021-08-12T10:00:00.000000Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] pub async fn swap_indexes( index_scheduler: GuardedData, Data>, params: AwebJson, DeserrJsonError>, From 8a2a1e4d27b2f0366c7eeaa63f926d85c9ee09bc Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 16:38:01 +0100 Subject: [PATCH 11/27] add the experimental features route --- crates/meilisearch/src/routes/features.rs | 70 ++++++++++++++++++++++- crates/meilisearch/src/routes/mod.rs | 6 +- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs index 5d93adc02..8347a9496 100644 --- a/crates/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -8,12 +8,27 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use serde::Serialize; use tracing::debug; +use utoipa::{OpenApi, ToSchema}; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +#[derive(OpenApi)] +#[openapi( + paths(get_features), + tags(( + name = "Experimental features", + description = "The `/experimental-features` route allows you to activate or deactivate some of Meilisearch's experimental features. + +This route is **synchronous**. This means that no task object will be returned, and any activated or deactivated features will be made available or unavailable immediately.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/experimental_features"), + + )), +)] +pub struct ExperimentalFeaturesApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") @@ -22,6 +37,32 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } +/// Get all experimental features +/// +/// Get a list of all experimental features that can be activated via the /experimental-features route and whether or not they are currently activated. +#[utoipa::path( + post, + path = "/", + tag = "Experimental features", + security(("Bearer" = ["experimental_features.get", "experimental_features.*", "*"])), + responses( + (status = OK, description = "Experimental features are returned", body = RuntimeTogglableFeatures, content_type = "application/json", example = json!( + { + "metrics": false, + "logsRoute": true, + "vectorSearch": false, + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_features( index_scheduler: GuardedData< ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_GET }>, @@ -35,8 +76,9 @@ async fn get_features( HttpResponse::Ok().json(features) } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct RuntimeTogglableFeatures { #[deserr(default)] pub vector_store: Option, @@ -79,6 +121,32 @@ impl Aggregate for PatchExperimentalFeatureAnalytics { } } +/// Configure experimental features +/// +/// Activate or deactivate experimental features. +#[utoipa::path( + patch, + path = "/", + tag = "Experimental features", + security(("Bearer" = ["experimental_features.update", "experimental_features.*", "*"])), + responses( + (status = OK, description = "Experimental features are returned", body = RuntimeTogglableFeatures, content_type = "application/json", example = json!( + { + "metrics": false, + "logsRoute": true, + "vectorSearch": false, + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn patch_features( index_scheduler: GuardedData< ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>, diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index da4cb5528..b597b82a1 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; +use crate::routes::features::RuntimeTogglableFeatures; use crate::routes::indexes::documents::DocumentEditionByFunction; use crate::routes::multi_search::SearchResults; use crate::routes::swap_indexes::SwapIndexesPayload; @@ -72,13 +73,14 @@ pub mod tasks; (path = "/logs", api = logs::LogsApi), (path = "/multi-search", api = multi_search::MultiSearchApi), (path = "/swap-indexes", api = swap_indexes::SwapIndexesApi), + (path = "/experimental-features", api = features::ExperimentalFeaturesApi), ), paths(get_health, get_version, get_stats), tags( (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; @@ -99,7 +101,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("/version").route(web::get().to(get_version))) // done .service(web::scope("/indexes").configure(indexes::configure)) // done .service(web::scope("/multi-search").configure(multi_search::configure)) // done - .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // TODO + .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // done .service(web::scope("/metrics").configure(metrics::configure)) // done .service(web::scope("/experimental-features").configure(features::configure)); } From 1dd33af8a3fbef70a1fd7edf394cda04360f1f0a Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 17:16:52 +0100 Subject: [PATCH 12/27] add the batches --- crates/meilisearch-types/src/batch_view.rs | 4 +- crates/meilisearch-types/src/batches.rs | 4 +- crates/meilisearch-types/src/task_view.rs | 1 + crates/meilisearch/src/routes/batches.rs | 123 ++++++++++++++++++++- crates/meilisearch/src/routes/features.rs | 2 +- crates/meilisearch/src/routes/mod.rs | 40 ++++--- crates/milli/src/progress.rs | 7 +- 7 files changed, 156 insertions(+), 25 deletions(-) diff --git a/crates/meilisearch-types/src/batch_view.rs b/crates/meilisearch-types/src/batch_view.rs index 08d25413c..112abd1dd 100644 --- a/crates/meilisearch-types/src/batch_view.rs +++ b/crates/meilisearch-types/src/batch_view.rs @@ -1,13 +1,15 @@ use milli::progress::ProgressView; use serde::Serialize; use time::{Duration, OffsetDateTime}; +use utoipa::ToSchema; use crate::batches::{Batch, BatchId, BatchStats}; use crate::task_view::DetailsView; use crate::tasks::serialize_duration; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct BatchView { pub uid: BatchId, pub progress: Option, diff --git a/crates/meilisearch-types/src/batches.rs b/crates/meilisearch-types/src/batches.rs index 664dafa7a..7910a5af4 100644 --- a/crates/meilisearch-types/src/batches.rs +++ b/crates/meilisearch-types/src/batches.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use milli::progress::ProgressView; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; +use utoipa::ToSchema; use crate::task_view::DetailsView; use crate::tasks::{Kind, Status}; @@ -25,8 +26,9 @@ pub struct Batch { pub finished_at: Option, } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct BatchStats { pub total_nb_tasks: BatchId, pub status: BTreeMap, diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 467408097..23af055d6 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -69,6 +69,7 @@ impl TaskView { #[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct DetailsView { /// Number of documents received for documentAdditionOrUpdate task. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/meilisearch/src/routes/batches.rs b/crates/meilisearch/src/routes/batches.rs index 36bf31605..a5fc77a3d 100644 --- a/crates/meilisearch/src/routes/batches.rs +++ b/crates/meilisearch/src/routes/batches.rs @@ -8,17 +8,77 @@ use meilisearch_types::deserr::DeserrQueryParamError; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use serde::Serialize; +use utoipa::{OpenApi, ToSchema}; use super::tasks::TasksFilterQuery; use super::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +#[derive(OpenApi)] +#[openapi( + paths(get_batch, get_batches), + tags(( + name = "Batches", + description = "The /batches route gives information about the progress of batches of asynchronous operations.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/batches"), + + )), +)] +pub struct BatchesApi; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::get().to(SeqHandler(get_batches)))) .service(web::resource("/{batch_id}").route(web::get().to(SeqHandler(get_batch)))); } +/// Get one batch +/// +/// Get a single batch. +#[utoipa::path( + get, + path = "/{batchUid}", + tag = "Batches", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params( + ("batchUid" = String, Path, example = "8685", description = "The unique batch id", nullable = false), + ), + responses( + (status = OK, description = "Return the batch", body = BatchView, content_type = "application/json", example = json!( + { + "uid": 1, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "progress": null, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "INDEX_NAME": 1 + } + }, + "duration": "PT0.364788S", + "startedAt": "2024-12-10T15:48:49.672141Z", + "finishedAt": "2024-12-10T15:48:50.036929Z" + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_batch( index_scheduler: GuardedData, Data>, batch_uid: web::Path, @@ -39,14 +99,14 @@ async fn get_batch( let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?; if let Some(batch) = batches.first() { - let task_view = BatchView::from_batch(batch); - Ok(HttpResponse::Ok().json(task_view)) + let batch_view = BatchView::from_batch(batch); + Ok(HttpResponse::Ok().json(batch_view)) } else { Err(index_scheduler::Error::BatchNotFound(batch_uid).into()) } } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct AllBatches { results: Vec, total: u64, @@ -55,6 +115,63 @@ pub struct AllBatches { next: Option, } +/// Get batches +/// +/// List all batches, regardless of index. The batch objects are contained in the results array. +/// Batches are always returned in descending order of uid. This means that by default, the most recently created batch objects appear first. +/// Batch results are paginated and can be filtered with query parameters. +#[utoipa::path( + get, + path = "/", + tag = "Batches", + security(("Bearer" = ["tasks.get", "tasks.*", "*"])), + params(TasksFilterQuery), + responses( + (status = OK, description = "Return the batches", body = AllBatches, content_type = "application/json", example = json!( + { + "results": [ + { + "uid": 2, + "details": { + "stopWords": [ + "of", + "the" + ] + }, + "progress": null, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "INDEX_NAME": 1 + } + }, + "duration": "PT0.110083S", + "startedAt": "2024-12-10T15:49:04.995321Z", + "finishedAt": "2024-12-10T15:49:05.105404Z" + } + ], + "total": 3, + "limit": 1, + "from": 2, + "next": 1 + } + )), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] async fn get_batches( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, diff --git a/crates/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs index 8347a9496..506e73b2e 100644 --- a/crates/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -17,7 +17,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_features), + paths(get_features, patch_features), tags(( name = "Experimental features", description = "The `/experimental-features` route allows you to activate or deactivate some of Meilisearch's experimental features. diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index b597b82a1..b97d129db 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -2,6 +2,9 @@ use std::collections::BTreeMap; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; +use crate::milli::progress::ProgressStepView; +use crate::milli::progress::ProgressView; +use crate::routes::batches::AllBatches; use crate::routes::features::RuntimeTogglableFeatures; use crate::routes::indexes::documents::DocumentEditionByFunction; use crate::routes::multi_search::SearchResults; @@ -16,6 +19,8 @@ use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; use index_scheduler::IndexScheduler; use meilisearch_auth::AuthController; +use meilisearch_types::batch_view::BatchView; +use meilisearch_types::batches::BatchStats; use meilisearch_types::error::{Code, ErrorType, ResponseError}; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::keys::CreateApiKey; @@ -63,6 +68,7 @@ pub mod tasks; #[openapi( nest( (path = "/tasks", api = tasks::TaskApi), + (path = "/batches", api = batches::BatchesApi), (path = "/indexes", api = indexes::IndexesApi), // We must stop the search path here because the rest must be configured by each route individually (path = "/indexes", api = indexes::search::SearchApi), @@ -80,29 +86,29 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; pub fn configure(cfg: &mut web::ServiceConfig) { let openapi = MeilisearchApi::openapi(); - cfg.service(web::scope("/tasks").configure(tasks::configure)) // done - .service(web::scope("/batches").configure(batches::configure)) // TODO - .service(Scalar::with_url("/scalar", openapi.clone())) // done - .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) // done - .service(Redoc::with_url("/redoc", openapi)) // done - .service(web::resource("/health").route(web::get().to(get_health))) // done - .service(web::scope("/logs").configure(logs::configure)) // done - .service(web::scope("/keys").configure(api_key::configure)) // done - .service(web::scope("/dumps").configure(dump::configure)) // done - .service(web::scope("/snapshots").configure(snapshot::configure)) // done - .service(web::resource("/stats").route(web::get().to(get_stats))) // done - .service(web::resource("/version").route(web::get().to(get_version))) // done - .service(web::scope("/indexes").configure(indexes::configure)) // done - .service(web::scope("/multi-search").configure(multi_search::configure)) // done - .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // done - .service(web::scope("/metrics").configure(metrics::configure)) // done + cfg.service(web::scope("/tasks").configure(tasks::configure)) + .service(web::scope("/batches").configure(batches::configure)) + .service(Scalar::with_url("/scalar", openapi.clone())) + .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) + .service(Redoc::with_url("/redoc", openapi)) + .service(web::resource("/health").route(web::get().to(get_health))) + .service(web::scope("/logs").configure(logs::configure)) + .service(web::scope("/keys").configure(api_key::configure)) + .service(web::scope("/dumps").configure(dump::configure)) + .service(web::scope("/snapshots").configure(snapshot::configure)) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))) + .service(web::scope("/indexes").configure(indexes::configure)) + .service(web::scope("/multi-search").configure(multi_search::configure)) + .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) + .service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/experimental-features").configure(features::configure)); } diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index accc2cf56..622ec9842 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use serde::Serialize; +use utoipa::ToSchema; pub trait Step: 'static + Send + Sync { fn name(&self) -> Cow<'static, str>; @@ -136,15 +137,17 @@ macro_rules! make_atomic_progress { make_atomic_progress!(Document alias AtomicDocumentStep => "document" ); make_atomic_progress!(Payload alias AtomicPayloadStep => "payload" ); -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct ProgressView { pub steps: Vec, pub percentage: f32, } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct ProgressStepView { pub current_step: Cow<'static, str>, pub finished: u32, From aab6ffec30e830e822d37f5a95a74d905b01ce9c Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 18:26:30 +0100 Subject: [PATCH 13/27] fix and review all the documents route --- crates/meilisearch-types/src/settings.rs | 2 +- .../src/routes/indexes/documents.rs | 83 ++++++++++--------- crates/meilisearch/src/routes/mod.rs | 3 +- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index a5416583b..b90289413 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -255,7 +255,7 @@ pub struct Settings { /// Embedder required for performing meaning-based search queries. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] - #[schema(value_type = String)] // TODO: TAMO + #[schema(value_type = Option>)] pub embedders: Setting>, /// Maximum duration of a search query. #[serde(default, skip_serializing_if = "Setting::is_not_set")] diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index bbe312089..d93f5df9f 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -107,14 +107,18 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -#[derive(Debug, Deserr, IntoParams)] +#[derive(Debug, Deserr, IntoParams, ToSchema)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] +#[schema(rename_all = "camelCase")] pub struct GetDocument { #[deserr(default, error = DeserrQueryParamError)] #[param(value_type = Option>)] + #[schema(value_type = Option>)] fields: OptionStarOrList, #[deserr(default, error = DeserrQueryParamError)] #[param(value_type = Option)] + #[schema(value_type = Option)] retrieve_vectors: Param, } @@ -195,8 +199,8 @@ impl Aggregate for DocumentsFetchAggregator { /// Get one document from its primary key. #[utoipa::path( get, - path = "/{indexUid}/documents/{documentId}", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents/{documentId}", + tag = "Documents", security(("Bearer" = ["documents.get", "documents.*", "*"])), params( ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), @@ -307,12 +311,12 @@ impl Aggregate for DocumentsDeletionAggregator { /// Delete a single document by id. #[utoipa::path( delete, - path = "/{indexUid}/documents/{documentsId}", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents/{documentId}", + tag = "Documents", security(("Bearer" = ["documents.delete", "documents.*", "*"])), params( ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), - ("documentsId" = String, Path, example = "movies", description = "Document Identifier", nullable = false), + ("documentId" = String, Path, example = "853", description = "Document Identifier", nullable = false), ), responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( @@ -386,6 +390,7 @@ pub struct BrowseQueryGet { #[derive(Debug, Deserr, IntoParams, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[schema(rename_all = "camelCase")] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct BrowseQuery { #[schema(default, example = 150)] #[deserr(default, error = DeserrJsonError)] @@ -409,13 +414,11 @@ pub struct BrowseQuery { /// Get a set of documents. #[utoipa::path( post, - path = "/{indexUid}/documents/fetch", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents/fetch", + tag = "Documents", security(("Bearer" = ["documents.delete", "documents.*", "*"])), - params( - ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - BrowseQuery, - ), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + request_body = BrowseQuery, responses( (status = 200, description = "Task successfully enqueued", body = PaginationView, content_type = "application/json", example = json!( { @@ -486,8 +489,8 @@ pub async fn documents_by_query_post( /// Get documents by batches. #[utoipa::path( get, - path = "/{indexUid}/documents", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents", + tag = "Documents", security(("Bearer" = ["documents.get", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), @@ -607,7 +610,7 @@ fn documents_by_query( #[derive(Deserialize, Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] -#[into_params(rename_all = "camelCase")] +#[into_params(parameter_in = Query, rename_all = "camelCase")] pub struct UpdateDocumentsQuery { /// The primary key of the documents. primaryKey is optional. If you want to set the primary key of your index through this route, /// it only has to be done the first time you add documents to the index. After which it will be ignored if given. @@ -683,14 +686,15 @@ impl Aggregate for DocumentsAggregator { /// > This object accepts keys corresponding to the different embedders defined your index settings. #[utoipa::path( post, - path = "/{indexUid}/documents", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents", + tag = "Documents", security(("Bearer" = ["documents.add", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), // Here we can use the post version of the browse query since it contains the exact same parameter UpdateDocumentsQuery, ), + request_body = serde_json::Value, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { @@ -783,14 +787,15 @@ pub async fn replace_documents( /// > This object accepts keys corresponding to the different embedders defined your index settings. #[utoipa::path( put, - path = "/{indexUid}/documents", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents", + tag = "Documents", security(("Bearer" = ["documents.add", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), // Here we can use the post version of the browse query since it contains the exact same parameter UpdateDocumentsQuery, ), + request_body = serde_json::Value, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { @@ -1045,18 +1050,18 @@ async fn copy_body_to_file( Ok(read_file) } -/// Delete documents +/// Delete documents by batch /// -/// Delete a selection of documents based on array of document id's. +/// Delete a set of documents based on an array of document ids. #[utoipa::path( - delete, - path = "/{indexUid}/documents", - tags = ["Indexes", "Documents"], + post, + path = "{indexUid}/delete-batch", + tag = "Documents", security(("Bearer" = ["documents.delete", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - // TODO: how to task an array of strings in parameter ), + request_body = Vec, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { @@ -1116,8 +1121,9 @@ pub async fn delete_documents_batch( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, IntoParams)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] pub struct DocumentDeletionByFilter { #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_document_filter)] filter: Value, @@ -1128,15 +1134,13 @@ pub struct DocumentDeletionByFilter { /// Delete a set of documents based on a filter. #[utoipa::path( post, - path = "/{indexUid}/documents/delete", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents/delete", + tag = "Documents", security(("Bearer" = ["documents.delete", "documents.*", "*"])), - params( - ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - DocumentDeletionByFilter, - ), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), + request_body = DocumentDeletionByFilter, responses( - (status = 202, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( + (status = ACCEPTED, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { "taskUid": 147, "indexUid": null, @@ -1242,8 +1246,8 @@ impl Aggregate for EditDocumentsByFunctionAggregator { /// Use a [RHAI function](https://rhai.rs/book/engine/hello-world.html) to edit one or more documents directly in Meilisearch. #[utoipa::path( post, - path = "/{indexUid}/documents/edit", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents/edit", + tag = "Documents", security(("Bearer" = ["documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), @@ -1343,13 +1347,10 @@ pub async fn edit_documents_by_function( /// Delete all documents in the specified index. #[utoipa::path( delete, - path = "/{indexUid}/documents", - tags = ["Indexes", "Documents"], + path = "{indexUid}/documents", + tag = "Documents", security(("Bearer" = ["documents.delete", "documents.*", "*"])), - params( - ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - UpdateDocumentsQuery, - ), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index b97d129db..bf51ed4de 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -6,6 +6,7 @@ use crate::milli::progress::ProgressStepView; use crate::milli::progress::ProgressView; use crate::routes::batches::AllBatches; use crate::routes::features::RuntimeTogglableFeatures; +use crate::routes::indexes::documents::DocumentDeletionByFilter; use crate::routes::indexes::documents::DocumentEditionByFunction; use crate::routes::multi_search::SearchResults; use crate::routes::swap_indexes::SwapIndexesPayload; @@ -86,7 +87,7 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; From 5f55e884841314fe35db3d7fd39c7cf4a95ce11d Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 26 Dec 2024 18:44:08 +0100 Subject: [PATCH 14/27] review all the parameters and tags --- crates/meilisearch/src/routes/api_key.rs | 6 +++--- .../src/routes/indexes/facet_search.rs | 8 +++----- crates/meilisearch/src/routes/indexes/mod.rs | 4 ++-- .../meilisearch/src/routes/indexes/settings.rs | 18 ++++++++++++------ .../meilisearch/src/routes/indexes/similar.rs | 12 +++++------- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/crates/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs index 2a08448db..a0c34c3e4 100644 --- a/crates/meilisearch/src/routes/api_key.rs +++ b/crates/meilisearch/src/routes/api_key.rs @@ -208,7 +208,7 @@ pub async fn list_api_keys( /// Get an API key from its `uid` or its `key` field. #[utoipa::path( get, - path = "/{key}", + path = "/{uidOrKey}", tag = "Keys", security(("Bearer" = ["keys.get", "keys.*", "*"])), params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), @@ -275,7 +275,7 @@ pub async fn get_api_key( /// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. #[utoipa::path( patch, - path = "/{key}", + path = "/{uidOrKey}", tag = "Keys", security(("Bearer" = ["keys.update", "keys.*", "*"])), params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), @@ -345,7 +345,7 @@ pub async fn patch_api_key( /// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. #[utoipa::path( delete, - path = "/{key}", + path = "/{uidOrKey}", tag = "Keys", security(("Bearer" = ["keys.delete", "keys.*", "*"])), params(("uidOrKey" = String, Path, format = Password, example = "7b198a7f-52a0-4188-8762-9ad93cd608b2", description = "The `uid` or `key` field of an existing API key", nullable = false)), diff --git a/crates/meilisearch/src/routes/indexes/facet_search.rs b/crates/meilisearch/src/routes/indexes/facet_search.rs index ab335c528..7a41f1f81 100644 --- a/crates/meilisearch/src/routes/indexes/facet_search.rs +++ b/crates/meilisearch/src/routes/indexes/facet_search.rs @@ -177,12 +177,10 @@ impl Aggregate for FacetSearchAggregator { /// Search for a facet value within a given facet. #[utoipa::path( post, - path = "/{indexUid}/facet-search", - tags = ["Indexes", "Facet Search"], + path = "{indexUid}/facet-search", + tag = "Facet Search", security(("Bearer" = ["search", "*"])), - params( - ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - ), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = FacetSearchQuery, responses( (status = 200, description = "The documents are returned", body = SearchResult, content_type = "application/json", example = json!( diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index c6f2a9397..96a7d0131 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -110,7 +110,7 @@ impl IndexView { #[derive(Deserr, Debug, Clone, Copy, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] -#[into_params(rename_all = "camelCase")] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct ListIndexes { /// The number of indexes to skip before starting to retrieve anything #[param(value_type = Option, default, example = 100)] @@ -515,7 +515,7 @@ impl From for IndexStats { #[utoipa::path( get, path = "/{indexUid}/stats", - tags = ["Indexes", "Stats"], + tag = "Stats", security(("Bearer" = ["stats.get", "stats.*", "*"])), params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), responses( diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 10d11b11b..17319f830 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -85,8 +85,9 @@ macro_rules! make_setting_route { #[utoipa::path( delete, path = concat!("{indexUid}/settings", $route), - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = $type, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( @@ -146,8 +147,9 @@ macro_rules! make_setting_route { #[utoipa::path( $update_verb, path = concat!("{indexUid}/settings", $route), - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = $type, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( @@ -229,8 +231,9 @@ macro_rules! make_setting_route { #[utoipa::path( get, path = concat!("{indexUid}/settings", $route), - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.get", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = $type, responses( (status = 200, description = concat!($camelcase_attr, " is returned"), body = SummarizedTaskView, content_type = "application/json", example = json!( @@ -500,8 +503,9 @@ make_setting_routes!( #[utoipa::path( patch, path = "{indexUid}/settings", - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = Settings, responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( @@ -609,8 +613,9 @@ pub async fn update_all( #[utoipa::path( get, path = "{indexUid}/settings", - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), responses( (status = 200, description = "Settings are returned", body = Settings, content_type = "application/json", example = json!( Settings::::default() @@ -644,8 +649,9 @@ pub async fn get_all( #[utoipa::path( delete, path = "{indexUid}/settings", - tags = ["Indexes", "Settings"], + tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), responses( (status = 200, description = "Task successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!( { diff --git a/crates/meilisearch/src/routes/indexes/similar.rs b/crates/meilisearch/src/routes/indexes/similar.rs index 63022b28f..4e0673a7d 100644 --- a/crates/meilisearch/src/routes/indexes/similar.rs +++ b/crates/meilisearch/src/routes/indexes/similar.rs @@ -51,8 +51,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) { /// Retrieve documents similar to a specific search result. #[utoipa::path( get, - path = "/{indexUid}/similar", - tags = ["Indexes", "Similar documents"], + path = "{indexUid}/similar", + tag = "Similar documents", security(("Bearer" = ["search", "*"])), params( ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), @@ -135,12 +135,10 @@ pub async fn similar_get( /// Retrieve documents similar to a specific search result. #[utoipa::path( post, - path = "/{indexUid}/similar", - tags = ["Indexes", "Similar documents"], + path = "{indexUid}/similar", + tag = "Similar documents", security(("Bearer" = ["search", "*"])), - params( - ("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false), - ), + params(("indexUid" = String, Path, example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = SimilarQuery, responses( (status = 200, description = "The documents are returned", body = SimilarResult, content_type = "application/json", example = json!( From ac944f0960e5a04378e83bbe2abd35223267ed64 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 30 Dec 2024 11:40:58 +0100 Subject: [PATCH 15/27] review all the return type --- crates/meilisearch-types/src/task_view.rs | 2 +- crates/meilisearch/src/routes/api_key.rs | 18 +++++++----------- .../src/routes/indexes/documents.rs | 15 ++++++++++----- .../meilisearch/src/routes/indexes/settings.rs | 3 +-- crates/meilisearch/src/routes/mod.rs | 4 +++- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 23af055d6..6032843aa 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -16,7 +16,7 @@ pub struct TaskView { #[schema(value_type = u32, example = 4312)] pub uid: TaskId, /// The unique identifier of the index where this task is operated. - #[schema(example = json!("movies"))] + #[schema(value_type = Option, example = json!("movies"))] pub batch_uid: Option, #[serde(default)] pub index_uid: Option, diff --git a/crates/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs index a0c34c3e4..e45326069 100644 --- a/crates/meilisearch/src/routes/api_key.rs +++ b/crates/meilisearch/src/routes/api_key.rs @@ -16,7 +16,7 @@ use time::OffsetDateTime; use utoipa::{IntoParams, OpenApi, ToSchema}; use uuid::Uuid; -use super::{PAGINATION_DEFAULT_LIMIT, PAGINATION_DEFAULT_LIMIT_FN}; +use super::{PaginationView, PAGINATION_DEFAULT_LIMIT, PAGINATION_DEFAULT_LIMIT_FN}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; @@ -134,7 +134,6 @@ impl ListApiKeys { /// Get API Keys /// /// List all API Keys -/// TODO: Tamo fix the return type #[utoipa::path( get, path = "/", @@ -142,7 +141,7 @@ impl ListApiKeys { security(("Bearer" = ["keys.get", "keys.*", "*"])), params(ListApiKeys), responses( - (status = 202, description = "List of keys", body = serde_json::Value, content_type = "application/json", example = json!( + (status = 202, description = "List of keys", body = PaginationView, content_type = "application/json", example = json!( { "results": [ { @@ -268,11 +267,10 @@ pub async fn get_api_key( } -/// Update an API Key +/// Update a Key /// -/// Update an API key from its `uid` or its `key` field. -/// Only the `name` and `description` of the api key can be updated. -/// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. +/// Update the name and description of an API key. +/// Updates to keys are partial. This means you should provide only the fields you intend to update, as any fields not present in the payload will remain unchanged. #[utoipa::path( patch, path = "/{uidOrKey}", @@ -338,11 +336,9 @@ pub async fn patch_api_key( -/// Update an API Key +/// Delete a key /// -/// Update an API key from its `uid` or its `key` field. -/// Only the `name` and `description` of the api key can be updated. -/// If there is an issue with the `key` or `uid` of a key, then you must recreate one from scratch. +/// Delete the specified API key. #[utoipa::path( delete, path = "/{uidOrKey}", diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index d93f5df9f..3da24859d 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -372,25 +372,30 @@ pub async fn delete_document( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr)] +#[derive(Debug, Deserr, IntoParams)] #[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)] +#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct BrowseQueryGet { + #[param(default, value_type = Option)] #[deserr(default, error = DeserrQueryParamError)] offset: Param, + #[param(default, value_type = Option)] #[deserr(default = Param(PAGINATION_DEFAULT_LIMIT), error = DeserrQueryParamError)] limit: Param, + #[param(default, value_type = Option>)] #[deserr(default, error = DeserrQueryParamError)] fields: OptionStarOrList, + #[param(default, value_type = Option)] #[deserr(default, error = DeserrQueryParamError)] retrieve_vectors: Param, + #[param(default, value_type = Option, example = "popularity > 1000")] #[deserr(default, error = DeserrQueryParamError)] filter: Option, } -#[derive(Debug, Deserr, IntoParams, ToSchema)] +#[derive(Debug, Deserr, ToSchema)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[schema(rename_all = "camelCase")] -#[into_params(rename_all = "camelCase", parameter_in = Query)] pub struct BrowseQuery { #[schema(default, example = 150)] #[deserr(default, error = DeserrJsonError)] @@ -404,7 +409,7 @@ pub struct BrowseQuery { #[schema(default, example = true)] #[deserr(default, error = DeserrJsonError)] retrieve_vectors: bool, - #[schema(default, example = "popularity > 1000")] + #[schema(default, value_type = Option, example = "popularity > 1000")] #[deserr(default, error = DeserrJsonError)] filter: Option, } @@ -494,7 +499,7 @@ pub async fn documents_by_query_post( security(("Bearer" = ["documents.get", "documents.*", "*"])), params( ("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false), - BrowseQuery + BrowseQueryGet ), responses( (status = 200, description = "The documents are returned", body = PaginationView, content_type = "application/json", example = json!( diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 17319f830..58197acda 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -234,9 +234,8 @@ macro_rules! make_setting_route { tag = "Settings", security(("Bearer" = ["settings.get", "settings.*", "*"])), params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), - request_body = $type, responses( - (status = 200, description = concat!($camelcase_attr, " is returned"), body = SummarizedTaskView, content_type = "application/json", example = json!( + (status = 200, description = concat!($camelcase_attr, " is returned"), body = $type, content_type = "application/json", example = json!( <$type>::default() )), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index bf51ed4de..684234be4 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -8,6 +8,7 @@ use crate::routes::batches::AllBatches; use crate::routes::features::RuntimeTogglableFeatures; use crate::routes::indexes::documents::DocumentDeletionByFilter; use crate::routes::indexes::documents::DocumentEditionByFunction; +use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; use crate::routes::swap_indexes::SwapIndexesPayload; use crate::search::{ @@ -87,7 +88,7 @@ pub mod tasks; (name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."), ), modifiers(&OpenApiAuth), - components(schemas(DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind)) )] pub struct MeilisearchApi; @@ -200,6 +201,7 @@ pub struct Pagination { #[derive(Debug, Clone, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] pub struct PaginationView { pub results: Vec, pub offset: usize, From 0b104b3efa5e738827df4df4abde1ce783517779 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 30 Dec 2024 11:46:31 +0100 Subject: [PATCH 16/27] fix the list indexes --- crates/meilisearch/src/routes/indexes/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 96a7d0131..2b1fddd6b 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -18,7 +18,9 @@ use time::OffsetDateTime; use tracing::debug; use utoipa::{IntoParams, OpenApi, ToSchema}; -use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT}; +use super::{ + get_task_id, Pagination, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, +}; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::{AuthenticationError, GuardedData}; @@ -138,7 +140,7 @@ impl ListIndexes { security(("Bearer" = ["indexes.get", "indexes.*", "*"])), params(ListIndexes), responses( - (status = 200, description = "Indexes are returned", body = serde_json::Value, content_type = "application/json", example = json!( + (status = 200, description = "Indexes are returned", body = PaginationView, content_type = "application/json", example = json!( { "results": [ { From 4456df5a461d48f3490e822b8b12d5ead63e84fa Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 30 Dec 2024 14:33:22 +0100 Subject: [PATCH 17/27] fix some tests --- Cargo.lock | 30 ------------------- .../after_registering_settings_task.snap | 9 +++++- ...ter_registering_settings_task_vectors.snap | 9 +++++- crates/meilisearch/Cargo.toml | 3 -- crates/meilisearch/src/routes/mod.rs | 13 ++++---- 5 files changed, 22 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb405d891..6592f4711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,7 +71,6 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "zstd", ] [[package]] @@ -93,7 +92,6 @@ dependencies = [ "bytestring", "cfg-if", "http 0.2.11", - "regex", "regex-lite", "serde", "tracing", @@ -199,7 +197,6 @@ dependencies = [ "mime", "once_cell", "pin-project-lite", - "regex", "regex-lite", "serde", "serde_json", @@ -3471,7 +3468,6 @@ dependencies = [ "clap", "crossbeam-channel", "deserr", - "doc-comment", "dump", "either", "file-store", @@ -3537,8 +3533,6 @@ dependencies = [ "url", "urlencoding", "utoipa", - "utoipa-rapidoc", - "utoipa-redoc", "utoipa-scalar", "uuid", "wiremock", @@ -5988,30 +5982,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "utoipa-rapidoc" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f5e784e313457e79d65c378bdfc2f74275f0db91a72252be7d34360ec2afbe1" -dependencies = [ - "actix-web", - "serde", - "serde_json", - "utoipa", -] - -[[package]] -name = "utoipa-redoc" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9218304bba9a0ea5e92085b0a427ccce5fd56eaaf6436d245b7578e6a95787e1" -dependencies = [ - "actix-web", - "serde", - "serde_json", - "utoipa", -] - [[package]] name = "utoipa-scalar" version = "0.2.0" diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap index 92e37550a..e18f80747 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap @@ -1,13 +1,20 @@ --- +<<<<<<< HEAD:crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap source: crates/index-scheduler/src/scheduler/test.rs snapshot_kind: text +||||||| parent of 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap +source: crates/index-scheduler/src/lib.rs +======= +source: crates/index-scheduler/src/lib.rs +snapshot_kind: text +>>>>>>> 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap index 33bd5c0d2..f2216a4a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap @@ -1,13 +1,20 @@ --- +<<<<<<< HEAD:crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap source: crates/index-scheduler/src/scheduler/test_embedders.rs snapshot_kind: text +||||||| parent of 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap +source: crates/index-scheduler/src/lib.rs +======= +source: crates/index-scheduler/src/lib.rs +snapshot_kind: text +>>>>>>> 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }, "B_small_hf": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }, "B_small_hf": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 2b6583526..86a83b741 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -107,9 +107,6 @@ roaring = "0.10.7" mopa-maintained = "0.2.3" utoipa = { version = "5.2.0", features = ["actix_extras", "macros", "non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } utoipa-scalar = { version = "0.2.0", features = ["actix-web"] } -utoipa-rapidoc = { version = "5.0.0", features = ["actix-web"] } -utoipa-redoc = { version = "5.0.0", features = ["actix-web"] } -doc-comment = "0.3.3" [dev-dependencies] actix-rt = "2.10.0" diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 684234be4..b3044b3dd 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -36,8 +36,6 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use tracing::debug; use utoipa::{OpenApi, ToSchema}; -use utoipa_rapidoc::RapiDoc; -use utoipa_redoc::{Redoc, Servable}; use utoipa_scalar::{Scalar, Servable as ScalarServable}; use self::api_key::KeyView; @@ -93,13 +91,8 @@ pub mod tasks; pub struct MeilisearchApi; pub fn configure(cfg: &mut web::ServiceConfig) { - let openapi = MeilisearchApi::openapi(); - cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::scope("/batches").configure(batches::configure)) - .service(Scalar::with_url("/scalar", openapi.clone())) - .service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) - .service(Redoc::with_url("/redoc", openapi)) .service(web::resource("/health").route(web::get().to(get_health))) .service(web::scope("/logs").configure(logs::configure)) .service(web::scope("/keys").configure(api_key::configure)) @@ -112,6 +105,12 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) .service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/experimental-features").configure(features::configure)); + + let now = std::time::Instant::now(); + let openapi = MeilisearchApi::openapi(); + println!("Took {:?} to generate the openapi file", now.elapsed()); + // #[cfg(feature = "webp")] + cfg.service(Scalar::with_url("/scalar", openapi.clone())); } pub fn get_task_id(req: &HttpRequest, opt: &Opt) -> Result, ResponseError> { From dd128656cbf7e137110601d9f2e8b31bf4431517 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 30 Dec 2024 15:02:12 +0100 Subject: [PATCH 18/27] fix all the tests --- .../after_registering_settings_task.snap | 2 +- .../after_registering_settings_task_vectors.snap | 2 +- crates/meilisearch-types/src/settings.rs | 11 +++++++++-- crates/meilisearch/tests/search/hybrid.rs | 2 +- crates/meilisearch/tests/settings/get_settings.rs | 8 ++++---- crates/meilisearch/tests/vector/binary_quantized.rs | 10 +++++----- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap index e18f80747..dcc2375e9 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap @@ -14,7 +14,7 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap index f2216a4a0..c89907f11 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap @@ -14,7 +14,7 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }, "B_small_hf": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }) }, "B_small_hf": SettingEmbeddingSettings { inner: Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet }) }}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index b90289413..8f8439d56 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -144,13 +144,20 @@ impl MergeWithError for DeserrJsonError)] pub inner: Setting, } +impl fmt::Debug for SettingEmbeddingSettings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + impl Deserr for SettingEmbeddingSettings { fn deserialize_from_value( value: deserr::Value, diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index 00a65d9aa..c8334858a 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -260,7 +260,7 @@ async fn distribution_shift() { snapshot!(code, @"202 Accepted"); let response = server.wait_task(response.uid()).await; - snapshot!(response["details"], @r###"{"embedders":{"default":{"distribution":{"mean":0.998,"sigma":0.01}}}}"###); + snapshot!(response["details"], @r#"{"embedders":{"default":{"distribution":{"mean":0.998,"sigma":0.01}}}}"#); let (response, code) = index.search_post(search).await; snapshot!(code, @"200 OK"); diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index 55d9441ee..eb7efabea 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -285,7 +285,7 @@ async fn secrets_are_hidden_in_settings() { let (response, code) = index.settings().await; meili_snap::snapshot!(code, @"200 OK"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response), @r#" { "displayedAttributes": [ "*" @@ -346,11 +346,11 @@ async fn secrets_are_hidden_in_settings() { "facetSearch": true, "prefixSearch": "indexingTime" } - "###); + "#); let (response, code) = server.get_task(settings_update_uid).await; meili_snap::snapshot!(code, @"200 OK"); - meili_snap::snapshot!(meili_snap::json_string!(response["details"]), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response["details"]), @r#" { "embedders": { "default": { @@ -363,7 +363,7 @@ async fn secrets_are_hidden_in_settings() { } } } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index 790df5459..266f84f7d 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -35,7 +35,7 @@ async fn retrieve_binary_quantize_status_in_the_settings() { let (settings, code) = index.settings().await; snapshot!(code, @"200 OK"); - snapshot!(settings["embedders"]["manual"], @r###"{"source":"userProvided","dimensions":3}"###); + snapshot!(settings["embedders"]["manual"], @r#"{"source":"userProvided","dimensions":3}"#); let (response, code) = index .update_settings(json!({ @@ -53,7 +53,7 @@ async fn retrieve_binary_quantize_status_in_the_settings() { let (settings, code) = index.settings().await; snapshot!(code, @"200 OK"); - snapshot!(settings["embedders"]["manual"], @r###"{"source":"userProvided","dimensions":3,"binaryQuantized":false}"###); + snapshot!(settings["embedders"]["manual"], @r#"{"source":"userProvided","dimensions":3,"binaryQuantized":false}"#); let (response, code) = index .update_settings(json!({ @@ -71,7 +71,7 @@ async fn retrieve_binary_quantize_status_in_the_settings() { let (settings, code) = index.settings().await; snapshot!(code, @"200 OK"); - snapshot!(settings["embedders"]["manual"], @r###"{"source":"userProvided","dimensions":3,"binaryQuantized":true}"###); + snapshot!(settings["embedders"]["manual"], @r#"{"source":"userProvided","dimensions":3,"binaryQuantized":true}"#); } #[actix_rt::test] @@ -300,7 +300,7 @@ async fn try_to_disable_binary_quantization() { .await; snapshot!(code, @"202 Accepted"); let ret = server.wait_task(response.uid()).await; - snapshot!(ret, @r###" + snapshot!(ret, @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -328,7 +328,7 @@ async fn try_to_disable_binary_quantization() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); } #[actix_rt::test] From 28162759a47b0aae17a351a95a880fec2248077f Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 6 Jan 2025 11:54:10 +0100 Subject: [PATCH 19/27] fix imports after rebase --- crates/meilisearch/src/routes/metrics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index bf5d6741c..daae66c2e 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -14,10 +14,8 @@ use utoipa::OpenApi; use time::OffsetDateTime; -use crate::extractors::authentication::policies::ActionPolicy; -use crate::extractors::authentication::{AuthenticationError, GuardedData}; -use crate::routes::create_all_stats; -use crate::search_queue::SearchQueue; +use index_scheduler::Query; +use meilisearch_types::tasks::Status; #[derive(OpenApi)] #[openapi(paths(get_metrics))] From 8b95c6ae5681ee91ed4a6ef6bed9b35ed4278e22 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 6 Jan 2025 11:54:39 +0100 Subject: [PATCH 20/27] improve the description of all the settings route --- crates/meilisearch/src/routes/indexes/settings.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 58197acda..10852bc1a 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -81,12 +81,13 @@ macro_rules! make_setting_route { #[allow(unused_imports)] use super::*; - #[doc = $camelcase_attr] #[utoipa::path( delete, path = concat!("{indexUid}/settings", $route), tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + summary = concat!("Reset ", $camelcase_attr), + description = concat!("Reset an index's ", $camelcase_attr, " to its default value"), params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = $type, responses( @@ -149,6 +150,8 @@ macro_rules! make_setting_route { path = concat!("{indexUid}/settings", $route), tag = "Settings", security(("Bearer" = ["settings.update", "settings.*", "*"])), + summary = concat!("Update ", $camelcase_attr), + description = concat!("Update an index's user defined ", $camelcase_attr), params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), request_body = $type, responses( @@ -232,6 +235,8 @@ macro_rules! make_setting_route { get, path = concat!("{indexUid}/settings", $route), tag = "Settings", + summary = concat!("Get ", $camelcase_attr), + description = concat!("Get an user defined ", $camelcase_attr), security(("Bearer" = ["settings.get", "settings.*", "*"])), params(("indexUid", example = "movies", description = "Index Unique Identifier", nullable = false)), responses( From ff49250c1a2435b2bdb6bcfdb4ed299d94a4c52b Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 6 Jan 2025 11:55:23 +0100 Subject: [PATCH 21/27] remove useless doc --- crates/meilisearch/src/routes/indexes/settings.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 10852bc1a..4307caea8 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -144,7 +144,6 @@ macro_rules! make_setting_route { } - #[doc = $camelcase_attr] #[utoipa::path( $update_verb, path = concat!("{indexUid}/settings", $route), @@ -230,7 +229,6 @@ macro_rules! make_setting_route { } - #[doc = $camelcase_attr] #[utoipa::path( get, path = concat!("{indexUid}/settings", $route), From e579554c84a82ea1220e769f6286d892bca30f40 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 6 Jan 2025 11:57:25 +0100 Subject: [PATCH 22/27] fmt --- crates/meilisearch/src/routes/api_key.rs | 7 ---- crates/meilisearch/src/routes/batches.rs | 1 - crates/meilisearch/src/routes/dump.rs | 1 - crates/meilisearch/src/routes/features.rs | 1 - crates/meilisearch/src/routes/logs.rs | 3 -- crates/meilisearch/src/routes/metrics.rs | 11 +++--- crates/meilisearch/src/routes/mod.rs | 36 +++++++++---------- crates/meilisearch/src/routes/multi_search.rs | 5 ++- crates/meilisearch/src/routes/snapshot.rs | 1 - crates/meilisearch/src/routes/tasks.rs | 2 -- 10 files changed, 22 insertions(+), 46 deletions(-) diff --git a/crates/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs index e45326069..9f832b0f2 100644 --- a/crates/meilisearch/src/routes/api_key.rs +++ b/crates/meilisearch/src/routes/api_key.rs @@ -31,7 +31,6 @@ use crate::routes::Pagination; You must have the master key or the default admin key to access the keys route. More information about the keys and their rights. Accessing any route under `/keys` without having set a master key will result in an error.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/keys"), - )), )] pub struct ApiKeyApi; @@ -50,7 +49,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } - /// Create an API Key /// /// Create an API Key. @@ -130,7 +128,6 @@ impl ListApiKeys { } } - /// Get API Keys /// /// List all API Keys @@ -201,7 +198,6 @@ pub async fn list_api_keys( Ok(HttpResponse::Ok().json(page_view)) } - /// Get an API Key /// /// Get an API key from its `uid` or its `key` field. @@ -266,7 +262,6 @@ pub async fn get_api_key( Ok(HttpResponse::Ok().json(res)) } - /// Update a Key /// /// Update the name and description of an API key. @@ -334,8 +329,6 @@ pub async fn patch_api_key( Ok(HttpResponse::Ok().json(res)) } - - /// Delete a key /// /// Delete the specified API key. diff --git a/crates/meilisearch/src/routes/batches.rs b/crates/meilisearch/src/routes/batches.rs index a5fc77a3d..7a801dae6 100644 --- a/crates/meilisearch/src/routes/batches.rs +++ b/crates/meilisearch/src/routes/batches.rs @@ -22,7 +22,6 @@ use crate::extractors::sequential_extractor::SeqHandler; name = "Batches", description = "The /batches route gives information about the progress of batches of asynchronous operations.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/batches"), - )), )] pub struct BatchesApi; diff --git a/crates/meilisearch/src/routes/dump.rs b/crates/meilisearch/src/routes/dump.rs index 37f06d4c6..8bcb167ee 100644 --- a/crates/meilisearch/src/routes/dump.rs +++ b/crates/meilisearch/src/routes/dump.rs @@ -28,7 +28,6 @@ all indexes contained in the indicated `.dump` file are imported along with thei Any existing index with the same uid as an index in the dump file will be overwritten. Dump imports are [performed at launch](https://www.meilisearch.com/docs/learn/advanced/dumps#importing-a-dump) using an option.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/dump"), - )), )] pub struct DumpApi; diff --git a/crates/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs index 506e73b2e..b7e85882f 100644 --- a/crates/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -24,7 +24,6 @@ use crate::extractors::sequential_extractor::SeqHandler; This route is **synchronous**. This means that no task object will be returned, and any activated or deactivated features will be made available or unavailable immediately.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/experimental_features"), - )), )] pub struct ExperimentalFeaturesApi; diff --git a/crates/meilisearch/src/routes/logs.rs b/crates/meilisearch/src/routes/logs.rs index dc6b6d14c..889ce824e 100644 --- a/crates/meilisearch/src/routes/logs.rs +++ b/crates/meilisearch/src/routes/logs.rs @@ -26,7 +26,6 @@ use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; use crate::{LogRouteHandle, LogStderrHandle}; - #[derive(OpenApi)] #[openapi( paths(get_logs, cancel_logs, update_stderr_target), @@ -35,7 +34,6 @@ use crate::{LogRouteHandle, LogStderrHandle}; description = "Everything about retrieving or customizing logs. Currently [experimental](https://www.meilisearch.com/docs/learn/experimental/overview).", external_docs(url = "https://www.meilisearch.com/docs/learn/experimental/log_customization"), - )), )] pub struct LogsApi; @@ -350,7 +348,6 @@ pub async fn get_logs( } } - /// Stop retrieving logs /// /// Call this route to make the engine stops sending logs through the `POST /logs/stream` route. diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index daae66c2e..192164288 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -5,17 +5,14 @@ use crate::search_queue::SearchQueue; use actix_web::http::header; use actix_web::web::{self, Data}; use actix_web::HttpResponse; -use index_scheduler::IndexScheduler; +use index_scheduler::{IndexScheduler, Query}; use meilisearch_auth::AuthController; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; -use prometheus::{Encoder, TextEncoder}; -use utoipa::OpenApi; - -use time::OffsetDateTime; - -use index_scheduler::Query; use meilisearch_types::tasks::Status; +use prometheus::{Encoder, TextEncoder}; +use time::OffsetDateTime; +use utoipa::OpenApi; #[derive(OpenApi)] #[openapi(paths(get_metrics))] diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index b3044b3dd..b0d6ac17f 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -1,22 +1,5 @@ use std::collections::BTreeMap; -use crate::extractors::authentication::policies::*; -use crate::extractors::authentication::GuardedData; -use crate::milli::progress::ProgressStepView; -use crate::milli::progress::ProgressView; -use crate::routes::batches::AllBatches; -use crate::routes::features::RuntimeTogglableFeatures; -use crate::routes::indexes::documents::DocumentDeletionByFilter; -use crate::routes::indexes::documents::DocumentEditionByFunction; -use crate::routes::indexes::IndexView; -use crate::routes::multi_search::SearchResults; -use crate::routes::swap_indexes::SwapIndexesPayload; -use crate::search::{ - FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, - SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, -}; -use crate::search_queue::SearchQueue; -use crate::Opt; use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; use index_scheduler::IndexScheduler; @@ -41,11 +24,24 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable}; use self::api_key::KeyView; use self::indexes::documents::BrowseQuery; use self::indexes::{IndexCreateRequest, IndexStats, UpdateIndexRequest}; -use self::logs::GetLogs; -use self::logs::LogMode; -use self::logs::UpdateStderrLogs; +use self::logs::{GetLogs, LogMode, UpdateStderrLogs}; use self::open_api_utils::OpenApiAuth; use self::tasks::AllTasks; +use crate::extractors::authentication::policies::*; +use crate::extractors::authentication::GuardedData; +use crate::milli::progress::{ProgressStepView, ProgressView}; +use crate::routes::batches::AllBatches; +use crate::routes::features::RuntimeTogglableFeatures; +use crate::routes::indexes::documents::{DocumentDeletionByFilter, DocumentEditionByFunction}; +use crate::routes::indexes::IndexView; +use crate::routes::multi_search::SearchResults; +use crate::routes::swap_indexes::SwapIndexesPayload; +use crate::search::{ + FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, + SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, +}; +use crate::search_queue::SearchQueue; +use crate::Opt; const PAGINATION_DEFAULT_LIMIT: usize = 20; const PAGINATION_DEFAULT_LIMIT_FN: fn() -> usize = || 20; diff --git a/crates/meilisearch/src/routes/multi_search.rs b/crates/meilisearch/src/routes/multi_search.rs index 711bdd03c..2d15d29bf 100644 --- a/crates/meilisearch/src/routes/multi_search.rs +++ b/crates/meilisearch/src/routes/multi_search.rs @@ -18,8 +18,8 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; use crate::routes::indexes::search::search_kind; use crate::search::{ - add_search_rules, perform_federated_search, perform_search, FederatedSearch, FederatedSearchResult, RetrieveVectors, - SearchQueryWithIndex, SearchResultWithIndex, + add_search_rules, perform_federated_search, perform_search, FederatedSearch, + FederatedSearchResult, RetrieveVectors, SearchQueryWithIndex, SearchResultWithIndex, }; use crate::search_queue::SearchQueue; @@ -30,7 +30,6 @@ use crate::search_queue::SearchQueue; name = "Multi-search", description = "The `/multi-search` route allows you to perform multiple search queries on one or more indexes by bundling them into a single HTTP request. Multi-search is also known as federated search.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/multi_search"), - )), )] pub struct MultiSearchApi; diff --git a/crates/meilisearch/src/routes/snapshot.rs b/crates/meilisearch/src/routes/snapshot.rs index b619d7411..b7bb116ed 100644 --- a/crates/meilisearch/src/routes/snapshot.rs +++ b/crates/meilisearch/src/routes/snapshot.rs @@ -24,7 +24,6 @@ During a snapshot export, all indexes of the current instance are exported—tog During a snapshot import, all indexes contained in the indicated .snapshot file are imported along with their associated documents and settings. Snapshot imports are performed at launch using an option.", external_docs(url = "https://www.meilisearch.com/docs/reference/api/snapshots"), - )), )] pub struct SnapshotApi; diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index 2f3871c1a..fce2bc8bf 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -33,7 +33,6 @@ use crate::{aggregate_methods, Opt}; name = "Tasks", description = "The tasks route gives information about the progress of the [asynchronous operations](https://docs.meilisearch.com/learn/advanced/asynchronous_operations.html).", external_docs(url = "https://www.meilisearch.com/docs/reference/api/tasks"), - )), )] pub struct TaskApi; @@ -496,7 +495,6 @@ pub struct AllTasks { next: Option, } - /// Get all tasks /// /// Get all [tasks](https://docs.meilisearch.com/learn/advanced/asynchronous_operations.html) From 21026f0ca83bc783ba6ad14db9df953827203ec7 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 6 Jan 2025 12:09:07 +0100 Subject: [PATCH 23/27] move the swagger behind a feature flag --- crates/meilisearch/Cargo.toml | 3 ++- crates/meilisearch/src/routes/mod.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 86a83b741..fb058c4cb 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -106,7 +106,7 @@ build-info = { version = "1.7.0", path = "../build-info" } roaring = "0.10.7" mopa-maintained = "0.2.3" utoipa = { version = "5.2.0", features = ["actix_extras", "macros", "non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] } -utoipa-scalar = { version = "0.2.0", features = ["actix-web"] } +utoipa-scalar = { version = "0.2.0", optional = true, features = ["actix-web"] } [dev-dependencies] actix-rt = "2.10.0" @@ -135,6 +135,7 @@ zip = { version = "2.1.3", optional = true } [features] default = ["meilisearch-types/all-tokenizations", "mini-dashboard"] +swagger = ["utoipa-scalar"] mini-dashboard = [ "static-files", "anyhow", diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index b0d6ac17f..131986712 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -19,7 +19,6 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use tracing::debug; use utoipa::{OpenApi, ToSchema}; -use utoipa_scalar::{Scalar, Servable as ScalarServable}; use self::api_key::KeyView; use self::indexes::documents::BrowseQuery; @@ -102,11 +101,12 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/experimental-features").configure(features::configure)); - let now = std::time::Instant::now(); - let openapi = MeilisearchApi::openapi(); - println!("Took {:?} to generate the openapi file", now.elapsed()); - // #[cfg(feature = "webp")] - cfg.service(Scalar::with_url("/scalar", openapi.clone())); + #[cfg(feature = "swagger")] + { + use utoipa_scalar::{Scalar, Servable as ScalarServable}; + let openapi = MeilisearchApi::openapi(); + cfg.service(Scalar::with_url("/scalar", openapi.clone())); + } } pub fn get_task_id(req: &HttpRequest, opt: &Opt) -> Result, ResponseError> { From 8ebfc9fa925a4a32a3d5e3dbf9b5fc3b73214811 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jan 2025 16:15:01 +0100 Subject: [PATCH 24/27] Update crates/meilisearch-types/src/settings.rs Co-authored-by: Louis Dureuil --- crates/meilisearch-types/src/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 8f8439d56..2ecef9383 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -259,7 +259,7 @@ pub struct Settings { #[schema(value_type = Option, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))] pub pagination: Setting, - /// Embedder required for performing meaning-based search queries. + /// Embedder required for performing semantic search queries. #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default, error = DeserrJsonError)] #[schema(value_type = Option>)] From ae5a04e85ce7daa0f0e7569d3c039c7681a60ab6 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jan 2025 16:24:26 +0100 Subject: [PATCH 25/27] apply review comments --- crates/meilisearch-types/src/tasks.rs | 4 ---- crates/meilisearch/src/search/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 7960951ed..167cfcd80 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -471,10 +471,6 @@ pub enum Kind { } impl Kind { - pub fn all_variants() -> Vec { - enum_iterator::all::().collect() - } - pub fn related_to_one_index(&self) -> bool { match self { Kind::DocumentAdditionOrUpdate diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 7cefb57b6..abeae55bd 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -673,7 +673,7 @@ pub struct SearchResult { #[serde(flatten)] pub hits_info: HitsInfo, #[serde(skip_serializing_if = "Option::is_none")] - #[schema(value_type = HashMap)] + #[schema(value_type = Option>)] pub facet_distribution: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub facet_stats: Option>, @@ -1043,7 +1043,7 @@ pub fn perform_search( #[derive(Debug, Clone, Default, Serialize, ToSchema)] pub struct ComputedFacets { - #[schema(value_type = Option>>)] + #[schema(value_type = BTreeMap>)] pub distribution: BTreeMap>, pub stats: BTreeMap, } From a8ef6f08e01000f2a4455cedf94dfd75b1d42915 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jan 2025 16:31:39 +0100 Subject: [PATCH 26/27] Update crates/meilisearch-types/src/settings.rs Co-authored-by: Louis Dureuil --- crates/meilisearch-types/src/settings.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 2ecef9383..658d7eec4 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -147,6 +147,13 @@ impl MergeWithError for DeserrJsonError)] pub inner: Setting, From 99f5e09a792d08cb15770f908c8cfc047257c88b Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jan 2025 16:42:37 +0100 Subject: [PATCH 27/27] fix the tests --- .../after_registering_settings_task.snap | 7 ----- ...ter_registering_settings_task_vectors.snap | 7 ----- crates/index-scheduler/src/scheduler/test.rs | 4 ++- .../src/scheduler/test_embedders.rs | 30 +++++++++++-------- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap index dcc2375e9..92e37550a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap @@ -1,13 +1,6 @@ --- -<<<<<<< HEAD:crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap source: crates/index-scheduler/src/scheduler/test.rs snapshot_kind: text -||||||| parent of 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap -source: crates/index-scheduler/src/lib.rs -======= -source: crates/index-scheduler/src/lib.rs -snapshot_kind: text ->>>>>>> 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap --- ### Autobatching Enabled = true ### Processing batch None: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap index c89907f11..33bd5c0d2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap @@ -1,13 +1,6 @@ --- -<<<<<<< HEAD:crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap source: crates/index-scheduler/src/scheduler/test_embedders.rs snapshot_kind: text -||||||| parent of 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap -source: crates/index-scheduler/src/lib.rs -======= -source: crates/index-scheduler/src/lib.rs -snapshot_kind: text ->>>>>>> 2e258cec7 (fix some tests):crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap --- ### Autobatching Enabled = true ### Processing batch None: diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index a2276107d..b705d3c33 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -5,6 +5,7 @@ use meili_snap::{json_string, snapshot}; use meilisearch_types::milli::index::IndexEmbeddingConfig; use meilisearch_types::milli::update::IndexDocumentsMethod::*; use meilisearch_types::milli::{self}; +use meilisearch_types::settings::SettingEmbeddingSettings; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; use roaring::RoaringBitmap; @@ -647,7 +648,8 @@ fn test_settings_update() { response: Setting::Set(serde_json::json!("{{embedding}}")), ..Default::default() }; - embedders.insert(S("default"), Setting::Set(embedding_settings)); + embedders + .insert(S("default"), SettingEmbeddingSettings { inner: Setting::Set(embedding_settings) }); new_settings.embedders = Setting::Set(embedders); index_scheduler diff --git a/crates/index-scheduler/src/scheduler/test_embedders.rs b/crates/index-scheduler/src/scheduler/test_embedders.rs index d21dc7548..5ec58bc53 100644 --- a/crates/index-scheduler/src/scheduler/test_embedders.rs +++ b/crates/index-scheduler/src/scheduler/test_embedders.rs @@ -7,7 +7,7 @@ use meilisearch_types::milli::index::IndexEmbeddingConfig; use meilisearch_types::milli::update::Setting; use meilisearch_types::milli::vector::settings::EmbeddingSettings; use meilisearch_types::milli::{self, obkv_to_json}; -use meilisearch_types::settings::{Settings, Unchecked}; +use meilisearch_types::settings::{SettingEmbeddingSettings, Settings, Unchecked}; use meilisearch_types::tasks::KindWithContent; use milli::update::IndexDocumentsMethod::*; @@ -30,7 +30,10 @@ fn import_vectors() { response: Setting::Set(serde_json::json!("{{embedding}}")), ..Default::default() }; - embedders.insert(S("A_fakerest"), Setting::Set(embedding_settings)); + embedders.insert( + S("A_fakerest"), + SettingEmbeddingSettings { inner: Setting::Set(embedding_settings) }, + ); let embedding_settings = milli::vector::settings::EmbeddingSettings { source: Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace), @@ -39,7 +42,10 @@ fn import_vectors() { document_template: Setting::Set(S("{{doc.doggo}} the {{doc.breed}} best doggo")), ..Default::default() }; - embedders.insert(S("B_small_hf"), Setting::Set(embedding_settings)); + embedders.insert( + S("B_small_hf"), + SettingEmbeddingSettings { inner: Setting::Set(embedding_settings) }, + ); new_settings.embedders = Setting::Set(embedders); @@ -356,13 +362,13 @@ fn import_vectors_first_and_embedder_later() { let setting = meilisearch_types::settings::Settings:: { embedders: Setting::Set(maplit::btreemap! { - S("my_doggo_embedder") => Setting::Set(EmbeddingSettings { + S("my_doggo_embedder") => SettingEmbeddingSettings { inner: Setting::Set(EmbeddingSettings { source: Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace), model: Setting::Set(S("sentence-transformers/all-MiniLM-L6-v2")), revision: Setting::Set(S("e4ce9877abf3edfe10b0d82785e83bdcb973e22e")), document_template: Setting::Set(S("{{doc.doggo}}")), ..Default::default() - }) + }) } }), ..Default::default() }; @@ -511,11 +517,11 @@ fn delete_document_containing_vector() { let setting = meilisearch_types::settings::Settings:: { embedders: Setting::Set(maplit::btreemap! { - S("manual") => Setting::Set(EmbeddingSettings { + S("manual") => SettingEmbeddingSettings { inner: Setting::Set(EmbeddingSettings { source: Setting::Set(milli::vector::settings::EmbedderSource::UserProvided), dimensions: Setting::Set(3), ..Default::default() - }) + }) } }), ..Default::default() }; @@ -677,18 +683,18 @@ fn delete_embedder_with_user_provided_vectors() { let setting = meilisearch_types::settings::Settings:: { embedders: Setting::Set(maplit::btreemap! { - S("manual") => Setting::Set(EmbeddingSettings { + S("manual") => SettingEmbeddingSettings { inner: Setting::Set(EmbeddingSettings { source: Setting::Set(milli::vector::settings::EmbedderSource::UserProvided), dimensions: Setting::Set(3), ..Default::default() - }), - S("my_doggo_embedder") => Setting::Set(EmbeddingSettings { + }) }, + S("my_doggo_embedder") => SettingEmbeddingSettings { inner: Setting::Set(EmbeddingSettings { source: Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace), model: Setting::Set(S("sentence-transformers/all-MiniLM-L6-v2")), revision: Setting::Set(S("e4ce9877abf3edfe10b0d82785e83bdcb973e22e")), document_template: Setting::Set(S("{{doc.doggo}}")), ..Default::default() - }), + }) }, }), ..Default::default() }; @@ -764,7 +770,7 @@ fn delete_embedder_with_user_provided_vectors() { { let setting = meilisearch_types::settings::Settings:: { embedders: Setting::Set(maplit::btreemap! { - S("manual") => Setting::Reset, + S("manual") => SettingEmbeddingSettings { inner: Setting::Reset }, }), ..Default::default() };