From 1e3dcbea3f32208d97f61fd33c40638ad73d6f13 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Thu, 9 Jun 2022 10:17:55 +0200 Subject: [PATCH] Plug the pagination.limited_to setting --- .../src/routes/indexes/settings.rs | 22 ++++++++++++++ meilisearch-http/tests/dumps/mod.rs | 18 ++++++------ .../tests/settings/get_settings.rs | 10 +++++-- meilisearch-lib/src/index/index.rs | 11 +++++-- meilisearch-lib/src/index/search.rs | 10 +++++-- meilisearch-lib/src/index/updates.rs | 29 +++++++++++++++++++ 6 files changed, 84 insertions(+), 16 deletions(-) diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 5620f9768..2d0b47121 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -304,6 +304,28 @@ make_setting_route!( } ); +make_setting_route!( + "/pagination", + patch, + meilisearch_lib::index::updates::PaginationSettings, + pagination, + "pagination", + analytics, + |setting: &Option, req: &HttpRequest| { + use serde_json::json; + + analytics.publish( + "Pagination Updated".to_string(), + json!({ + "pagination": { + "limited_to": setting.as_ref().and_then(|s| s.limited_to.set()), + }, + }), + Some(req), + ); + } +); + macro_rules! generate_configure { ($($mod:ident),*) => { pub fn configure(cfg: &mut web::ServiceConfig) { diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index 8f1c88bd9..75562a5a2 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -61,7 +61,7 @@ async fn import_dump_v2_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -125,7 +125,7 @@ async fn import_dump_v2_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -189,7 +189,7 @@ async fn import_dump_v2_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }}) + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 }}) ); let (tasks, code) = index.list_tasks().await; @@ -253,7 +253,7 @@ async fn import_dump_v3_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -317,7 +317,7 @@ async fn import_dump_v3_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -381,7 +381,7 @@ async fn import_dump_v3_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -445,7 +445,7 @@ async fn import_dump_v4_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -509,7 +509,7 @@ async fn import_dump_v4_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -573,7 +573,7 @@ async fn import_dump_v4_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 } }) + json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) ); let (tasks, code) = index.list_tasks().await; diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 94cc674a6..1cc60d652 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -49,7 +49,7 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 10); + assert_eq!(settings.keys().len(), 11); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["filterableAttributes"], json!([])); @@ -70,7 +70,13 @@ async fn get_settings() { assert_eq!( settings["faceting"], json!({ - "maxValuesPerFacet": 100 + "maxValuesPerFacet": 100, + }) + ); + assert_eq!( + settings["pagination"], + json!({ + "limitedTo": 1000, }) ); } diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index 84b81a0ac..d4772b73b 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -14,11 +14,12 @@ use serde_json::{Map, Value}; use time::OffsetDateTime; use uuid::Uuid; +use crate::index::search::DEFAULT_PAGINATION_LIMITED_TO; use crate::EnvSizer; use super::error::IndexError; use super::error::Result; -use super::updates::{FacetingSettings, MinWordSizeTyposSetting, TypoSettings}; +use super::updates::{FacetingSettings, MinWordSizeTyposSetting, PaginationSettings, TypoSettings}; use super::{Checked, Settings}; pub type Document = Map; @@ -200,7 +201,12 @@ impl Index { ), }; - dbg!(&faceting); + let pagination = PaginationSettings { + limited_to: Setting::Set( + self.pagination_limited_to(txn)? + .unwrap_or(DEFAULT_PAGINATION_LIMITED_TO), + ), + }; Ok(Settings { displayed_attributes: match displayed_attributes { @@ -222,6 +228,7 @@ impl Index { synonyms: Setting::Set(synonyms), typo_tolerance: Setting::Set(typo_tolerance), faceting: Setting::Set(faceting), + pagination: Setting::Set(pagination), _kind: PhantomData, }) } diff --git a/meilisearch-lib/src/index/search.rs b/meilisearch-lib/src/index/search.rs index 25d619457..781a5bb66 100644 --- a/meilisearch-lib/src/index/search.rs +++ b/meilisearch-lib/src/index/search.rs @@ -29,7 +29,7 @@ pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "".to_string(); /// The maximimum number of results that the engine /// will be able to return in one search call. -pub const HARD_RESULT_LIMIT: usize = 1000; +pub const DEFAULT_PAGINATION_LIMITED_TO: usize = 1000; #[derive(Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -91,10 +91,14 @@ impl Index { search.query(query); } + let pagination_limited_to = self + .pagination_limited_to(&rtxn)? + .unwrap_or(DEFAULT_PAGINATION_LIMITED_TO); + // Make sure that a user can't get more documents than the hard limit, // we align that on the offset too. - let offset = min(query.offset.unwrap_or(0), HARD_RESULT_LIMIT); - let limit = min(query.limit, HARD_RESULT_LIMIT.saturating_sub(offset)); + let offset = min(query.offset.unwrap_or(0), pagination_limited_to); + let limit = min(query.limit, pagination_limited_to.saturating_sub(offset)); search.offset(offset); search.limit(limit); diff --git a/meilisearch-lib/src/index/updates.rs b/meilisearch-lib/src/index/updates.rs index ab7d08bb5..95edbbf9d 100644 --- a/meilisearch-lib/src/index/updates.rs +++ b/meilisearch-lib/src/index/updates.rs @@ -79,6 +79,16 @@ pub struct FacetingSettings { pub max_values_per_facet: Setting, } +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct PaginationSettings { + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + pub limited_to: Setting, +} + /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings /// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a /// call to `check` will return a `Settings` from a `Settings`. @@ -128,6 +138,9 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] pub faceting: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + pub pagination: Setting, #[serde(skip)] pub _kind: PhantomData, @@ -146,6 +159,7 @@ impl Settings { distinct_attribute: Setting::Reset, typo_tolerance: Setting::Reset, faceting: Setting::Reset, + pagination: Setting::Reset, _kind: PhantomData, } } @@ -162,6 +176,7 @@ impl Settings { distinct_attribute, typo_tolerance, faceting, + pagination, .. } = self; @@ -176,6 +191,7 @@ impl Settings { distinct_attribute, typo_tolerance, faceting, + pagination, _kind: PhantomData, } } @@ -216,6 +232,7 @@ impl Settings { distinct_attribute: self.distinct_attribute, typo_tolerance: self.typo_tolerance, faceting: self.faceting, + pagination: self.pagination, _kind: PhantomData, } } @@ -455,6 +472,16 @@ pub fn apply_settings_to_builder( Setting::Reset => builder.reset_max_values_per_facet(), Setting::NotSet => (), } + + match settings.pagination { + Setting::Set(ref value) => match value.limited_to { + Setting::Set(val) => builder.set_pagination_limited_to(val), + Setting::Reset => builder.reset_pagination_limited_to(), + Setting::NotSet => (), + }, + Setting::Reset => builder.reset_pagination_limited_to(), + Setting::NotSet => (), + } } #[cfg(test)] @@ -485,6 +512,7 @@ pub(crate) mod test { distinct_attribute: Setting::NotSet, typo_tolerance: Setting::NotSet, faceting: Setting::NotSet, + pagination: Setting::NotSet, _kind: PhantomData::, }; @@ -508,6 +536,7 @@ pub(crate) mod test { distinct_attribute: Setting::NotSet, typo_tolerance: Setting::NotSet, faceting: Setting::NotSet, + pagination: Setting::NotSet, _kind: PhantomData::, };