From ddad6cc0691de278becdc04b47033d2267dd3c44 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 25 May 2022 11:51:26 +0200 Subject: [PATCH] feat(http): update the documents resource - Return Documents API resources on `/documents` in an array in the the results field. - Add limit, offset and total in the response body. - Rename `attributesToRetrieve` into `fields` (only for the `/documents` endpoints, not for the `/search` ones). - The `displayedAttributes` settings does not impact anymore the displayed fields returned in the `/documents` endpoints. These settings only impacts the `/search` endpoint. Fix #2372 --- .../src/routes/indexes/documents.rs | 59 +++--- meilisearch-http/src/routes/mod.rs | 34 ++++ meilisearch-http/src/routes/tasks.rs | 37 +--- meilisearch-http/tests/common/index.rs | 16 +- .../tests/documents/add_documents.rs | 4 +- .../tests/documents/delete_documents.rs | 10 +- .../tests/documents/get_documents.rs | 179 ++++++++++-------- meilisearch-http/tests/dumps/mod.rs | 36 ++-- meilisearch-lib/src/index/index.rs | 29 ++- meilisearch-lib/src/index/mod.rs | 6 +- meilisearch-lib/src/index_controller/mod.rs | 7 +- 11 files changed, 217 insertions(+), 200 deletions(-) diff --git a/meilisearch-http/src/routes/indexes/documents.rs b/meilisearch-http/src/routes/indexes/documents.rs index 66551ec77..4c87044db 100644 --- a/meilisearch-http/src/routes/indexes/documents.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -13,7 +13,8 @@ use meilisearch_lib::MeiliSearch; use mime::Mime; use once_cell::sync::Lazy; use serde::Deserialize; -use serde_json::Value; +use serde_cs::vec::CS; +use serde_json::{json, Value}; use tokio::sync::mpsc; use crate::analytics::Analytics; @@ -21,11 +22,9 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::{fold_star_or, StarOr}; use crate::task::SummarizedTaskView; -const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; -const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; - static ACCEPTED_CONTENT_TYPE: Lazy> = Lazy::new(|| { vec![ "application/json".to_string(), @@ -86,14 +85,24 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetDocument { + fields: Option>>, +} + pub async fn get_document( meilisearch: GuardedData, MeiliSearch>, path: web::Path, + params: web::Query, ) -> Result { let index = path.index_uid.clone(); let id = path.document_id.clone(); + let GetDocument { fields } = params.into_inner(); + let attributes_to_retrieve = fields.map(CS::into_inner).and_then(fold_star_or); + let document = meilisearch - .document(index, id, None as Option>) + .document(index, id, attributes_to_retrieve) .await?; debug!("returns: {:?}", document); Ok(HttpResponse::Ok().json(document)) @@ -113,12 +122,16 @@ pub async fn delete_document( Ok(HttpResponse::Accepted().json(task)) } +const PAGINATION_DEFAULT_LIMIT: fn() -> usize = || 20; + #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct BrowseQuery { - offset: Option, - limit: Option, - attributes_to_retrieve: Option, + #[serde(default)] + offset: usize, + #[serde(default = "PAGINATION_DEFAULT_LIMIT")] + limit: usize, + fields: Option>>, } pub async fn get_all_documents( @@ -127,27 +140,21 @@ pub async fn get_all_documents( params: web::Query, ) -> Result { debug!("called with params: {:?}", params); - let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| { - let mut names = Vec::new(); - for name in attrs.split(',').map(String::from) { - if name == "*" { - return None; - } - names.push(name); - } - Some(names) - }); + let BrowseQuery { + offset, + limit, + fields, + } = params.into_inner(); + let attributes_to_retrieve = fields.map(CS::into_inner).and_then(fold_star_or); - let documents = meilisearch - .documents( - path.into_inner(), - params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), - params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), - attributes_to_retrieve, - ) + let (total, documents) = meilisearch + .documents(path.into_inner(), offset, limit, attributes_to_retrieve) .await?; + debug!("returns: {:?}", documents); - Ok(HttpResponse::Ok().json(documents)) + Ok(HttpResponse::Ok().json(json!( + { "limit": limit, "offset": offset, "total": total, "results": documents } + ))) } #[derive(Deserialize, Debug)] diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 49397444f..a34b7578d 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use actix_web::{web, HttpResponse}; use log::debug; use serde::{Deserialize, Serialize}; @@ -24,6 +26,38 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/indexes").configure(indexes::configure)); } +/// A type that tries to match either a star (*) or +/// any other thing that implements `FromStr`. +#[derive(Debug)] +pub enum StarOr { + Star, + Other(T), +} + +impl FromStr for StarOr { + type Err = T::Err; + + fn from_str(s: &str) -> Result { + if s.trim() == "*" { + Ok(StarOr::Star) + } else { + T::from_str(s).map(StarOr::Other) + } + } +} + +/// Extracts the raw values from the `StarOr` types and +/// return None if a `StarOr::Star` is encountered. +pub fn fold_star_or(content: impl IntoIterator>) -> Option> { + content + .into_iter() + .map(|value| match value { + StarOr::Star => None, + StarOr::Other(val) => Some(val), + }) + .collect() +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(clippy::large_enum_variant)] #[serde(tag = "name")] diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 1fe903abf..096e44f8d 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -6,13 +6,14 @@ use meilisearch_lib::{IndexUid, MeiliSearch}; use serde::Deserialize; use serde_cs::vec::CS; use serde_json::json; -use std::str::FromStr; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; use crate::task::{TaskListView, TaskStatus, TaskType, TaskView}; +use super::{fold_star_or, StarOr}; + pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks)))) .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task)))); @@ -27,40 +28,6 @@ pub struct TaskFilterQuery { index_uid: Option>>, } -/// A type that tries to match either a star (*) or -/// any other thing that implements `FromStr`. -#[derive(Debug)] -enum StarOr { - Star, - Other(T), -} - -impl FromStr for StarOr { - type Err = T::Err; - - fn from_str(s: &str) -> Result { - if s.trim() == "*" { - Ok(StarOr::Star) - } else { - T::from_str(s).map(StarOr::Other) - } - } -} - -/// Extracts the raw values from the `StarOr` types and -/// return None if a `StarOr::Star` is encountered. -fn fold_star_or(content: Vec>) -> Option> { - content - .into_iter() - .fold(Some(Vec::new()), |acc, val| match (acc, val) { - (None, _) | (_, StarOr::Star) => None, - (Some(mut acc), StarOr::Other(uid)) => { - acc.push(uid); - Some(acc) - } - }) -} - #[rustfmt::skip] fn task_type_matches_content(type_: &TaskType, content: &TaskContent) -> bool { matches!((type_, content), diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index bdce22db2..e21dbcb67 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -145,9 +145,12 @@ impl Index<'_> { pub async fn get_document( &self, id: u64, - _options: Option, + options: Option, ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + if let Some(fields) = options.and_then(|o| o.fields) { + url.push_str(&format!("?fields={}", fields.join(","))); + } self.service.get(url).await } @@ -162,10 +165,7 @@ impl Index<'_> { } if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { - url.push_str(&format!( - "attributesToRetrieve={}&", - attributes_to_retrieve.join(",") - )); + url.push_str(&format!("fields={}&", attributes_to_retrieve.join(","))); } self.service.get(url).await @@ -245,7 +245,9 @@ impl Index<'_> { make_settings_test_routes!(distinct_attribute); } -pub struct GetDocumentOptions; +pub struct GetDocumentOptions { + pub fields: Option>, +} #[derive(Debug, Default)] pub struct GetAllDocumentsOptions { diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index ab271ce18..8ef8c54fd 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -828,7 +828,7 @@ async fn add_larger_dataset() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 77); + assert_eq!(response["results"].as_array().unwrap().len(), 77); } #[actix_rt::test] @@ -849,7 +849,7 @@ async fn update_larger_dataset() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 77); + assert_eq!(response["results"].as_array().unwrap().len(), 77); } #[actix_rt::test] diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index 5198b2bfb..8c7ddaa7b 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -72,7 +72,7 @@ async fn clear_all_documents() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -89,7 +89,7 @@ async fn clear_all_documents_empty_index() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -125,8 +125,8 @@ async fn delete_batch() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!(response.as_array().unwrap()[0]["id"], 3); + assert_eq!(response["results"].as_array().unwrap().len(), 1); + assert_eq!(response["results"][0]["id"], json!(3)); } #[actix_rt::test] @@ -143,5 +143,5 @@ async fn delete_no_document_batch() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 3); + assert_eq!(response["results"].as_array().unwrap().len(), 3); } diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 6c93b9c13..cad656088 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,5 +1,5 @@ -use crate::common::GetAllDocumentsOptions; use crate::common::Server; +use crate::common::{GetAllDocumentsOptions, GetDocumentOptions}; use serde_json::json; @@ -39,7 +39,7 @@ async fn get_document() { let documents = serde_json::json!([ { "id": 0, - "content": "foobar", + "nested": { "content": "foobar" }, } ]); let (_, code) = index.add_documents(documents, None).await; @@ -49,11 +49,45 @@ async fn get_document() { assert_eq!(code, 200); assert_eq!( response, - serde_json::json!( { + serde_json::json!({ "id": 0, - "content": "foobar", + "nested": { "content": "foobar" }, }) ); + + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["id"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + serde_json::json!({ + "id": 0, + }) + ); + + /* This currently doesn't work but should be fixed by #2433 + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["nested.content"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + serde_json::json!({ + "nested": { "content": "foobar" }, + }) + ); + */ } #[actix_rt::test] @@ -88,7 +122,7 @@ async fn get_no_document() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -101,7 +135,7 @@ async fn get_all_documents_no_options() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - let arr = response.as_array().unwrap(); + let arr = response["results"].as_array().unwrap(); assert_eq!(arr.len(), 20); let first = serde_json::json!({ "id":0, @@ -137,8 +171,11 @@ async fn test_get_all_documents_limit() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 5); - assert_eq!(response.as_array().unwrap()[0]["id"], 0); + assert_eq!(response["results"].as_array().unwrap().len(), 5); + assert_eq!(response["results"][0]["id"], json!(0)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(5)); + assert_eq!(response["total"], json!(77)); } #[actix_rt::test] @@ -154,8 +191,11 @@ async fn test_get_all_documents_offset() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0]["id"], 5); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + assert_eq!(response["results"][0]["id"], json!(5)); + assert_eq!(response["offset"], json!(5)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); } #[actix_rt::test] @@ -171,20 +211,14 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 1 - ); - assert!(response.as_array().unwrap()[0] - .as_object() - .unwrap() - .get("name") - .is_some()); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 1); + assert!(results["name"] != json!(null)); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -193,15 +227,13 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 0 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 0); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -210,15 +242,13 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 0 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 0); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -227,15 +257,12 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 2 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 2); + assert!(results["name"] != json!(null)); + assert!(results["tags"] != json!(null)); + } let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -244,15 +271,10 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 16 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 16); + } let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -261,19 +283,14 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 16 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 16); + } } #[actix_rt::test] -async fn get_documents_displayed_attributes() { +async fn get_documents_displayed_attributes_is_ignored() { let server = Server::new().await; let index = server.index("test"); index @@ -285,23 +302,19 @@ async fn get_documents_displayed_attributes() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response["results"].as_array().unwrap().len(), 20); assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 1 + response["results"][0].as_object().unwrap().keys().count(), + 16 ); - assert!(response.as_array().unwrap()[0] - .as_object() - .unwrap() - .get("gender") - .is_some()); + assert!(response["results"][0]["gender"] != json!(null)); + + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); - assert_eq!(response.as_object().unwrap().keys().count(), 1); + assert_eq!(response.as_object().unwrap().keys().count(), 16); assert!(response.as_object().unwrap().get("gender").is_some()); } diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index 6d6e6494a..c24fbe19d 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -142,21 +142,21 @@ async fn import_dump_v2_movie_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) ); let (document, code) = index.get_document(500, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) ); let (document, code) = index.get_document(10006, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) ); } @@ -211,21 +211,21 @@ async fn import_dump_v2_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({"name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"}) + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) ); let (document, code) = index.get_document(191940, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"}) + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) ); let (document, code) = index.get_document(159227, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"}) + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) ); } @@ -341,21 +341,21 @@ async fn import_dump_v3_movie_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) ); let (document, code) = index.get_document(500, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) ); let (document, code) = index.get_document(10006, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) ); } @@ -410,21 +410,21 @@ async fn import_dump_v3_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({"name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"}) + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) ); let (document, code) = index.get_document(191940, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"}) + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) ); let (document, code) = index.get_document(159227, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({"name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"}) + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) ); } @@ -540,21 +540,21 @@ async fn import_dump_v4_movie_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) ); let (document, code) = index.get_document(500, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({ "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) ); let (document, code) = index.get_document(10006, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({ "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) ); } @@ -609,20 +609,20 @@ async fn import_dump_v4_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( document, - json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"}) + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) ); let (document, code) = index.get_document(191940, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"}) + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) ); let (document, code) = index.get_document(159227, None).await; assert_eq!(code, 200); assert_eq!( document, - json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"}) + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) ); } diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index f5122c8c1..bcf94bb0c 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::BTreeSet; use std::fs::create_dir_all; use std::marker::PhantomData; use std::ops::Deref; @@ -218,17 +218,17 @@ impl Index { }) } + /// Return the total number of documents contained in the index + the selected documents. pub fn retrieve_documents>( &self, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result>> { + ) -> Result<(u64, Vec)> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = self.fields_to_display(&attributes_to_retrieve, &fields_ids_map)?; let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); @@ -240,20 +240,20 @@ impl Index { documents.push(object); } - Ok(documents) + let number_of_documents = self.number_of_documents(&txn)?; + + Ok((number_of_documents, documents)) } pub fn retrieve_document>( &self, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let fields_to_display = self.fields_to_display(&attributes_to_retrieve, &fields_ids_map)?; let internal_id = self .external_documents_ids(&txn)? @@ -278,25 +278,18 @@ impl Index { fn fields_to_display>( &self, - txn: &milli::heed::RoTxn, attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, ) -> Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? { - Some(ids) => ids.into_iter().collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - let attributes_to_retrieve_ids = match attributes_to_retrieve { Some(attrs) => attrs .iter() .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), + .collect(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; - displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); - Ok(displayed_fields_ids) + Ok(attributes_to_retrieve_ids) } pub fn snapshot(&self, path: impl AsRef) -> Result<()> { diff --git a/meilisearch-lib/src/index/mod.rs b/meilisearch-lib/src/index/mod.rs index 3a42b2617..b46d97849 100644 --- a/meilisearch-lib/src/index/mod.rs +++ b/meilisearch-lib/src/index/mod.rs @@ -32,11 +32,11 @@ pub mod test { use milli::update::IndexerConfig; use milli::update::{DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsMethod}; use nelson::Mocker; - use serde_json::{Map, Value}; use uuid::Uuid; use super::error::Result; use super::index::Index; + use super::Document; use super::{Checked, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings}; use crate::update_file_store::UpdateFileStore; @@ -102,7 +102,7 @@ pub mod test { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result>> { + ) -> Result<(u64, Vec)> { match self { MockIndex::Real(index) => { index.retrieve_documents(offset, limit, attributes_to_retrieve) @@ -115,7 +115,7 @@ pub mod test { &self, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result { match self { MockIndex::Real(index) => index.retrieve_document(doc_id, attributes_to_retrieve), MockIndex::Mock(_) => todo!(), diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 7ec159684..1381a251f 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -524,18 +524,19 @@ where Ok(settings) } + /// Return the total number of documents contained in the index + the selected documents. pub async fn documents( &self, uid: String, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result<(u64, Vec)> { let index = self.index_resolver.get_index(uid).await?; - let documents = + let result = spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)) .await??; - Ok(documents) + Ok(result) } pub async fn document(