Merge pull request #520 from meilisearch/fix-http-issues

Fix http issues
This commit is contained in:
Clément Renault 2020-03-11 15:21:50 +01:00 committed by GitHub
commit cf6e481c14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 266 additions and 60 deletions

View File

@ -118,9 +118,9 @@ fn find_primary_key(document: &IndexMap<String, Value>) -> Option<String> {
} }
#[derive(Default, Deserialize)] #[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateDocumentsQuery { struct UpdateDocumentsQuery {
document_id: Option<String>, primary_key: Option<String>,
} }
async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) -> SResult<Response> { async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) -> SResult<Response> {
@ -141,11 +141,11 @@ async fn update_multiple_documents(mut ctx: Request<Data>, is_partial: bool) ->
.ok_or(ResponseError::internal("schema not found"))?; .ok_or(ResponseError::internal("schema not found"))?;
if schema.primary_key().is_none() { if schema.primary_key().is_none() {
let id = match query.document_id { let id = match query.primary_key {
Some(id) => id, Some(id) => id,
None => match data.first().and_then(|docs| find_primary_key(docs)) { None => match data.first().and_then(|docs| find_primary_key(docs)) {
Some(id) => id, Some(id) => id,
None => return Err(ResponseError::bad_request("Could not infer a schema")), None => return Err(ResponseError::bad_request("Could not infer a primary key")),
}, },
}; };

View File

@ -1,5 +1,4 @@
use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES}; use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES};
use serde::Deserialize;
use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::collections::{BTreeMap, BTreeSet, HashSet};
use tide::{Request, Response}; use tide::{Request, Response};
@ -73,36 +72,13 @@ pub async fn get_all(ctx: Request<Data>) -> SResult<Response> {
Ok(tide::Response::new(200).body_json(&settings).unwrap()) Ok(tide::Response::new(200).body_json(&settings).unwrap())
} }
#[derive(Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UpdateSettings {
pub ranking_rules: Option<Vec<String>>,
pub distinct_attribute: Option<String>,
pub primary_key: Option<String>,
pub searchable_attributes: Option<Vec<String>>,
pub displayed_attributes: Option<HashSet<String>>,
pub stop_words: Option<BTreeSet<String>>,
pub synonyms: Option<BTreeMap<String, Vec<String>>>,
pub accept_new_fields: Option<bool>,
}
pub async fn update_all(mut ctx: Request<Data>) -> SResult<Response> { pub async fn update_all(mut ctx: Request<Data>) -> SResult<Response> {
ctx.is_allowed(Private)?; ctx.is_allowed(Private)?;
let index = ctx.index()?; let index = ctx.index()?;
let settings_update: UpdateSettings = let settings: Settings =
ctx.body_json().await.map_err(ResponseError::bad_request)?; ctx.body_json().await.map_err(ResponseError::bad_request)?;
let db = &ctx.state().db; let db = &ctx.state().db;
let settings = Settings {
ranking_rules: Some(settings_update.ranking_rules),
distinct_attribute: Some(settings_update.distinct_attribute),
searchable_attributes: Some(settings_update.searchable_attributes),
displayed_attributes: Some(settings_update.displayed_attributes),
stop_words: Some(settings_update.stop_words),
synonyms: Some(settings_update.synonyms),
accept_new_fields: Some(settings_update.accept_new_fields),
};
let mut writer = db.update_write_txn()?; let mut writer = db.update_write_txn()?;
let settings = settings.into_update().map_err(ResponseError::bad_request)?; let settings = settings.into_update().map_err(ResponseError::bad_request)?;
let update_id = index.settings_update(&mut writer, settings)?; let update_id = index.settings_update(&mut writer, settings)?;

View File

@ -44,30 +44,23 @@ impl Server {
} }
} }
fn wait_update_id(&mut self, update_id: u64) { pub fn wait_update_id(&mut self, update_id: u64) {
loop { loop {
let req = http::Request::get(format!("/indexes/{}/updates/{}", self.uid, update_id)) let (response, status_code) = self.get_update_status(update_id);
.body(Body::empty()) assert_eq!(status_code, 200);
.unwrap();
let res = self.mock.simulate(req).unwrap();
assert_eq!(res.status(), 200);
let mut buf = Vec::new();
block_on(res.into_body().read_to_end(&mut buf)).unwrap();
let response: Value = serde_json::from_slice(&buf).unwrap();
if response["status"] == "processed" || response["status"] == "error" { if response["status"] == "processed" || response["status"] == "error" {
eprintln!("{:#?}", response); eprintln!("{:#?}", response);
return; return;
} }
block_on(sleep(Duration::from_secs(1))); block_on(sleep(Duration::from_secs(1)));
} }
} }
// // Global Http request GET/POST/DELETE async or sync // Global Http request GET/POST/DELETE async or sync
fn get_request(&mut self, url: &str) -> (Value, StatusCode) { pub fn get_request(&mut self, url: &str) -> (Value, StatusCode) {
eprintln!("get_request: {}", url); eprintln!("get_request: {}", url);
let req = http::Request::get(url).body(Body::empty()).unwrap(); let req = http::Request::get(url).body(Body::empty()).unwrap();
let res = self.mock.simulate(req).unwrap(); let res = self.mock.simulate(req).unwrap();
@ -79,7 +72,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub fn post_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("post_request: {}", url); eprintln!("post_request: {}", url);
let body_bytes = body.to_string().into_bytes(); let body_bytes = body.to_string().into_bytes();
@ -95,7 +88,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("post_request_async: {}", url); eprintln!("post_request_async: {}", url);
let (response, status_code) = self.post_request(url, body); let (response, status_code) = self.post_request(url, body);
assert_eq!(status_code, 202); assert_eq!(status_code, 202);
@ -104,7 +97,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("put_request: {}", url); eprintln!("put_request: {}", url);
let body_bytes = body.to_string().into_bytes(); let body_bytes = body.to_string().into_bytes();
@ -120,7 +113,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { pub fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
eprintln!("put_request_async: {}", url); eprintln!("put_request_async: {}", url);
let (response, status_code) = self.put_request(url, body); let (response, status_code) = self.put_request(url, body);
assert!(response["updateId"].as_u64().is_some()); assert!(response["updateId"].as_u64().is_some());
@ -129,7 +122,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { pub fn delete_request(&mut self, url: &str) -> (Value, StatusCode) {
eprintln!("delete_request: {}", url); eprintln!("delete_request: {}", url);
let req = http::Request::delete(url).body(Body::empty()).unwrap(); let req = http::Request::delete(url).body(Body::empty()).unwrap();
let res = self.mock.simulate(req).unwrap(); let res = self.mock.simulate(req).unwrap();
@ -141,7 +134,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { pub fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) {
eprintln!("delete_request_async: {}", url); eprintln!("delete_request_async: {}", url);
let (response, status_code) = self.delete_request(url); let (response, status_code) = self.delete_request(url);
assert!(response["updateId"].as_u64().is_some()); assert!(response["updateId"].as_u64().is_some());
@ -150,7 +143,7 @@ impl Server {
(response, status_code) (response, status_code)
} }
// // All Routes // All Routes
pub fn list_indexes(&mut self) -> (Value, StatusCode) { pub fn list_indexes(&mut self) -> (Value, StatusCode) {
self.get_request("/indexes") self.get_request("/indexes")
@ -205,6 +198,11 @@ impl Server {
self.post_request_async(&url, body); self.post_request_async(&url, body);
} }
pub fn add_or_replace_multiple_documents_sync(&mut self, body: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents", self.uid);
self.post_request(&url, body)
}
pub fn add_or_update_multiple_documents(&mut self, body: Value) { pub fn add_or_update_multiple_documents(&mut self, body: Value) {
let url = format!("/indexes/{}/documents", self.uid); let url = format!("/indexes/{}/documents", self.uid);
self.put_request_async(&url, body); self.put_request_async(&url, body);

View File

@ -0,0 +1,38 @@
use serde_json::json;
mod common;
// Test issue https://github.com/meilisearch/MeiliSearch/issues/519
#[test]
fn check_add_documents_with_primary_key_param() {
let mut server = common::Server::with_uid("movies");
// 1 - Create the index with no primary_key
let body = json!({
"uid": "movies",
});
let (response, status_code) = server.create_index(body);
assert_eq!(status_code, 201);
assert_eq!(response["primaryKey"], json!(null));
// 2 - Add documents
let body = json!([{
"title": "Test",
"comment": "comment test"
}]);
let url = "/indexes/movies/documents?primaryKey=title";
let (response, status_code) = server.post_request(&url, body);
eprintln!("{:#?}", response);
assert_eq!(status_code, 202);
let update_id = response["updateId"].as_u64().unwrap();
server.wait_update_id(update_id);
// 3 - Check update sucess
let (response, status_code) = server.get_update_status(update_id);
assert_eq!(status_code, 200);
assert_eq!(response["status"], "processed");
}

View File

@ -625,3 +625,35 @@ fn create_index_without_primary_key_and_search() {
assert_eq!(status_code, 200); assert_eq!(status_code, 200);
assert_eq!(response["hits"].as_array().unwrap().len(), 0); assert_eq!(response["hits"].as_array().unwrap().len(), 0);
} }
// Test the error message when we push an document update and impossibility to find primary key
// Test issue https://github.com/meilisearch/MeiliSearch/issues/517
#[test]
fn check_add_documents_without_primary_key() {
let mut server = common::Server::with_uid("movies");
// 1 - Create the index with no primary_key
let body = json!({
"uid": "movies",
});
let (response, status_code) = server.create_index(body);
assert_eq!(status_code, 201);
assert_eq!(response["primaryKey"], json!(null));
// 2- Add document
let body = json!([{
"title": "Test",
"comment": "comment test"
}]);
let (response, status_code) = server.add_or_replace_multiple_documents_sync(body);
let expected = json!({
"message": "Could not infer a primary key"
});
assert_eq!(status_code, 400);
assert_json_eq!(response, expected, ordered: false);
}

View File

@ -644,7 +644,6 @@ fn search_with_settings_basic() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"tagline", "tagline",
@ -751,7 +750,6 @@ fn search_with_settings_stop_words() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"tagline", "tagline",
@ -858,7 +856,6 @@ fn search_with_settings_synonyms() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"tagline", "tagline",
@ -970,7 +967,6 @@ fn search_with_settings_ranking_rules() {
"desc(popularity)" "desc(popularity)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"tagline", "tagline",
@ -1077,7 +1073,6 @@ fn search_with_settings_searchable_attributes() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"tagline", "tagline",
"overview", "overview",
@ -1183,7 +1178,6 @@ fn search_with_settings_displayed_attributes() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"tagline", "tagline",
@ -1254,7 +1248,6 @@ fn search_with_settings_searchable_attributes_2() {
"desc(vote_average)" "desc(vote_average)"
], ],
"distinctAttribute": null, "distinctAttribute": null,
"primaryKey": "id",
"searchableAttributes": [ "searchableAttributes": [
"tagline", "tagline",
"overview", "overview",

View File

@ -192,6 +192,7 @@ fn write_all_and_update() {
"exactness", "exactness",
"desc(release_date)", "desc(release_date)",
], ],
"distinctAttribute": null,
"searchableAttributes": [ "searchableAttributes": [
"title", "title",
"description", "description",
@ -204,8 +205,7 @@ fn write_all_and_update() {
"rank", "rank",
"poster", "poster",
], ],
"stopWords": [ "stopWords": [],
],
"synonyms": { "synonyms": {
"wolverine": ["xmen", "logan"], "wolverine": ["xmen", "logan"],
"logan": ["wolverine", "xmen"], "logan": ["wolverine", "xmen"],
@ -321,3 +321,111 @@ fn test_default_settings_2() {
assert_json_eq!(body, response, ordered: false); assert_json_eq!(body, response, ordered: false);
} }
// Test issue https://github.com/meilisearch/MeiliSearch/issues/516
#[test]
fn write_setting_and_update_partial() {
let mut server = common::Server::with_uid("movies");
let body = json!({
"uid": "movies",
});
server.create_index(body);
// 2 - Send the settings
let body = json!({
"searchableAttributes": [
"uid",
"movie_id",
"title",
"description",
"poster",
"release_date",
"rank",
],
"displayedAttributes": [
"title",
"description",
"poster",
"release_date",
"rank",
]
});
server.update_all_settings(body.clone());
// 2 - Send the settings
let body = json!({
"rankingRules": [
"typo",
"words",
"proximity",
"attribute",
"wordsPosition",
"exactness",
"desc(release_date)",
"desc(rank)",
],
"distinctAttribute": "movie_id",
"stopWords": [
"the",
"a",
"an",
],
"synonyms": {
"wolverine": ["xmen", "logan"],
"logan": ["wolverine"],
},
"acceptNewFields": false,
});
server.update_all_settings(body.clone());
// 2 - Send the settings
let expected = json!({
"rankingRules": [
"typo",
"words",
"proximity",
"attribute",
"wordsPosition",
"exactness",
"desc(release_date)",
"desc(rank)",
],
"distinctAttribute": "movie_id",
"searchableAttributes": [
"uid",
"movie_id",
"title",
"description",
"poster",
"release_date",
"rank",
],
"displayedAttributes": [
"title",
"description",
"poster",
"release_date",
"rank",
],
"stopWords": [
"the",
"a",
"an",
],
"synonyms": {
"wolverine": ["xmen", "logan"],
"logan": ["wolverine"],
},
"acceptNewFields": false,
});
let (response, _status_code) = server.get_all_settings();
assert_json_eq!(expected, response, ordered: false);
}

View File

@ -288,3 +288,62 @@ fn index_new_fields_false_then_true() {
assert_eq!(status_code, 200); assert_eq!(status_code, 200);
assert_json_eq!(response, expected); assert_json_eq!(response, expected);
} }
// Fix issue https://github.com/meilisearch/MeiliSearch/issues/518
#[test]
fn accept_new_fields_does_not_take_into_account_the_primary_key () {
let mut server = common::Server::with_uid("movies");
// 1 - Create an index with no primary-key
let body = json!({
"uid": "movies",
});
let (response, status_code) = server.create_index(body);
assert_eq!(status_code, 201);
assert_eq!(response["primaryKey"], json!(null));
// 2 - Add searchable and displayed attributes as: ["title"] & Set acceptNewFields to false
let body = json!({
"searchableAttributes": ["title"],
"displayedAttributes": ["title"],
"acceptNewFields": false,
});
server.update_all_settings(body);
// 4 - Add a document
let body = json!([{
"id": 1,
"title": "Test",
"comment": "comment test"
}]);
server.add_or_replace_multiple_documents(body);
// 5 - Get settings, they should not changed
let (response, _status_code) = server.get_all_settings();
let expected = json!({
"rankingRules": [
"typo",
"words",
"proximity",
"attribute",
"wordsPosition",
"exactness",
],
"distinctAttribute": null,
"searchableAttributes": ["title"],
"displayedAttributes": ["title"],
"stopWords": [],
"synonyms": {},
"acceptNewFields": false,
});
assert_json_eq!(response, expected, ordered: false);
}

View File

@ -63,8 +63,10 @@ impl Schema {
let id = self.insert(name)?; let id = self.insert(name)?;
self.primary_key = Some(id); self.primary_key = Some(id);
self.set_indexed(name)?; if self.accept_new_fields {
self.set_displayed(name)?; self.set_indexed(name)?;
self.set_displayed(name)?;
}
Ok(id) Ok(id)
} }