From f9ab85adbea6b7e1713ed70ef13fddb39ddabd30 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 26 Oct 2020 17:47:55 +0100 Subject: [PATCH 1/4] deunicase synonyms --- .../src/update/settings_update.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/meilisearch-core/src/update/settings_update.rs b/meilisearch-core/src/update/settings_update.rs index 94e337265..b02ad4fe0 100644 --- a/meilisearch-core/src/update/settings_update.rs +++ b/meilisearch-core/src/update/settings_update.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, btree_map::Entry}; use heed::Result as ZResult; use fst::{set::OpBuilder, SetBuilder}; @@ -126,7 +126,7 @@ pub fn apply_settings_update( } match settings.synonyms { - UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, synonyms)?, + UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, transform_synonyms(synonyms))? , UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?, UpdateState::Nothing => (), } @@ -138,6 +138,23 @@ pub fn apply_settings_update( Ok(()) } +fn transform_synonyms(synonyms: BTreeMap>) -> BTreeMap> { + synonyms + .into_iter() + .fold(BTreeMap::new(), |mut map, (key, values)| { + let deunicoded = deunicode::deunicode(&key); + match map.entry(deunicoded) { + Entry::Vacant(entry) => { + entry.insert(values); + } + Entry::Occupied(mut entry) => { + entry.get_mut().extend_from_slice(&values); + } + } + map + }) +} + fn apply_attributes_for_faceting_update( writer: &mut heed::RwTxn, index: &store::Index, From 91363daeaab879bf76cdd306840ad3827f53b75d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 26 Oct 2020 17:48:13 +0100 Subject: [PATCH 2/4] add tests --- meilisearch-http/tests/search.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/meilisearch-http/tests/search.rs b/meilisearch-http/tests/search.rs index 92879bb7b..3cce31972 100644 --- a/meilisearch-http/tests/search.rs +++ b/meilisearch-http/tests/search.rs @@ -1829,3 +1829,35 @@ async fn update_documents_with_facet_distribution() { let (response2, _) = server.search_post(search).await; assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); } + +#[actix_rt::test] +async fn test_search_synonyms_unicased() { + let mut server = common::Server::with_uid("test"); + let body = json!({ "uid": "test" }); + server.create_index(body).await; + let settings = json!({ + "synonyms": { + "cáse": ["truc"], + "case": ["machin"] + } + }); + server.update_all_settings(settings).await; + + let (response, _) = server.get_synonyms().await; + println!("response: {}", response); + assert_json_eq!(response, json!({"case":["machin", "truc"]})); + + let update = json!([ + { + "id": "1", + "title": "truc" + }, + ]); + server.add_or_update_multiple_documents(update).await; + + let search = json!({ + "q": "case", + }); + let (response, _) = server.search_post(search).await; + assert_eq!(response["hits"].as_array().unwrap().len(), 1); +} From 33c7c5a7e3d5544a2c1efba42e6c2ca5c2af8cd2 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 26 Oct 2020 21:33:39 +0100 Subject: [PATCH 3/4] remove del_synonyms function --- meilisearch-core/src/store/synonyms.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/meilisearch-core/src/store/synonyms.rs b/meilisearch-core/src/store/synonyms.rs index bf7472f96..c83445014 100644 --- a/meilisearch-core/src/store/synonyms.rs +++ b/meilisearch-core/src/store/synonyms.rs @@ -19,10 +19,6 @@ impl Synonyms { self.synonyms.put(writer, word, bytes) } - pub fn del_synonyms(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult { - self.synonyms.delete(writer, word) - } - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { self.synonyms.clear(writer) } From d35a104ad344e39ea61d26fa451b05b3878fa3f3 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 27 Oct 2020 11:40:31 +0100 Subject: [PATCH 4/4] requested changes --- meilisearch-core/src/store/synonyms.rs | 4 +-- .../src/update/settings_update.rs | 29 ++++++++----------- meilisearch-http/tests/search.rs | 5 +++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/meilisearch-core/src/store/synonyms.rs b/meilisearch-core/src/store/synonyms.rs index c83445014..31d4bb10f 100644 --- a/meilisearch-core/src/store/synonyms.rs +++ b/meilisearch-core/src/store/synonyms.rs @@ -12,14 +12,14 @@ pub struct Synonyms { } impl Synonyms { - pub fn put_synonyms(self, writer: &mut heed::RwTxn, word: &[u8], synonyms: &fst::Set) -> ZResult<()> + pub(crate) fn put_synonyms(self, writer: &mut heed::RwTxn, word: &[u8], synonyms: &fst::Set) -> ZResult<()> where A: AsRef<[u8]>, { let bytes = synonyms.as_fst().as_bytes(); self.synonyms.put(writer, word, bytes) } - pub fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { + pub(crate) fn clear(self, writer: &mut heed::RwTxn) -> ZResult<()> { self.synonyms.clear(writer) } diff --git a/meilisearch-core/src/update/settings_update.rs b/meilisearch-core/src/update/settings_update.rs index b02ad4fe0..7fa0f983c 100644 --- a/meilisearch-core/src/update/settings_update.rs +++ b/meilisearch-core/src/update/settings_update.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet, btree_map::Entry}; +use std::collections::{BTreeMap, BTreeSet}; use heed::Result as ZResult; use fst::{set::OpBuilder, SetBuilder}; @@ -126,7 +126,7 @@ pub fn apply_settings_update( } match settings.synonyms { - UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, transform_synonyms(synonyms))? , + UpdateState::Update(synonyms) => apply_synonyms_update(writer, index, canonicalize_synonyms(synonyms))? , UpdateState::Clear => apply_synonyms_update(writer, index, BTreeMap::new())?, UpdateState::Nothing => (), } @@ -138,21 +138,16 @@ pub fn apply_settings_update( Ok(()) } -fn transform_synonyms(synonyms: BTreeMap>) -> BTreeMap> { - synonyms - .into_iter() - .fold(BTreeMap::new(), |mut map, (key, values)| { - let deunicoded = deunicode::deunicode(&key); - match map.entry(deunicoded) { - Entry::Vacant(entry) => { - entry.insert(values); - } - Entry::Occupied(mut entry) => { - entry.get_mut().extend_from_slice(&values); - } - } - map - }) +fn canonicalize_synonyms(synonyms: BTreeMap>) -> BTreeMap> { + let mut canonicalized = BTreeMap::new(); + for (key, values) in synonyms { + let deunicoded = deunicode::deunicode(&key); + canonicalized + .entry(deunicoded) + .or_insert_with(Vec::new) + .extend_from_slice(&values); + } + canonicalized } fn apply_attributes_for_faceting_update( diff --git a/meilisearch-http/tests/search.rs b/meilisearch-http/tests/search.rs index 3cce31972..f4ce60219 100644 --- a/meilisearch-http/tests/search.rs +++ b/meilisearch-http/tests/search.rs @@ -1844,7 +1844,6 @@ async fn test_search_synonyms_unicased() { server.update_all_settings(settings).await; let (response, _) = server.get_synonyms().await; - println!("response: {}", response); assert_json_eq!(response, json!({"case":["machin", "truc"]})); let update = json!([ @@ -1860,4 +1859,8 @@ async fn test_search_synonyms_unicased() { }); let (response, _) = server.search_post(search).await; assert_eq!(response["hits"].as_array().unwrap().len(), 1); + + server.delete_synonyms().await; + let (response, _) = server.get_synonyms().await; + assert_json_eq!(response, json!({})); }