diff --git a/meilisearch/src/search.rs b/meilisearch/src/search.rs index b7d5b9114..c287f1ba0 100644 --- a/meilisearch/src/search.rs +++ b/meilisearch/src/search.rs @@ -375,9 +375,10 @@ pub fn perform_search( &displayed_ids, ); - let tokenizer = TokenizerBuilder::default().build(); + let mut tokenizer_buidler = TokenizerBuilder::default(); + tokenizer_buidler.create_char_map(true); - let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer); + let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_buidler.build()); formatter_builder.crop_marker(query.crop_marker); formatter_builder.highlight_prefix(query.highlight_pre_tag); formatter_builder.highlight_suffix(query.highlight_post_tag); diff --git a/meilisearch/tests/search/formatted.rs b/meilisearch/tests/search/formatted.rs index 191720602..076c14fe0 100644 --- a/meilisearch/tests/search/formatted.rs +++ b/meilisearch/tests/search/formatted.rs @@ -442,3 +442,37 @@ async fn displayedattr_2_smol() { ) .await; } + +#[cfg(feature = "default")] +#[actix_rt::test] +async fn test_cjk_highlight() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { "id": 0, "title": "この度、クーポンで無料で頂きました。" }, + { "id": 1, "title": "大卫到了扫罗那里" }, + ]); + index.add_documents(documents, None).await; + index.wait_task(0).await; + + index + .search(json!({"q": "で", "attributesToHighlight": ["title"]}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0]["_formatted"]["title"], + json!("この度、クーポン無料頂きました。") + ); + }) + .await; + + index + .search(json!({"q": "大卫", "attributesToHighlight": ["title"]}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0]["_formatted"]["title"], + json!("大卫到了扫罗那里") + ); + }) + .await; +}