Merge #3530

3530: Fix highlighter bug r=Kerollmops a=ManyTheFish

# Pull Request

There was a highlighting issue on CJK's character, we were highlighting too many characters and these additional characters were duplicated after the highlight tag.

## Related issue
Fixes #3517 
Fixes #3526 

## What does this PR do?
- add a test showcasing the bug
- fix the bug by activating the char_map creation of the tokenizer during the highlighting process


Co-authored-by: ManyTheFish <many@meilisearch.com>

meilisearch/src/search.rs

@@ -375,9 +375,10 @@ pub fn perform_search(
         &displayed_ids,
     );
 
-    let tokenizer = TokenizerBuilder::default().build();
+    let mut tokenizer_buidler = TokenizerBuilder::default();
+    tokenizer_buidler.create_char_map(true);
 
-    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer);
+    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_buidler.build());
     formatter_builder.crop_marker(query.crop_marker);
     formatter_builder.highlight_prefix(query.highlight_pre_tag);
     formatter_builder.highlight_suffix(query.highlight_post_tag);

meilisearch/tests/search/formatted.rs

@@ -442,3 +442,37 @@ async fn displayedattr_2_smol() {
         )
         .await;
 }
+
+#[cfg(feature = "default")]
+#[actix_rt::test]
+async fn test_cjk_highlight() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        { "id": 0, "title": "この度、クーポンで無料で頂きました。" },
+        { "id": 1, "title": "大卫到了扫罗那里" },
+    ]);
+    index.add_documents(documents, None).await;
+    index.wait_task(0).await;
+
+    index
+        .search(json!({"q": "で", "attributesToHighlight": ["title"]}), |response, code| {
+            assert_eq!(code, 200, "{}", response);
+            assert_eq!(
+                response["hits"][0]["_formatted"]["title"],
+                json!("この度、クーポン<em>で</em>無料<em>で</em>頂きました。")
+            );
+        })
+        .await;
+
+    index
+        .search(json!({"q": "大卫", "attributesToHighlight": ["title"]}), |response, code| {
+            assert_eq!(code, 200, "{}", response);
+            assert_eq!(
+                response["hits"][0]["_formatted"]["title"],
+                json!("<em>大卫</em>到了扫罗那里")
+            );
+        })
+        .await;
+}