From ae2b0e7aa785cb557d20a9643dcd96d4270c5a6d Mon Sep 17 00:00:00 2001
From: many <maxime@meilisearch.com>
Date: Mon, 6 Dec 2021 17:18:28 +0100
Subject: [PATCH] Use milli reexported tokenizer instead of importing
 meilisearch-tokenizer dependency

---
 Cargo.lock                          | 20 +-------------------
 meilisearch-http/Cargo.toml         |  1 -
 meilisearch-lib/Cargo.toml          |  1 -
 meilisearch-lib/src/index/search.rs |  2 +-
 4 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2c4235494..8d6e071bd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1685,7 +1685,6 @@ dependencies = [
  "meilisearch-auth",
  "meilisearch-error",
  "meilisearch-lib",
- "meilisearch-tokenizer 0.2.5",
  "mime",
  "num_cpus",
  "obkv",
@@ -1751,7 +1750,6 @@ dependencies = [
  "lazy_static",
  "log",
  "meilisearch-error",
- "meilisearch-tokenizer 0.2.5",
  "milli",
  "mime",
  "mockall",
@@ -1784,22 +1782,6 @@ dependencies = [
  "whoami",
 ]
 
-[[package]]
-name = "meilisearch-tokenizer"
-version = "0.2.5"
-source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.5#c0b5cf741ed9485147f2cbe523f2214d4fa4c395"
-dependencies = [
- "character_converter",
- "cow-utils",
- "deunicode",
- "fst",
- "jieba-rs",
- "once_cell",
- "slice-group-by",
- "unicode-segmentation",
- "whatlang",
-]
-
 [[package]]
 name = "meilisearch-tokenizer"
 version = "0.2.6"
@@ -1867,7 +1849,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer 0.2.6",
+ "meilisearch-tokenizer",
  "memmap2",
  "obkv",
  "once_cell",
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index a3631d278..2c6bc8924 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -50,7 +50,6 @@ log = "0.4.14"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-lib = { path = "../meilisearch-lib" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 obkv = "0.2.0"
diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml
index 32b1816f3..e0dd7bf91 100644
--- a/meilisearch-lib/Cargo.toml
+++ b/meilisearch-lib/Cargo.toml
@@ -29,7 +29,6 @@ itertools = "0.10.1"
 lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.21.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
diff --git a/meilisearch-lib/src/index/search.rs b/meilisearch-lib/src/index/search.rs
index 8ff6f13be..3f05e7dc1 100644
--- a/meilisearch-lib/src/index/search.rs
+++ b/meilisearch-lib/src/index/search.rs
@@ -4,7 +4,7 @@ use std::time::Instant;
 
 use either::Either;
 use indexmap::IndexMap;
-use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
+use milli::tokenizer::{Analyzer, AnalyzerConfig, Token};
 use milli::{AscDesc, FieldId, FieldsIdsMap, Filter, MatchingWords, SortError};
 use regex::Regex;
 use serde::{Deserialize, Serialize};