From 942d49314c75d00aa5e43df220d5acfef47e6b64 Mon Sep 17 00:00:00 2001
From: Louis Dureuil <louis.dureuil@xinra.net>
Date: Mon, 18 Dec 2023 22:14:26 +0100
Subject: [PATCH] Remove dependency that requires libstdc++

---
 Cargo.lock       | 5 -----
 milli/Cargo.toml | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d57d381ed..edcd26bf9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1592,9 +1592,6 @@ name = "esaxx-rs"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
-dependencies = [
- "cc",
-]
 
 [[package]]
 name = "fancy-regex"
@@ -5310,11 +5307,9 @@ version = "0.14.1"
 source = "git+https://github.com/huggingface/tokenizers.git?tag=v0.14.1#6357206cdcce4d78ffb1e0372feb456caea09375"
 dependencies = [
  "aho-corasick",
- "clap",
  "derive_builder",
  "esaxx-rs",
  "getrandom",
- "indicatif",
  "itertools 0.11.0",
  "lazy_static",
  "log",
diff --git a/milli/Cargo.toml b/milli/Cargo.toml
index b977d64f1..3e10c175a 100644
--- a/milli/Cargo.toml
+++ b/milli/Cargo.toml
@@ -77,7 +77,7 @@ csv = "1.2.1"
 candle-core = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
 candle-transformers = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
 candle-nn = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
-tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.14.1", version = "0.14.1" }
+tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.14.1", version = "0.14.1", default_features = false, features = ["onig"] }
 hf-hub = { git = "https://github.com/dureuill/hf-hub.git", branch = "rust_tls", default_features = false, features = [
     "online",
 ] }