From 8f73251012af42a0384d226f878870fb9f1973e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Wed, 10 Aug 2022 12:31:09 +0200 Subject: [PATCH 1/5] Use mimalloc for benchmarks on macOS --- benchmarks/Cargo.toml | 4 ++++ benchmarks/benches/formatting.rs | 4 ++++ benchmarks/benches/indexing.rs | 4 ++++ benchmarks/benches/search_geo.rs | 4 ++++ benchmarks/benches/search_songs.rs | 4 ++++ benchmarks/benches/search_wiki.rs | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 896ccd739..1dc9941c3 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -13,6 +13,10 @@ csv = "1.1.6" [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" +[target.'cfg(target_os = "macos")'.dependencies] +mimalloc = { version = "0.1.29", default-features = false } + + [dev-dependencies] heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } criterion = { version = "0.3.5", features = ["html_reports"] } diff --git a/benchmarks/benches/formatting.rs b/benchmarks/benches/formatting.rs index f0ef8ea15..be9d965a9 100644 --- a/benchmarks/benches/formatting.rs +++ b/benchmarks/benches/formatting.rs @@ -6,6 +6,10 @@ use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords}; #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(target_os = "macos")] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + struct Conf<'a> { name: &'a str, text: &'a str, diff --git a/benchmarks/benches/indexing.rs b/benchmarks/benches/indexing.rs index c756583e6..d0a091298 100644 --- a/benchmarks/benches/indexing.rs +++ b/benchmarks/benches/indexing.rs @@ -18,6 +18,10 @@ use roaring::RoaringBitmap; #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(target_os = "macos")] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + const BENCHMARK_ITERATION: usize = 10; fn setup_dir(path: impl AsRef) { diff --git a/benchmarks/benches/search_geo.rs b/benchmarks/benches/search_geo.rs index 84448c32d..65aeef01e 100644 --- a/benchmarks/benches/search_geo.rs +++ b/benchmarks/benches/search_geo.rs @@ -9,6 +9,10 @@ use utils::Conf; #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(target_os = "macos")] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + fn base_conf(builder: &mut Settings) { let displayed_fields = ["geonameid", "name", "asciiname", "alternatenames", "_geo", "population"] diff --git a/benchmarks/benches/search_songs.rs b/benchmarks/benches/search_songs.rs index 6b11799ec..05ba39cdd 100644 --- a/benchmarks/benches/search_songs.rs +++ b/benchmarks/benches/search_songs.rs @@ -9,6 +9,10 @@ use utils::Conf; #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(target_os = "macos")] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + fn base_conf(builder: &mut Settings) { let displayed_fields = ["id", "title", "album", "artist", "genre", "country", "released", "duration"] diff --git a/benchmarks/benches/search_wiki.rs b/benchmarks/benches/search_wiki.rs index 9ef75efeb..20d62fba6 100644 --- a/benchmarks/benches/search_wiki.rs +++ b/benchmarks/benches/search_wiki.rs @@ -9,6 +9,10 @@ use utils::Conf; #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(target_os = "macos")] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + fn base_conf(builder: &mut Settings) { let displayed_fields = ["title", "body", "url"].iter().map(|s| s.to_string()).collect(); builder.set_displayed_fields(displayed_fields); From fb2b6c0c28dc6c9135ce5a33f87f3918c6062cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Wed, 10 Aug 2022 16:56:42 +0200 Subject: [PATCH 2/5] Use mimalloc for benchmarks on all platforms --- benchmarks/Cargo.toml | 6 ------ benchmarks/benches/formatting.rs | 5 ----- benchmarks/benches/indexing.rs | 5 ----- benchmarks/benches/search_geo.rs | 5 ----- benchmarks/benches/search_songs.rs | 5 ----- benchmarks/benches/search_wiki.rs | 5 ----- 6 files changed, 31 deletions(-) diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 1dc9941c3..e63210573 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -9,14 +9,8 @@ milli = { path = "../milli" } anyhow = "1.0.56" serde_json = { version = "1.0.79", features = ["preserve_order"] } csv = "1.1.6" - -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" - -[target.'cfg(target_os = "macos")'.dependencies] mimalloc = { version = "0.1.29", default-features = false } - [dev-dependencies] heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } criterion = { version = "0.3.5", features = ["html_reports"] } diff --git a/benchmarks/benches/formatting.rs b/benchmarks/benches/formatting.rs index be9d965a9..25c5a0ba8 100644 --- a/benchmarks/benches/formatting.rs +++ b/benchmarks/benches/formatting.rs @@ -2,11 +2,6 @@ use criterion::{criterion_group, criterion_main}; use milli::tokenizer::TokenizerBuilder; use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords}; -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(target_os = "macos")] #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/benchmarks/benches/indexing.rs b/benchmarks/benches/indexing.rs index d0a091298..d532c85d9 100644 --- a/benchmarks/benches/indexing.rs +++ b/benchmarks/benches/indexing.rs @@ -14,11 +14,6 @@ use rand::seq::SliceRandom; use rand_chacha::rand_core::SeedableRng; use roaring::RoaringBitmap; -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(target_os = "macos")] #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/benchmarks/benches/search_geo.rs b/benchmarks/benches/search_geo.rs index 65aeef01e..faea4e3e0 100644 --- a/benchmarks/benches/search_geo.rs +++ b/benchmarks/benches/search_geo.rs @@ -5,11 +5,6 @@ use criterion::{criterion_group, criterion_main}; use milli::update::Settings; use utils::Conf; -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(target_os = "macos")] #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/benchmarks/benches/search_songs.rs b/benchmarks/benches/search_songs.rs index 05ba39cdd..a1245528f 100644 --- a/benchmarks/benches/search_songs.rs +++ b/benchmarks/benches/search_songs.rs @@ -5,11 +5,6 @@ use criterion::{criterion_group, criterion_main}; use milli::update::Settings; use utils::Conf; -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(target_os = "macos")] #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/benchmarks/benches/search_wiki.rs b/benchmarks/benches/search_wiki.rs index 20d62fba6..b792c2645 100644 --- a/benchmarks/benches/search_wiki.rs +++ b/benchmarks/benches/search_wiki.rs @@ -5,11 +5,6 @@ use criterion::{criterion_group, criterion_main}; use milli::update::Settings; use utils::Conf; -#[cfg(target_os = "linux")] -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(target_os = "macos")] #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; From 20be69e1b94aa375077b8a55630a88b050d652a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Tue, 16 Aug 2022 20:09:36 +0200 Subject: [PATCH 3/5] Always use mimalloc as the global allocator --- cli/Cargo.toml | 3 +-- cli/src/main.rs | 3 +-- helpers/Cargo.toml | 3 +-- helpers/src/main.rs | 3 +-- http-ui/Cargo.toml | 3 +-- http-ui/src/main.rs | 3 +-- infos/Cargo.toml | 4 +--- infos/src/main.rs | 3 +-- milli/fuzz/Cargo.toml | 4 +--- milli/fuzz/fuzz_targets/indexing.rs | 3 +-- 10 files changed, 10 insertions(+), 22 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e4de70031..9ca03894b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -18,6 +18,5 @@ byte-unit = { version = "4.0.14", features = ["serde"] } bimap = "0.6.2" csv = "1.1.6" stderrlog = "0.5.1" +mimalloc = { version = "0.1.29", default-features = false } -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" diff --git a/cli/src/main.rs b/cli/src/main.rs index 35fef95c6..8485560f5 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -16,9 +16,8 @@ use milli::update::{self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerCon use milli::{Index, Object}; use structopt::StructOpt; -#[cfg(target_os = "linux")] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; #[derive(Debug, StructOpt)] #[structopt(name = "Milli CLI", about = "A simple CLI to manipulate a milli index.")] diff --git a/helpers/Cargo.toml b/helpers/Cargo.toml index 46c50de43..4d32dc32a 100644 --- a/helpers/Cargo.toml +++ b/helpers/Cargo.toml @@ -14,5 +14,4 @@ milli = { path = "../milli" } stderrlog = "0.5.1" structopt = { version = "0.3.26", default-features = false } -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" +mimalloc = { version = "0.1.29", default-features = false } \ No newline at end of file diff --git a/helpers/src/main.rs b/helpers/src/main.rs index b325aef89..0081965ad 100644 --- a/helpers/src/main.rs +++ b/helpers/src/main.rs @@ -5,9 +5,8 @@ use heed::{CompactionOption, Env, EnvOpenOptions}; use structopt::StructOpt; use Command::*; -#[cfg(target_os = "linux")] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; #[derive(Debug, StructOpt)] /// Some helpers commands for milli. diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index 43e046c11..b3763409e 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -30,6 +30,7 @@ serde_json = { version = "1.0.79", features = ["preserve_order"] } tokio = { version = "1.17.0", features = ["full"] } tokio-stream = { version = "0.1.8", default-features = false, features = ["sync"] } warp = "0.3.2" +mimalloc = { version = "0.1.29", default-features = false } # logging log = "0.4.14" @@ -45,5 +46,3 @@ csv = "1.1.6" maplit = "1.0.2" serde_test = "1.0.136" -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs index 83fce9a9c..de5d3c5ab 100644 --- a/http-ui/src/main.rs +++ b/http-ui/src/main.rs @@ -42,9 +42,8 @@ use warp::Filter; use self::update_store::UpdateStore; -#[cfg(target_os = "linux")] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; static GLOBAL_CONFIG: OnceCell = OnceCell::new(); diff --git a/infos/Cargo.toml b/infos/Cargo.toml index ea1ee9193..7cbe52693 100644 --- a/infos/Cargo.toml +++ b/infos/Cargo.toml @@ -15,6 +15,4 @@ roaring = "0.9.0" serde_json = "1.0.79" stderrlog = "0.5.1" structopt = { version = "0.3.26", default-features = false } - -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" +mimalloc = { version = "0.1.29", default-features = false } diff --git a/infos/src/main.rs b/infos/src/main.rs index feec17557..2862fed7a 100644 --- a/infos/src/main.rs +++ b/infos/src/main.rs @@ -11,9 +11,8 @@ use milli::{FieldId, Index}; use structopt::StructOpt; use Command::*; -#[cfg(target_os = "linux")] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; const ALL_DATABASE_NAMES: &[&str] = &[ MAIN, diff --git a/milli/fuzz/Cargo.toml b/milli/fuzz/Cargo.toml index e734936fb..7e1bea3c5 100644 --- a/milli/fuzz/Cargo.toml +++ b/milli/fuzz/Cargo.toml @@ -16,9 +16,7 @@ serde_json = { version = "1.0.62", features = ["preserve_order"] } anyhow = "1.0" tempfile = "3.3" arbitrary-json = "0.1.0" - -[target.'cfg(target_os = "linux")'.dependencies] -jemallocator = "0.3.2" +mimalloc = { version = "0.1.29", default-features = false } [dependencies.milli] path = ".." diff --git a/milli/fuzz/fuzz_targets/indexing.rs b/milli/fuzz/fuzz_targets/indexing.rs index e4f42655e..a447aebe2 100644 --- a/milli/fuzz/fuzz_targets/indexing.rs +++ b/milli/fuzz/fuzz_targets/indexing.rs @@ -12,9 +12,8 @@ use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Setting use milli::Index; use serde_json::{Map, Value}; -#[cfg(target_os = "linux")] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; /// reads json from input and write an obkv batch to writer. pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result { From f20e588ec1a19bcfbbd28de7b0636951edc40d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Wed, 17 Aug 2022 07:44:33 +0200 Subject: [PATCH 4/5] Make sure there is one newline at eof in cargo.toml --- cli/Cargo.toml | 1 - helpers/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 9ca03894b..e012c8a33 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,4 +19,3 @@ bimap = "0.6.2" csv = "1.1.6" stderrlog = "0.5.1" mimalloc = { version = "0.1.29", default-features = false } - diff --git a/helpers/Cargo.toml b/helpers/Cargo.toml index 4d32dc32a..1167bd353 100644 --- a/helpers/Cargo.toml +++ b/helpers/Cargo.toml @@ -14,4 +14,4 @@ milli = { path = "../milli" } stderrlog = "0.5.1" structopt = { version = "0.3.26", default-features = false } -mimalloc = { version = "0.1.29", default-features = false } \ No newline at end of file +mimalloc = { version = "0.1.29", default-features = false } From 5d59bfde8a102f2da004619b8981fd8030316bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Wed, 17 Aug 2022 11:46:56 +0200 Subject: [PATCH 5/5] Sort Cargo.toml dependencies --- benchmarks/Cargo.toml | 8 ++++---- cli/Cargo.toml | 18 +++++++++--------- helpers/Cargo.toml | 3 +-- http-ui/Cargo.toml | 6 +++--- infos/Cargo.toml | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index e63210573..600525372 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -5,15 +5,15 @@ edition = "2018" publish = false [dependencies] -milli = { path = "../milli" } anyhow = "1.0.56" -serde_json = { version = "1.0.79", features = ["preserve_order"] } csv = "1.1.6" +milli = { path = "../milli" } mimalloc = { version = "0.1.29", default-features = false } +serde_json = { version = "1.0.79", features = ["preserve_order"] } [dev-dependencies] -heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } criterion = { version = "0.3.5", features = ["html_reports"] } +heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } rand = "0.8.5" rand_chacha = "0.3.1" roaring = "0.9.0" @@ -21,8 +21,8 @@ roaring = "0.9.0" [build-dependencies] anyhow = "1.0.56" bytes = "1.1.0" -flate2 = "1.0.22" convert_case = "0.5.0" +flate2 = "1.0.22" reqwest = { version = "0.11.9", features = ["blocking", "rustls-tls"], default-features = false } [[bench]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e012c8a33..504df712e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,16 +6,16 @@ description = "A CLI to interact with a milli index" publish = false [dependencies] +bimap = "0.6.2" +byte-unit = { version = "4.0.14", features = ["serde"] } +color-eyre = "0.6.1" +csv = "1.1.6" +eyre = "0.6.7" +heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] } indicatif = "0.16.2" +milli = { path = "../milli" } +mimalloc = { version = "0.1.29", default-features = false } serde = "1.0.136" serde_json = "1.0.79" -structopt = "0.3.26" -milli = { path = "../milli" } -eyre = "0.6.7" -color-eyre = "0.6.1" -heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] } -byte-unit = { version = "4.0.14", features = ["serde"] } -bimap = "0.6.2" -csv = "1.1.6" stderrlog = "0.5.1" -mimalloc = { version = "0.1.29", default-features = false } +structopt = "0.3.26" diff --git a/helpers/Cargo.toml b/helpers/Cargo.toml index 1167bd353..bd09574f3 100644 --- a/helpers/Cargo.toml +++ b/helpers/Cargo.toml @@ -11,7 +11,6 @@ anyhow = "1.0.56" byte-unit = { version = "4.0.14", default-features = false, features = ["std"] } heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } milli = { path = "../milli" } +mimalloc = { version = "0.1.29", default-features = false } stderrlog = "0.5.1" structopt = { version = "0.3.26", default-features = false } - -mimalloc = { version = "0.1.29", default-features = false } diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index b3763409e..993818f93 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -13,6 +13,7 @@ crossbeam-channel = "0.5.2" heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } memmap2 = "0.5.3" milli = { path = "../milli" } +mimalloc = { version = "0.1.29", default-features = false } once_cell = "1.10.0" rayon = "1.5.1" structopt = { version = "0.3.26", default-features = false, features = ["wrap_help"] } @@ -30,17 +31,16 @@ serde_json = { version = "1.0.79", features = ["preserve_order"] } tokio = { version = "1.17.0", features = ["full"] } tokio-stream = { version = "0.1.8", default-features = false, features = ["sync"] } warp = "0.3.2" -mimalloc = { version = "0.1.29", default-features = false } # logging +fst = "0.4.7" log = "0.4.14" stderrlog = "0.5.1" -fst = "0.4.7" # Temporary fix for bitvec, remove once fixed. (https://github.com/bitvecto-rs/bitvec/issues/105) -funty = "2.0.0" bimap = "0.6.2" csv = "1.1.6" +funty = "2.0.0" [dev-dependencies] maplit = "1.0.2" diff --git a/infos/Cargo.toml b/infos/Cargo.toml index 7cbe52693..8c92ae649 100644 --- a/infos/Cargo.toml +++ b/infos/Cargo.toml @@ -11,8 +11,8 @@ byte-unit = { version = "4.0.14", default-features = false, features = ["std"] } csv = "1.1.6" heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" } milli = { path = "../milli" } +mimalloc = { version = "0.1.29", default-features = false } roaring = "0.9.0" serde_json = "1.0.79" stderrlog = "0.5.1" structopt = { version = "0.3.26", default-features = false } -mimalloc = { version = "0.1.29", default-features = false }