From 8fecc6238ddd60937564a4fa55cfe1642d1d62f2 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 21 Mar 2022 10:50:55 +0100 Subject: [PATCH 1/4] Make the test use the default CLI options --- meilisearch-http/tests/common/server.rs | 27 +++---------------------- meilisearch-lib/src/options.rs | 13 ------------ 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index dcb4b6266..9a94085a4 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -1,4 +1,6 @@ #![allow(dead_code)] + +use clap::Parser; use std::path::Path; use actix_web::http::StatusCode; @@ -126,36 +128,13 @@ pub fn default_settings(dir: impl AsRef) -> Opt { Opt { db_path: dir.as_ref().join("db"), dumps_dir: dir.as_ref().join("dump"), - http_addr: "127.0.0.1:7700".to_owned(), - master_key: None, env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), - ssl_cert_path: None, - ssl_key_path: None, - ssl_auth_path: None, - ssl_ocsp_path: None, - ssl_require_auth: false, - ssl_resumption: false, - ssl_tickets: false, - import_snapshot: None, - ignore_missing_snapshot: false, - ignore_snapshot_if_db_exists: false, snapshot_dir: ".".into(), - schedule_snapshot: false, - snapshot_interval_sec: 0, - import_dump: None, - ignore_missing_dump: false, - ignore_dump_if_db_exists: false, - indexer_options: IndexerOpts { - // memory has to be unlimited because several meilisearch are running in test context. - max_memory: MaxMemory::unlimited(), - ..Default::default() - }, - log_level: "off".into(), - scheduler_options: meilisearch_lib::options::SchedulerConfig::default(), + ..Parser::parse_from(None as Option<&str>) } } diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index 195576799..e04396627 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -90,19 +90,6 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { } } -impl Default for IndexerOpts { - fn default() -> Self { - Self { - log_every_n: 100_000, - max_nb_chunks: None, - max_memory: MaxMemory::default(), - chunk_compression_type: CompressionType::None, - chunk_compression_level: None, - indexing_jobs: None, - } - } -} - /// A type used to detect the max memory available and use 2/3 of it. #[derive(Debug, Clone, Copy)] pub struct MaxMemory(Option); From acdb10a3076f07a7a22d6a40da323f0fae4e3389 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 21 Mar 2022 10:55:15 +0100 Subject: [PATCH 2/4] Remove some useless indexer options --- meilisearch-http/tests/common/server.rs | 1 - meilisearch-lib/src/options.rs | 15 +-------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 9a94085a4..7a2b0da86 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use meilisearch_auth::AuthController; use meilisearch_http::setup_meilisearch; -use meilisearch_lib::options::{IndexerOpts, MaxMemory}; use once_cell::sync::Lazy; use serde_json::Value; use tempfile::TempDir; diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index e04396627..12f0c0a12 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -3,7 +3,7 @@ use std::{convert::TryFrom, ops::Deref, str::FromStr}; use byte_unit::{Byte, ByteError}; use clap::Parser; -use milli::{update::IndexerConfig, CompressionType}; +use milli::update::IndexerConfig; use serde::Serialize; use sysinfo::{RefreshKind, System, SystemExt}; @@ -28,17 +28,6 @@ pub struct IndexerOpts { #[clap(long, default_value_t)] pub max_memory: MaxMemory, - /// The name of the compression algorithm to use when compressing intermediate - /// Grenad chunks while indexing documents. - /// - /// Choosing a fast algorithm will make the indexing faster but may consume more memory. - #[clap(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] - pub chunk_compression_type: CompressionType, - - /// The level of compression of the chosen algorithm. - #[clap(long, requires = "chunk-compression-type")] - pub chunk_compression_level: Option, - /// Number of parallel jobs for indexing, defaults to # of CPUs. #[clap(long)] pub indexing_jobs: Option, @@ -81,8 +70,6 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { log_every_n: Some(other.log_every_n), max_nb_chunks: other.max_nb_chunks, max_memory: (*other.max_memory).map(|b| b.get_bytes() as usize), - chunk_compression_type: other.chunk_compression_type, - chunk_compression_level: other.chunk_compression_level, thread_pool: Some(thread_pool), max_positions_per_attributes: None, ..Default::default() From b3a11e04afb849082ab9c1b734070b209e6eb07a Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 22 Mar 2022 16:38:52 +0100 Subject: [PATCH 3/4] Implement Default on IndexerOpts again --- meilisearch-lib/src/options.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index 12f0c0a12..ef91ce695 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -69,7 +69,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { Ok(Self { log_every_n: Some(other.log_every_n), max_nb_chunks: other.max_nb_chunks, - max_memory: (*other.max_memory).map(|b| b.get_bytes() as usize), + max_memory: other.max_memory.map(|b| b.get_bytes() as usize), thread_pool: Some(thread_pool), max_positions_per_attributes: None, ..Default::default() @@ -77,6 +77,17 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { } } +impl Default for IndexerOpts { + fn default() -> Self { + Self { + log_every_n: 100_000, + max_nb_chunks: None, + max_memory: MaxMemory::default(), + indexing_jobs: None, + } + } +} + /// A type used to detect the max memory available and use 2/3 of it. #[derive(Debug, Clone, Copy)] pub struct MaxMemory(Option); From 891d042164cfbe04fb8bb7018405173eb60b2eca Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 21 Mar 2022 12:10:12 +0100 Subject: [PATCH 4/4] Remove the memory limit to let Windows tests pass --- meilisearch-http/tests/common/server.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 7a2b0da86..a1877c78e 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -7,6 +7,7 @@ use actix_web::http::StatusCode; use byte_unit::{Byte, ByteUnit}; use meilisearch_auth::AuthController; use meilisearch_http::setup_meilisearch; +use meilisearch_lib::options::{IndexerOpts, MaxMemory}; use once_cell::sync::Lazy; use serde_json::Value; use tempfile::TempDir; @@ -134,6 +135,11 @@ pub fn default_settings(dir: impl AsRef) -> Opt { max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), snapshot_dir: ".".into(), + indexer_options: IndexerOpts { + // memory has to be unlimited because several meilisearch are running in test context. + max_memory: MaxMemory::unlimited(), + ..Parser::parse_from(None as Option<&str>) + }, ..Parser::parse_from(None as Option<&str>) } }