Merge #3278

3278: Remove `--max-index-size` and `--max-task-db-size` flags r=Kerollmops a=dureuill

# Pull Request

## Related issue
Fixes #3231 

## What does this PR do?
- Remove `--max-index-size` and `--max-task-db-size` flags from the CLI, config file and environment variable
- Set the size of all indexes to **500GiB** and the size of the task DB to **10GiB**.  Reviewers might want to review these values carefully.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Louis Dureuil <louis@meilisearch.com>

config.toml

@@ -31,14 +31,6 @@ log_level = "INFO"
 # Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#log-level
 
-max_index_size = "100 GiB"
-# Sets the maximum size of the index.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-index-size
-
-max_task_db_size = "100 GiB"
-# Sets the maximum size of the task database.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-task-db-size
-
 # max_indexing_memory = "2 GiB"
 # Sets the maximum amount of RAM Meilisearch can use when indexing.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-memory

index-scheduler/src/lib.rs

@@ -227,9 +227,9 @@ pub struct IndexSchedulerOptions {
     pub snapshots_path: PathBuf,
     /// The path to the folder containing the dumps.
     pub dumps_path: PathBuf,
-    /// The maximum size, in bytes, of each meilisearch index.
+    /// The maximum size, in bytes, of the task index.
     pub task_db_size: usize,
-    /// The maximum size, in bytes, of the tasks index.
+    /// The maximum size, in bytes, of each meilisearch index.
     pub index_size: usize,
     /// Configuration used during indexing for each meilisearch index.
     pub indexer_config: IndexerConfig,

meilisearch/src/analytics/segment_analytics.rs

@@ -225,8 +225,6 @@ struct Infos {
     ignore_missing_snapshot: bool,
     ignore_snapshot_if_db_exists: bool,
     http_addr: bool,
-    max_index_size: Byte,
-    max_task_db_size: Byte,
     http_payload_size_limit: Byte,
     log_level: String,
     max_indexing_memory: MaxMemory,
@@ -251,8 +249,8 @@ impl From<Opt> for Infos {
             http_addr,
             master_key: _,
             env,
-            max_index_size,
-            max_task_db_size,
+            max_index_size: _,
+            max_task_db_size: _,
             http_payload_size_limit,
             ssl_cert_path,
             ssl_key_path,
@@ -305,8 +303,6 @@ impl From<Opt> for Infos {
             ignore_missing_snapshot,
             ignore_snapshot_if_db_exists,
             http_addr: http_addr != default_http_addr(),
-            max_index_size,
-            max_task_db_size,
             http_payload_size_limit,
             log_level: log_level.to_string(),
             max_indexing_memory,

meilisearch/src/option.rs

@@ -29,8 +29,6 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
 #[cfg(all(not(debug_assertions), feature = "analytics"))]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
-const MEILI_MAX_INDEX_SIZE: &str = "MEILI_MAX_INDEX_SIZE";
-const MEILI_MAX_TASK_DB_SIZE: &str = "MEILI_MAX_TASK_DB_SIZE";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
 const MEILI_SSL_KEY_PATH: &str = "MEILI_SSL_KEY_PATH";
@@ -57,8 +55,6 @@ const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
 const DEFAULT_DB_PATH: &str = "./data.ms";
 const DEFAULT_HTTP_ADDR: &str = "localhost:7700";
 const DEFAULT_ENV: &str = "development";
-const DEFAULT_MAX_INDEX_SIZE: &str = "100 GiB";
-const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB";
 const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
 const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
 const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
@@ -69,6 +65,13 @@ const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY";
 const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS";
 const DEFAULT_LOG_EVERY_N: usize = 100000;
 
+// Each environment (index and task-db) is taking space in the virtual address space.
+//
+// The size of the virtual address space is limited by the OS. About 100TB for Linux and about 10TB for Windows.
+// This means that the number of indexes is limited to about 200 for Linux and about 20 for Windows.
+pub const INDEX_SIZE: u64 = 536_870_912_000; // 500 GiB
+pub const TASK_DB_SIZE: u64 = 10_737_418_240; // 10 GiB
+
 #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
 #[serde(rename_all = "UPPERCASE")]
 pub enum LogLevel {
@@ -157,14 +160,14 @@ pub struct Opt {
     pub no_analytics: bool,
 
     /// Sets the maximum size of the index. Value must be given in bytes or explicitly stating a base unit (for instance: 107374182400, '107.7Gb', or '107374 Mb').
-    #[clap(long, env = MEILI_MAX_INDEX_SIZE, default_value_t = default_max_index_size())]
-    #[serde(default = "default_max_index_size")]
+    #[clap(skip = default_max_index_size())]
+    #[serde(skip, default = "default_max_index_size")]
     pub max_index_size: Byte,
 
     /// Sets the maximum size of the task database. Value must be given in bytes or explicitly stating a
     /// base unit (for instance: 107374182400, '107.7Gb', or '107374 Mb').
-    #[clap(long, env = MEILI_MAX_TASK_DB_SIZE, default_value_t = default_max_task_db_size())]
-    #[serde(default = "default_max_task_db_size")]
+    #[clap(skip = default_max_task_db_size())]
+    #[serde(skip, default = "default_max_task_db_size")]
     pub max_task_db_size: Byte,
 
     /// Sets the maximum size of accepted payloads. Value must be given in bytes or explicitly stating a
@@ -361,8 +364,8 @@ impl Opt {
             http_addr,
             master_key,
             env,
-            max_index_size,
-            max_task_db_size,
+            max_index_size: _,
+            max_task_db_size: _,
             http_payload_size_limit,
             ssl_cert_path,
             ssl_key_path,
@@ -399,8 +402,6 @@ impl Opt {
         {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
         }
-        export_to_env_if_not_present(MEILI_MAX_INDEX_SIZE, max_index_size.to_string());
-        export_to_env_if_not_present(MEILI_MAX_TASK_DB_SIZE, max_task_db_size.to_string());
         export_to_env_if_not_present(
             MEILI_HTTP_PAYLOAD_SIZE_LIMIT,
             http_payload_size_limit.to_string(),
@@ -724,11 +725,11 @@ fn default_env() -> String {
 }
 
 fn default_max_index_size() -> Byte {
-    Byte::from_str(DEFAULT_MAX_INDEX_SIZE).unwrap()
+    Byte::from_bytes(INDEX_SIZE)
 }
 
 fn default_max_task_db_size() -> Byte {
-    Byte::from_str(DEFAULT_MAX_TASK_DB_SIZE).unwrap()
+    Byte::from_bytes(TASK_DB_SIZE)
 }
 
 fn default_http_payload_size_limit() -> Byte {