From ecd36b15f0aff3c4bf787a8a2e20351969953d6f Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 13 Sep 2023 18:17:56 +0200 Subject: [PATCH] exposes all the s3 arguments --- index-scheduler/src/insta_snapshot.rs | 1 + index-scheduler/src/lib.rs | 13 ++++++- meilisearch/src/lib.rs | 12 +++++-- meilisearch/src/option.rs | 51 +++++++++++++++++++++++++-- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs index 2b87bc762..d42797b13 100644 --- a/index-scheduler/src/insta_snapshot.rs +++ b/index-scheduler/src/insta_snapshot.rs @@ -41,6 +41,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { planned_failures: _, run_loop_iteration: _, zookeeper: _, + options: _, } = inner.deref(); let rtxn = env.read_txn().unwrap(); diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 1be2df932..20dc058d6 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -468,7 +468,12 @@ impl IndexScheduler { let s3 = inner.options.s3.as_ref().unwrap(); let task = s3.get_object(format!("/tasks/{id:0>10}")).unwrap(); - + assert_eq!( + task.status_code(), + 200, + "could not reach the s3: {:?}", + task.as_str() + ); let task = serde_json::from_slice(task.as_slice()).unwrap(); inner.register_raw_task(&mut wtxn, &task).unwrap(); // we received a new tasks, we must wake up @@ -507,6 +512,12 @@ impl IndexScheduler { .unwrap(); let s3 = inner.options.s3.as_ref().unwrap(); let task = s3.get_object(format!("tasks/{id:0>10}")).unwrap(); + assert_eq!( + task.status_code(), + 200, + "could not reach the s3: {:?}", + task.as_str() + ); let task = serde_json::from_slice(task.as_slice()).unwrap(); inner.register_raw_task(&mut wtxn, &task).unwrap(); wtxn.commit().unwrap(); diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 4a4cdcc08..e8674ae21 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -247,9 +247,15 @@ fn open_or_create_database_unchecked( s3: opt.s3_url.as_ref().map(|url| { Arc::new( Bucket::new( - "test-rust-s3", - Region::Custom { region: "eu-central-1".to_owned(), endpoint: url.clone() }, - Credentials::default().unwrap(), + opt.s3_bucket.as_deref().unwrap(), + Region::Custom { region: opt.s3_region.clone(), endpoint: url.clone() }, + Credentials { + access_key: opt.s3_access_key.clone(), + secret_key: opt.s3_secret_key.clone(), + security_token: opt.s3_security_token.clone(), + session_token: None, + expiration: None, + }, ) .unwrap() .with_path_style(), diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 4e9d8ee9e..924f6855a 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -30,6 +30,11 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY"; const MEILI_ENV: &str = "MEILI_ENV"; const MEILI_ZK_URL: &str = "MEILI_ZK_URL"; const MEILI_S3_URL: &str = "MEILI_S3_URL"; +const MEILI_S3_BUCKET: &str = "MEILI_S3_BUCKET"; +const MEILI_S3_ACCESS_KEY: &str = "MEILI_S3_ACCESS_KEY"; +const MEILI_S3_SECRET_KEY: &str = "MEILI_S3_SECRET_KEY"; +const MEILI_S3_SECURITY_TOKEN: &str = "MEILI_S3_SECURITY_TOKEN"; +const MEILI_S3_REGION: &str = "MEILI_S3_REGION"; #[cfg(all(not(debug_assertions), feature = "analytics"))] const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; @@ -58,6 +63,7 @@ const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_DB_PATH: &str = "./data.ms"; const DEFAULT_HTTP_ADDR: &str = "localhost:7700"; const DEFAULT_ENV: &str = "development"; +const DEFAULT_S3_REGION: &str = "eu-central-1"; const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; @@ -161,10 +167,31 @@ pub struct Opt { #[clap(long, env = MEILI_ZK_URL)] pub zk_url: Option, - /// Sets the HTTP address and port used to communicate with the S3 bucket. + /// Sets the address and port used to communicate with the S3 bucket. #[clap(long, env = MEILI_S3_URL)] pub s3_url: Option, + /// Sets the region used to communicate with the s3 bucket. + #[clap(long, env = MEILI_S3_REGION, default_value_t = default_s3_region())] + #[serde(default = "default_s3_region")] + pub s3_region: String, + + /// Sets the S3 bucket name to use. + #[clap(long, env = MEILI_S3_BUCKET)] + pub s3_bucket: Option, + + /// Set the S3 access key. If used you must also set the secret key. + #[clap(long, env = MEILI_S3_ACCESS_KEY)] + pub s3_access_key: Option, + + /// Set the S3 secret key. If used you must also set the access key. + #[clap(long, env = MEILI_S3_SECRET_KEY)] + pub s3_secret_key: Option, + + /// Security token, can't be used with access key and secret key. + #[clap(long, env = MEILI_S3_SECURITY_TOKEN)] + pub s3_security_token: Option, + /// Deactivates Meilisearch's built-in telemetry when provided. /// /// Meilisearch automatically collects data from all instances that do not opt out using this flag. @@ -381,6 +408,11 @@ impl Opt { env, zk_url, s3_url, + s3_region, + s3_bucket, + s3_access_key, + s3_secret_key, + s3_security_token, max_index_size: _, max_task_db_size: _, http_payload_size_limit, @@ -420,6 +452,19 @@ impl Opt { if let Some(s3_url) = s3_url { export_to_env_if_not_present(MEILI_S3_URL, s3_url); } + export_to_env_if_not_present(MEILI_S3_REGION, s3_region); + if let Some(s3_bucket) = s3_bucket { + export_to_env_if_not_present(MEILI_S3_BUCKET, s3_bucket); + } + if let Some(s3_access_key) = s3_access_key { + export_to_env_if_not_present(MEILI_S3_ACCESS_KEY, s3_access_key); + } + if let Some(s3_secret_key) = s3_secret_key { + export_to_env_if_not_present(MEILI_S3_SECRET_KEY, s3_secret_key); + } + if let Some(s3_security_token) = s3_security_token { + export_to_env_if_not_present(MEILI_S3_SECURITY_TOKEN, s3_security_token); + } #[cfg(all(not(debug_assertions), feature = "analytics"))] { export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string()); @@ -734,8 +779,8 @@ fn default_env() -> String { DEFAULT_ENV.to_string() } -pub fn default_zk_url() -> String { - DEFAULT_HTTP_ADDR.to_string() +fn default_s3_region() -> String { + DEFAULT_S3_REGION.to_string() } fn default_max_index_size() -> Byte {