searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-04 10:35:46 +01:00
Code Issues Releases Wiki Activity

exposes all the s3 arguments

Browse Source
This commit is contained in:
Tamo 2023-09-13 18:17:56 +02:00
parent 8a2e8a887f
commit ecd36b15f0
4 changed files with 70 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
index-scheduler/src/insta_snapshot.rs
View File

@ -41,6 +41,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
planned_failures: _,
run_loop_iteration: _,
zookeeper: _,
options: _,
} = inner.deref();
let rtxn = env.read_txn().unwrap();

13
index-scheduler/src/lib.rs
View File

@ -468,7 +468,12 @@ impl IndexScheduler {
let s3 = inner.options.s3.as_ref().unwrap();
let task =
s3.get_object(format!("/tasks/{id:0>10}")).unwrap();
assert_eq!(
task.status_code(),
200,
"could not reach the s3: {:?}",
task.as_str()
);
let task = serde_json::from_slice(task.as_slice()).unwrap();
inner.register_raw_task(&mut wtxn, &task).unwrap();
// we received a new tasks, we must wake up
@ -507,6 +512,12 @@ impl IndexScheduler {
.unwrap();
let s3 = inner.options.s3.as_ref().unwrap();
let task = s3.get_object(format!("tasks/{id:0>10}")).unwrap();
assert_eq!(
task.status_code(),
200,
"could not reach the s3: {:?}",
task.as_str()
);
let task = serde_json::from_slice(task.as_slice()).unwrap();
inner.register_raw_task(&mut wtxn, &task).unwrap();
wtxn.commit().unwrap();

12
meilisearch/src/lib.rs
View File

@ -247,9 +247,15 @@ fn open_or_create_database_unchecked(
s3: opt.s3_url.as_ref().map(|url| {
Arc::new(
Bucket::new(
"test-rust-s3",
Region::Custom { region: "eu-central-1".to_owned(), endpoint: url.clone() },
Credentials::default().unwrap(),
opt.s3_bucket.as_deref().unwrap(),
Region::Custom { region: opt.s3_region.clone(), endpoint: url.clone() },
Credentials {
access_key: opt.s3_access_key.clone(),
secret_key: opt.s3_secret_key.clone(),
security_token: opt.s3_security_token.clone(),
session_token: None,
expiration: None,
},
)
.unwrap()
.with_path_style(),

51
meilisearch/src/option.rs
View File

@ -30,6 +30,11 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
const MEILI_ENV: &str = "MEILI_ENV";
const MEILI_ZK_URL: &str = "MEILI_ZK_URL";
const MEILI_S3_URL: &str = "MEILI_S3_URL";
const MEILI_S3_BUCKET: &str = "MEILI_S3_BUCKET";
const MEILI_S3_ACCESS_KEY: &str = "MEILI_S3_ACCESS_KEY";
const MEILI_S3_SECRET_KEY: &str = "MEILI_S3_SECRET_KEY";
const MEILI_S3_SECURITY_TOKEN: &str = "MEILI_S3_SECURITY_TOKEN";
const MEILI_S3_REGION: &str = "MEILI_S3_REGION";
#[cfg(all(not(debug_assertions), feature = "analytics"))]
const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
@ -58,6 +63,7 @@ const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
const DEFAULT_DB_PATH: &str = "./data.ms";
const DEFAULT_HTTP_ADDR: &str = "localhost:7700";
const DEFAULT_ENV: &str = "development";
const DEFAULT_S3_REGION: &str = "eu-central-1";
const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB";
const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/";
const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400;
@ -161,10 +167,31 @@ pub struct Opt {
#[clap(long, env = MEILI_ZK_URL)]
pub zk_url: Option<String>,
/// Sets the HTTP address and port used to communicate with the S3 bucket.
/// Sets the address and port used to communicate with the S3 bucket.
#[clap(long, env = MEILI_S3_URL)]
pub s3_url: Option<String>,
/// Sets the region used to communicate with the s3 bucket.
#[clap(long, env = MEILI_S3_REGION, default_value_t = default_s3_region())]
#[serde(default = "default_s3_region")]
pub s3_region: String,
/// Sets the S3 bucket name to use.
#[clap(long, env = MEILI_S3_BUCKET)]
pub s3_bucket: Option<String>,
/// Set the S3 access key. If used you must also set the secret key.
#[clap(long, env = MEILI_S3_ACCESS_KEY)]
pub s3_access_key: Option<String>,
/// Set the S3 secret key. If used you must also set the access key.
#[clap(long, env = MEILI_S3_SECRET_KEY)]
pub s3_secret_key: Option<String>,
/// Security token, can't be used with access key and secret key.
#[clap(long, env = MEILI_S3_SECURITY_TOKEN)]
pub s3_security_token: Option<String>,
/// Deactivates Meilisearch's built-in telemetry when provided.
///
/// Meilisearch automatically collects data from all instances that do not opt out using this flag.
@ -381,6 +408,11 @@ impl Opt {
env,
zk_url,
s3_url,
s3_region,
s3_bucket,
s3_access_key,
s3_secret_key,
s3_security_token,
max_index_size: _,
max_task_db_size: _,
http_payload_size_limit,
@ -420,6 +452,19 @@ impl Opt {
if let Some(s3_url) = s3_url {
export_to_env_if_not_present(MEILI_S3_URL, s3_url);
}
export_to_env_if_not_present(MEILI_S3_REGION, s3_region);
if let Some(s3_bucket) = s3_bucket {
export_to_env_if_not_present(MEILI_S3_BUCKET, s3_bucket);
}
if let Some(s3_access_key) = s3_access_key {
export_to_env_if_not_present(MEILI_S3_ACCESS_KEY, s3_access_key);
}
if let Some(s3_secret_key) = s3_secret_key {
export_to_env_if_not_present(MEILI_S3_SECRET_KEY, s3_secret_key);
}
if let Some(s3_security_token) = s3_security_token {
export_to_env_if_not_present(MEILI_S3_SECURITY_TOKEN, s3_security_token);
}
#[cfg(all(not(debug_assertions), feature = "analytics"))]
{
export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
@ -734,8 +779,8 @@ fn default_env() -> String {
DEFAULT_ENV.to_string()
}
pub fn default_zk_url() -> String {
DEFAULT_HTTP_ADDR.to_string()
fn default_s3_region() -> String {
DEFAULT_S3_REGION.to_string()
}
fn default_max_index_size() -> Byte {