searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

retry downloading the benchmarks datasets

Browse Source
This commit is contained in:
Irevoire 2022-08-17 19:25:05 +02:00
parent 087da5621a
commit 84a784834e
No known key found for this signature in database
GPG Key ID: 7A6A970C96104F1B
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
benchmarks/build.rs
View File

@ -80,7 +80,7 @@ fn main() -> anyhow::Result<()> {
} }
let url = format!("{}/{}.{}.gz", BASE_URL, dataset, extension); let url = format!("{}/{}.{}.gz", BASE_URL, dataset, extension);
eprintln!("downloading: {}", url); eprintln!("downloading: {}", url);
let bytes = download_dataset(url.clone())?; let bytes = retry(|| download_dataset(url.clone()), 10)?;
eprintln!("{} downloaded successfully", url); eprintln!("{} downloaded successfully", url);
eprintln!("uncompressing in {}", out_file.display()); eprintln!("uncompressing in {}", out_file.display());
uncompress_in_file(bytes, &out_file)?; uncompress_in_file(bytes, &out_file)?;
@ -89,6 +89,15 @@ fn main() -> anyhow::Result<()> {
Ok(()) Ok(())
} }
fn retry<Ok, Err>(fun: impl Fn() -> Result<Ok, Err>, times: usize) -> Result<Ok, Err> {
for _ in 0..times {
if let ok @ Ok(_) = fun() {
return ok;
}
}
fun()
}
fn download_dataset<U: IntoUrl>(url: U) -> anyhow::Result<Cursor<Bytes>> { fn download_dataset<U: IntoUrl>(url: U) -> anyhow::Result<Cursor<Bytes>> {
let bytes = let bytes =
reqwest::blocking::Client::builder().timeout(None).build()?.get(url).send()?.bytes()?; reqwest::blocking::Client::builder().timeout(None).build()?.get(url).send()?.bytes()?;