Change benchmark outputs

- logs to stderr instead of stdout
- prints links to the dashboard when there is a dashboard
This commit is contained in:
Louis Dureuil 2024-05-23 15:29:06 +02:00
parent abe29772db
commit e340705634
No known key found for this signature in database
3 changed files with 76 additions and 15 deletions

View File

@ -55,6 +55,10 @@ impl Client {
pub fn delete(&self, route: &str) -> reqwest::RequestBuilder {
self.request(reqwest::Method::DELETE, route)
}
pub fn base_url(&self) -> Option<&str> {
self.base_url.as_deref()
}
}
#[derive(Debug, Clone, Copy, Deserialize)]

View File

@ -18,12 +18,9 @@ pub enum DashboardClient {
}
impl DashboardClient {
pub fn new(dashboard_url: &str, api_key: Option<&str>) -> anyhow::Result<Self> {
let dashboard_client = Client::new(
Some(format!("{}/api/v1", dashboard_url)),
api_key,
Some(std::time::Duration::from_secs(60)),
)?;
pub fn new(dashboard_url: String, api_key: Option<&str>) -> anyhow::Result<Self> {
let dashboard_client =
Client::new(Some(dashboard_url), api_key, Some(std::time::Duration::from_secs(60)))?;
Ok(Self::Client(dashboard_client))
}
@ -36,7 +33,7 @@ impl DashboardClient {
let Self::Client(dashboard_client) = self else { return Ok(()) };
let response = dashboard_client
.put("machine")
.put("/api/v1/machine")
.json(&json!({"hostname": env.hostname}))
.send()
.await
@ -62,7 +59,7 @@ impl DashboardClient {
let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) };
let response = dashboard_client
.put("invocation")
.put("/api/v1/invocation")
.json(&json!({
"commit": {
"sha1": build_info.commit_sha1,
@ -97,7 +94,7 @@ impl DashboardClient {
let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) };
let response = dashboard_client
.put("workload")
.put("/api/v1/workload")
.json(&json!({
"invocation_uuid": invocation_uuid,
"name": &workload.name,
@ -124,7 +121,7 @@ impl DashboardClient {
let Self::Client(dashboard_client) = self else { return Ok(()) };
let response = dashboard_client
.put("run")
.put("/api/v1/run")
.json(&json!({
"workload_uuid": workload_uuid,
"data": report
@ -159,7 +156,7 @@ impl DashboardClient {
pub async fn mark_as_failed(&self, invocation_uuid: Uuid, failure_reason: Option<String>) {
if let DashboardClient::Client(client) = self {
let response = client
.post("cancel-invocation")
.post("/api/v1/cancel-invocation")
.json(&json!({
"invocation_uuid": invocation_uuid,
"failure_reason": failure_reason,
@ -186,4 +183,28 @@ impl DashboardClient {
tracing::warn!(%invocation_uuid, "marked invocation as failed or canceled");
}
/// Result URL in markdown
pub(crate) fn result_url(
&self,
workload_name: &str,
build_info: &build_info::BuildInfo,
baseline_branch: &str,
) -> String {
let Self::Client(client) = self else { return Default::default() };
let Some(base_url) = client.base_url() else { return Default::default() };
let Some(commit_sha1) = build_info.commit_sha1 else { return Default::default() };
// https://bench.meilisearch.dev/view_spans?commit_sha1=500ddc76b549fb9f1af54b2dd6abfa15960381bb&workload_name=settings-add-remove-filters.json&target_branch=reduce-transform-disk-usage&baseline_branch=main
let mut url = format!(
"{base_url}/view_spans?commit_sha1={commit_sha1}&workload_name={workload_name}"
);
if let Some(target_branch) = build_info.branch {
url += &format!("&target_branch={target_branch}&baseline_branch={baseline_branch}");
}
format!("[{workload_name} compared with {baseline_branch}]({url})")
}
}

View File

@ -6,6 +6,7 @@ mod env_info;
mod meili_process;
mod workload;
use std::io::LineWriter;
use std::path::PathBuf;
use anyhow::Context;
@ -90,6 +91,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(|| LineWriter::new(std::io::stderr()))
.with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
.with_filter(filter),
);
@ -110,7 +112,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
let dashboard_client = if args.no_dashboard {
dashboard::DashboardClient::new_dry()
} else {
dashboard::DashboardClient::new(&args.dashboard_url, args.api_key.as_deref())?
dashboard::DashboardClient::new(args.dashboard_url.clone(), args.api_key.as_deref())?
};
// reporting uses its own client because keeping the stream open to wait for entries
@ -136,7 +138,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
let commit_message = build_info.commit_msg.context("missing commit message")?.split('\n').next().unwrap();
let max_workloads = args.workload_file.len();
let reason: Option<&str> = args.reason.as_deref();
let invocation_uuid = dashboard_client.create_invocation( build_info, commit_message, env, max_workloads, reason).await?;
let invocation_uuid = dashboard_client.create_invocation(build_info.clone(), commit_message, env, max_workloads, reason).await?;
tracing::info!(workload_count = args.workload_file.len(), "handling workload files");
@ -144,6 +146,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
let workload_runs = tokio::spawn(
{
let dashboard_client = dashboard_client.clone();
let mut dashboard_urls = Vec::new();
async move {
for workload_file in args.workload_file.iter() {
let workload: Workload = serde_json::from_reader(
@ -152,6 +155,8 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
)
.with_context(|| format!("error parsing {} as JSON", workload_file.display()))?;
let workload_name = workload.name.clone();
workload::execute(
&assets_client,
&dashboard_client,
@ -163,8 +168,23 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
&args,
)
.await?;
let result_url = dashboard_client.result_url(&workload_name, &build_info, "main");
if !result_url.is_empty() {
dashboard_urls.push(result_url);
}
Ok::<(), anyhow::Error>(())
if let Some(branch) = build_info.branch {
let result_url = dashboard_client.result_url(&workload_name, &build_info, branch);
if !result_url.is_empty() {
dashboard_urls.push(result_url);
}
}
}
Ok::<_, anyhow::Error>(dashboard_urls)
}});
// handle ctrl-c
@ -176,13 +196,19 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
// wait for the end of the main task, handle result
match workload_runs.await {
Ok(Ok(_)) => {
Ok(Ok(urls)) => {
tracing::info!("Success");
println!("☀️ Benchmark invocation completed, please find the results for your workloads below:");
for url in urls {
println!("- {url}");
}
Ok::<(), anyhow::Error>(())
}
Ok(Err(error)) => {
tracing::error!(%invocation_uuid, error = %error, "invocation failed, attempting to report the failure to dashboard");
dashboard_client.mark_as_failed(invocation_uuid, Some(error.to_string())).await;
println!("☔️ Benchmark invocation failed...");
println!("{error}");
tracing::warn!(%invocation_uuid, "invocation marked as failed following error");
Err(error)
},
@ -191,10 +217,20 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
Ok(panic) => {
tracing::error!("invocation panicked, attempting to report the failure to dashboard");
dashboard_client.mark_as_failed( invocation_uuid, Some("Panicked".into())).await;
println!("‼️ Benchmark invocation panicked 😱");
let msg = match panic.downcast_ref::<&'static str>() {
Some(s) => *s,
None => match panic.downcast_ref::<String>() {
Some(s) => &s[..],
None => "Box<dyn Any>",
},
};
println!("panicked at {msg}");
std::panic::resume_unwind(panic)
}
Err(_) => {
tracing::warn!("task was canceled");
println!("🚫 Benchmark invocation was canceled");
Ok(())
}
}