diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 418a23717..36af79460 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -43,4 +43,11 @@ jobs: - name: Run benchmarks on PR ${{ github.event.issue.id }} run: | - cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" -- ${{ steps.command.outputs.command-arguments }} \ No newline at end of file + cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" \ + --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" \ + --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" \ + -- ${{ steps.command.outputs.command-arguments }} > benchlinks.txt + + - name: Send comment in PR + run: | + gh pr comment ${{github.event.issue.number}} --body-file benchlinks.txt diff --git a/xtask/src/bench/client.rs b/xtask/src/bench/client.rs index 3e46615cc..1c2b743af 100644 --- a/xtask/src/bench/client.rs +++ b/xtask/src/bench/client.rs @@ -55,6 +55,10 @@ impl Client { pub fn delete(&self, route: &str) -> reqwest::RequestBuilder { self.request(reqwest::Method::DELETE, route) } + + pub fn base_url(&self) -> Option<&str> { + self.base_url.as_deref() + } } #[derive(Debug, Clone, Copy, Deserialize)] diff --git a/xtask/src/bench/dashboard.rs b/xtask/src/bench/dashboard.rs index 3ba0ca58b..67353f7bb 100644 --- a/xtask/src/bench/dashboard.rs +++ b/xtask/src/bench/dashboard.rs @@ -18,12 +18,9 @@ pub enum DashboardClient { } impl DashboardClient { - pub fn new(dashboard_url: &str, api_key: Option<&str>) -> anyhow::Result { - let dashboard_client = Client::new( - Some(format!("{}/api/v1", dashboard_url)), - api_key, - Some(std::time::Duration::from_secs(60)), - )?; + pub fn new(dashboard_url: String, api_key: Option<&str>) -> anyhow::Result { + let dashboard_client = + Client::new(Some(dashboard_url), api_key, Some(std::time::Duration::from_secs(60)))?; Ok(Self::Client(dashboard_client)) } @@ -36,7 +33,7 @@ impl DashboardClient { let Self::Client(dashboard_client) = self else { return Ok(()) }; let response = dashboard_client - .put("machine") + .put("/api/v1/machine") .json(&json!({"hostname": env.hostname})) .send() .await @@ -62,7 +59,7 @@ impl DashboardClient { let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) }; let response = dashboard_client - .put("invocation") + .put("/api/v1/invocation") .json(&json!({ "commit": { "sha1": build_info.commit_sha1, @@ -97,7 +94,7 @@ impl DashboardClient { let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) }; let response = dashboard_client - .put("workload") + .put("/api/v1/workload") .json(&json!({ "invocation_uuid": invocation_uuid, "name": &workload.name, @@ -124,7 +121,7 @@ impl DashboardClient { let Self::Client(dashboard_client) = self else { return Ok(()) }; let response = dashboard_client - .put("run") + .put("/api/v1/run") .json(&json!({ "workload_uuid": workload_uuid, "data": report @@ -159,7 +156,7 @@ impl DashboardClient { pub async fn mark_as_failed(&self, invocation_uuid: Uuid, failure_reason: Option) { if let DashboardClient::Client(client) = self { let response = client - .post("cancel-invocation") + .post("/api/v1/cancel-invocation") .json(&json!({ "invocation_uuid": invocation_uuid, "failure_reason": failure_reason, @@ -186,4 +183,28 @@ impl DashboardClient { tracing::warn!(%invocation_uuid, "marked invocation as failed or canceled"); } + + /// Result URL in markdown + pub(crate) fn result_url( + &self, + workload_name: &str, + build_info: &build_info::BuildInfo, + baseline_branch: &str, + ) -> String { + let Self::Client(client) = self else { return Default::default() }; + let Some(base_url) = client.base_url() else { return Default::default() }; + + let Some(commit_sha1) = build_info.commit_sha1 else { return Default::default() }; + + // https://bench.meilisearch.dev/view_spans?commit_sha1=500ddc76b549fb9f1af54b2dd6abfa15960381bb&workload_name=settings-add-remove-filters.json&target_branch=reduce-transform-disk-usage&baseline_branch=main + let mut url = format!( + "{base_url}/view_spans?commit_sha1={commit_sha1}&workload_name={workload_name}" + ); + + if let Some(target_branch) = build_info.branch { + url += &format!("&target_branch={target_branch}&baseline_branch={baseline_branch}"); + } + + format!("[{workload_name} compared with {baseline_branch}]({url})") + } } diff --git a/xtask/src/bench/mod.rs b/xtask/src/bench/mod.rs index 844b64f63..fdb2c4963 100644 --- a/xtask/src/bench/mod.rs +++ b/xtask/src/bench/mod.rs @@ -6,6 +6,7 @@ mod env_info; mod meili_process; mod workload; +use std::io::LineWriter; use std::path::PathBuf; use anyhow::Context; @@ -90,6 +91,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { let subscriber = tracing_subscriber::registry().with( tracing_subscriber::fmt::layer() + .with_writer(|| LineWriter::new(std::io::stderr())) .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) .with_filter(filter), ); @@ -110,7 +112,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { let dashboard_client = if args.no_dashboard { dashboard::DashboardClient::new_dry() } else { - dashboard::DashboardClient::new(&args.dashboard_url, args.api_key.as_deref())? + dashboard::DashboardClient::new(args.dashboard_url.clone(), args.api_key.as_deref())? }; // reporting uses its own client because keeping the stream open to wait for entries @@ -136,7 +138,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { let commit_message = build_info.commit_msg.context("missing commit message")?.split('\n').next().unwrap(); let max_workloads = args.workload_file.len(); let reason: Option<&str> = args.reason.as_deref(); - let invocation_uuid = dashboard_client.create_invocation( build_info, commit_message, env, max_workloads, reason).await?; + let invocation_uuid = dashboard_client.create_invocation(build_info.clone(), commit_message, env, max_workloads, reason).await?; tracing::info!(workload_count = args.workload_file.len(), "handling workload files"); @@ -144,6 +146,7 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { let workload_runs = tokio::spawn( { let dashboard_client = dashboard_client.clone(); + let mut dashboard_urls = Vec::new(); async move { for workload_file in args.workload_file.iter() { let workload: Workload = serde_json::from_reader( @@ -152,6 +155,8 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { ) .with_context(|| format!("error parsing {} as JSON", workload_file.display()))?; + let workload_name = workload.name.clone(); + workload::execute( &assets_client, &dashboard_client, @@ -163,8 +168,23 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { &args, ) .await?; + + let result_url = dashboard_client.result_url(&workload_name, &build_info, "main"); + + if !result_url.is_empty() { + dashboard_urls.push(result_url); + } + + if let Some(branch) = build_info.branch { + let result_url = dashboard_client.result_url(&workload_name, &build_info, branch); + + + if !result_url.is_empty() { + dashboard_urls.push(result_url); + } + } } - Ok::<(), anyhow::Error>(()) + Ok::<_, anyhow::Error>(dashboard_urls) }}); // handle ctrl-c @@ -176,13 +196,19 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { // wait for the end of the main task, handle result match workload_runs.await { - Ok(Ok(_)) => { + Ok(Ok(urls)) => { tracing::info!("Success"); + println!("☀️ Benchmark invocation completed, please find the results for your workloads below:"); + for url in urls { + println!("- {url}"); + } Ok::<(), anyhow::Error>(()) } Ok(Err(error)) => { tracing::error!(%invocation_uuid, error = %error, "invocation failed, attempting to report the failure to dashboard"); dashboard_client.mark_as_failed(invocation_uuid, Some(error.to_string())).await; + println!("☔️ Benchmark invocation failed..."); + println!("{error}"); tracing::warn!(%invocation_uuid, "invocation marked as failed following error"); Err(error) }, @@ -191,10 +217,20 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> { Ok(panic) => { tracing::error!("invocation panicked, attempting to report the failure to dashboard"); dashboard_client.mark_as_failed( invocation_uuid, Some("Panicked".into())).await; + println!("‼️ Benchmark invocation panicked 😱"); + let msg = match panic.downcast_ref::<&'static str>() { + Some(s) => *s, + None => match panic.downcast_ref::() { + Some(s) => &s[..], + None => "Box", + }, + }; + println!("panicked at {msg}"); std::panic::resume_unwind(panic) } Err(_) => { tracing::warn!("task was canceled"); + println!("🚫 Benchmark invocation was canceled"); Ok(()) } }