From 49a6d2d5f13776dce5d7ed3398f5d4511dde1078 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Wed, 25 Aug 2021 12:23:02 +0200 Subject: [PATCH 1/2] run all benchmarks once every friday --- .github/workflows/benchmarks.yml | 8 +-- .../workflows/cron_benchmarks_indexing.yml | 70 +++++++++++++++++++ .../cron_benchmarks_search_songs.yml | 70 +++++++++++++++++++ .../workflows/cron_benchmarks_search_wiki.yml | 70 +++++++++++++++++++ 4 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/cron_benchmarks_indexing.yml create mode 100644 .github/workflows/cron_benchmarks_search_songs.yml create mode 100644 .github/workflows/cron_benchmarks_search_wiki.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c1475e281..c64c6a64b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -9,7 +9,7 @@ on: default: 'search_songs' env: - HOME: "/home/runner" # The actions-rs/toolchain@v1 can fail we have no $HOME defined + BENCH_NAME: ${{ github.event.inputs.dataset_name }} jobs: benchmarks: @@ -38,14 +38,14 @@ jobs: id: commit_sha - name: Set file basename with format "dataset_branch_commitSHA" shell: bash - run: echo "##[set-output name=basename;]$(echo ${{ github.event.inputs.dataset_name }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" + run: echo "##[set-output name=basename;]$(echo ${BENCH_NAME}${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" id: file # Run benchmarks - - name: Run benchmarks - Dataset ${{ github.event.inputs.dataset_name }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} + - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | cd benchmarks - cargo bench --bench ${{ github.event.inputs.dataset_name }} -- --save-baseline ${{ steps.file.outputs.basename }} + cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files - name: Install critcmp diff --git a/.github/workflows/cron_benchmarks_indexing.yml b/.github/workflows/cron_benchmarks_indexing.yml new file mode 100644 index 000000000..8b7446e8d --- /dev/null +++ b/.github/workflows/cron_benchmarks_indexing.yml @@ -0,0 +1,70 @@ +name: Benchmarks indexing + +on: + schedule: + - cron: "30 0 * * FRI" # every friday at 00:30 + +env: + BENCH_NAME: "indexing" + +jobs: + benchmarks: + name: Run and upload benchmarks + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Set variables + - name: Set current branch name + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/})" + id: current_branch + - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3 + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" + id: normalized_current_branch + - name: Set shorter commit SHA + shell: bash + run: echo "##[set-output name=short;]$(echo $GITHUB_SHA | cut -c1-8)" + id: commit_sha + - name: Set file basename with format "dataset_branch_commitSHA" + shell: bash + run: echo "##[set-output name=basename;]$(echo ${BENCH_NAME}${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" + id: file + + # Run benchmarks + - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} + run: | + cd benchmarks + cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} + + # Generate critcmp files + - name: Install critcmp + run: cargo install critcmp + - name: Export cripcmp file + run: | + critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json + + # Upload benchmarks + - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3 + uses: BetaHuhn/do-spaces-action@v2 + with: + access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }} + secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }} + space_name: ${{ secrets.DO_SPACES_SPACE_NAME }} + space_region: ${{ secrets.DO_SPACES_SPACE_REGION }} + source: ${{ steps.file.outputs.basename }}.json + out_dir: critcmp_results + + # Helper + - name: 'README: compare with another benchmark' + run: | + echo "${{ steps.file.outputs.basename }}.json has just been pushed." + echo 'How to compare this benchmark with another one?' + echo ' - Check the available files with: ./benchmarks/scripts/list.sh' + echo " - Run the following command: ./benchmaks/scipts/compare.sh ${{ steps.file.outputs.basename }}.json" diff --git a/.github/workflows/cron_benchmarks_search_songs.yml b/.github/workflows/cron_benchmarks_search_songs.yml new file mode 100644 index 000000000..67c4c0d6d --- /dev/null +++ b/.github/workflows/cron_benchmarks_search_songs.yml @@ -0,0 +1,70 @@ +name: Benchmarks search songs + +on: + schedule: + - cron: "30 08 * * FRI" # every friday at 08:30 + +env: + BENCH_NAME: "search_songs" + +jobs: + benchmarks: + name: Run and upload benchmarks + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Set variables + - name: Set current branch name + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/})" + id: current_branch + - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3 + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" + id: normalized_current_branch + - name: Set shorter commit SHA + shell: bash + run: echo "##[set-output name=short;]$(echo $GITHUB_SHA | cut -c1-8)" + id: commit_sha + - name: Set file basename with format "dataset_branch_commitSHA" + shell: bash + run: echo "##[set-output name=basename;]$(echo ${BENCH_NAME}${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" + id: file + + # Run benchmarks + - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} + run: | + cd benchmarks + cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} + + # Generate critcmp files + - name: Install critcmp + run: cargo install critcmp + - name: Export cripcmp file + run: | + critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json + + # Upload benchmarks + - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3 + uses: BetaHuhn/do-spaces-action@v2 + with: + access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }} + secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }} + space_name: ${{ secrets.DO_SPACES_SPACE_NAME }} + space_region: ${{ secrets.DO_SPACES_SPACE_REGION }} + source: ${{ steps.file.outputs.basename }}.json + out_dir: critcmp_results + + # Helper + - name: 'README: compare with another benchmark' + run: | + echo "${{ steps.file.outputs.basename }}.json has just been pushed." + echo 'How to compare this benchmark with another one?' + echo ' - Check the available files with: ./benchmarks/scripts/list.sh' + echo " - Run the following command: ./benchmaks/scipts/compare.sh ${{ steps.file.outputs.basename }}.json" diff --git a/.github/workflows/cron_benchmarks_search_wiki.yml b/.github/workflows/cron_benchmarks_search_wiki.yml new file mode 100644 index 000000000..3cad5182d --- /dev/null +++ b/.github/workflows/cron_benchmarks_search_wiki.yml @@ -0,0 +1,70 @@ +name: Benchmarks search wikipedia articles + +on: + schedule: + - cron: "30 16 * * FRI" # every friday at 16:30 (it’s snacky snack-time!) + +env: + BENCH_NAME: "search_wiki" + +jobs: + benchmarks: + name: Run and upload benchmarks + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Set variables + - name: Set current branch name + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/})" + id: current_branch + - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3 + shell: bash + run: echo "##[set-output name=name;]$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" + id: normalized_current_branch + - name: Set shorter commit SHA + shell: bash + run: echo "##[set-output name=short;]$(echo $GITHUB_SHA | cut -c1-8)" + id: commit_sha + - name: Set file basename with format "dataset_branch_commitSHA" + shell: bash + run: echo "##[set-output name=basename;]$(echo ${BENCH_NAME}${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" + id: file + + # Run benchmarks + - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} + run: | + cd benchmarks + cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} + + # Generate critcmp files + - name: Install critcmp + run: cargo install critcmp + - name: Export cripcmp file + run: | + critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json + + # Upload benchmarks + - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3 + uses: BetaHuhn/do-spaces-action@v2 + with: + access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }} + secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }} + space_name: ${{ secrets.DO_SPACES_SPACE_NAME }} + space_region: ${{ secrets.DO_SPACES_SPACE_REGION }} + source: ${{ steps.file.outputs.basename }}.json + out_dir: critcmp_results + + # Helper + - name: 'README: compare with another benchmark' + run: | + echo "${{ steps.file.outputs.basename }}.json has just been pushed." + echo 'How to compare this benchmark with another one?' + echo ' - Check the available files with: ./benchmarks/scripts/list.sh' + echo " - Run the following command: ./benchmaks/scipts/compare.sh ${{ steps.file.outputs.basename }}.json" From 5e639bc0c120045b40266626eb3665f3c22a6269 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 30 Aug 2021 13:49:38 +0200 Subject: [PATCH 2/2] postfix all action name with (cron) --- .github/workflows/cron_benchmarks_indexing.yml | 2 +- .github/workflows/cron_benchmarks_search_songs.yml | 2 +- .github/workflows/cron_benchmarks_search_wiki.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cron_benchmarks_indexing.yml b/.github/workflows/cron_benchmarks_indexing.yml index 8b7446e8d..e806edc84 100644 --- a/.github/workflows/cron_benchmarks_indexing.yml +++ b/.github/workflows/cron_benchmarks_indexing.yml @@ -1,4 +1,4 @@ -name: Benchmarks indexing +name: Benchmarks indexing (cron) on: schedule: diff --git a/.github/workflows/cron_benchmarks_search_songs.yml b/.github/workflows/cron_benchmarks_search_songs.yml index 67c4c0d6d..018c20817 100644 --- a/.github/workflows/cron_benchmarks_search_songs.yml +++ b/.github/workflows/cron_benchmarks_search_songs.yml @@ -1,4 +1,4 @@ -name: Benchmarks search songs +name: Benchmarks search songs (cron) on: schedule: diff --git a/.github/workflows/cron_benchmarks_search_wiki.yml b/.github/workflows/cron_benchmarks_search_wiki.yml index 3cad5182d..78f940e38 100644 --- a/.github/workflows/cron_benchmarks_search_wiki.yml +++ b/.github/workflows/cron_benchmarks_search_wiki.yml @@ -1,4 +1,4 @@ -name: Benchmarks search wikipedia articles +name: Benchmarks search wikipedia articles (cron) on: schedule: