Allows to trigger on labelling and clean-up the process (#400)

* Allows to trigger on comment and on labelling

* Remove previous runs and fix typo

* This should work?

* Godamm parenthesis!

* Update benchmark.yml

* Update benchmark.yml

* New try?

* Use created instead of opened

* Inplace comment

* Remove comment workflow

* Correct naming

* Missing the most important part

* Add the whole workflow from JET.jl

* Apply suggestions from code review

Co-authored-by: David Widmann <[email protected]>

* Update benchmark.yml

Co-authored-by: David Widmann <[email protected]>

Author: theogf

.github/workflows/benchmark-comment.yml

@@ -0,0 +1,62 @@
+# Credit to @aviastek, where this code comes from in JET.jl
+
+# To workaroud https://github.com/actions/first-interaction/issues/10 in a secure way,
+# we take the following steps to generate and comment a performance benchmark result:
+# 1. first "performance tracking" workflow will generate the benchmark results in an unprivileged environment triggered on `pull_request` event
+# 2. then this "performance tracking (comment)" workflow will show the result to us as a PR comment in a privileged environment
+# Note that this workflow can only be modifed by getting checked-in to the default branch
+# and thus is secure even though this workflow is granted with write permissions, etc.
+# xref: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+name: Post Benchmark Comments
+
+on:
+  workflow_run:
+    workflows:
+      - performance tracking
+    types:
+      - completed
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: >
+      ${{ github.event.workflow_run.event == 'pull_request' &&
+          github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v2
+      # restore records from the artifacts
+      - uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: benchmark.yml
+          name: Benchmarking
+          workflow_conclusion: success
+      - name: output benchmark result
+        id: output-result-markdown
+        run: |
+          echo ::set-output name=body::$(cat ./benchmark-result.artifact)
+      - name: output pull request number
+        id: output-pull-request-number
+        run: |
+          echo ::set-output name=body::$(cat ./pull-request-number.artifact)
+      # check if the previous comment exists
+      - name: find comment
+        uses: peter-evans/find-comment@v1
+        id: fc
+        with:
+          issue-number: ${{ steps.output-pull-request-number.outputs.body }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Kernel Benchmark Result
+      # create/update comment
+      - name: create comment
+        if: ${{ steps.fc.outputs.comment-id == 0 }}
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ steps.output-pull-request-number.outputs.body }}
+          body: ${{ steps.output-result-markdown.outputs.body }}
+      - name: update comment
+        if: ${{ steps.fc.outputs.comment-id != 0 }}
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: ${{ steps.output-result-markdown.outputs.body }}

.github/workflows/benchmark.yml

@@ -2,23 +2,52 @@ name: Run benchmarks
 
 on:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  Benchmark:
+  Benchmarking:
     runs-on: ubuntu-latest
     if: contains(github.event.pull_request.labels.*.name, 'performance critical')
-    env:
-      JULIA_DEBUG: BenchmarkCI
     steps:
+      # setup
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: 1.6
-      - name: Install dependencies
-        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
-      - name: Run benchmarks
-        run: julia -e "using BenchmarkCI; BenchmarkCI.judge()"
-      - name: Post results
-        run: julia -e "using BenchmarkCI; BenchmarkCI.postjudge()"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          version: '1.7'
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark [email protected]"'
+      # run the benchmark suite
+      - name: run benchmarks
+        run: |
+          using BenchmarkCI
+          BenchmarkCI.judge()
+          BenchmarkCI.displayjudgement()
+        shell: julia --color=yes {0}
+      # generate and record the benchmark result as markdown
+      - name: generate benchmark result
+        run: |
+          using BenchmarkCI
+          judgement = BenchmarkCI._loadjudge(BenchmarkCI.DEFAULT_WORKSPACE)
+          title = "Kernel Benchmark Result"
+          ciresult = BenchmarkCI.CIResult(; judgement, title)
+          comment = sprint() do io
+              return BenchmarkCI.printcommentmd(io, ciresult)
+          end
+          comment = replace(comment, "%" => "%25", "\\n" => "%0A", "\\r" => "%0D")
+          write("benchmark-result.artifact", comment)
+        shell: julia --color=yes {0}
+      # record the pull request number
+      - name: record pull request number
+        run: echo ${{ github.event.pull_request.number }} > ./pull-request-number.artifact
+      # save as artifacts (performance tracking (comment) workflow will use it)
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Benchmarking
+          path: ./*.artifact