[libc++][CI] Also restart failed jobs when they fail for a spurious reason

Since we moved to a Docker-in-Docker setup, CI jobs sometimes fail due
to the Docker VM dying with 'context cancelled' errors. This is currently
not recognized as a spurious failure, which leads to the job not being
automatically restarted. This patch fixes that.

This commit only adds a test job with the new logic since this workflow
triggers on workflow_run, which means that the changes need to be on
`main` before they can be tested. Once this is tested to work properly,
I'll make it the default definition for the workflow.

Author: ldionne

.github/workflows/libcxx-restart-preempted-jobs.yaml

@@ -130,3 +130,91 @@ jobs:
                 run_id: context.payload.workflow_run.id
               })
             await create_check_run('success', 'Restarted workflow run due to preempted job')
+
+  restart-test:
+    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled') && github.event.actor.login == 'ldionne' # TESTING ONLY
+    name: "Restart Job"
+    permissions:
+      statuses: read
+      checks: write
+      actions: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Restart Job"
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+        with:
+          script: |
+            const FAILURE_REGEX = /Process completed with exit code 1./
+            const PREEMPTION_REGEX = /The runner has received a shutdown signal|The operation was canceled/
+
+            function log(msg) {
+              core.notice(msg)
+            }
+
+            const wf_run = context.payload.workflow_run
+            log(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
+
+            log('Listing check runs for suite')
+            const check_suites = await github.rest.checks.listForSuite({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              check_suite_id: context.payload.workflow_run.check_suite_id,
+              per_page: 100 // FIXME: We don't have 100 check runs yet, but we should handle this better.
+            })
+
+            preemptions = [];
+            legitimate_failures = [];
+            for (check_run of check_suites.data.check_runs) {
+              log(`Checking check run: ${check_run.id}`);
+              if (check_run.status != 'completed') {
+                log('Check run was not completed. Skipping.');
+                continue;
+              }
+
+              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
+                log(`Check run had conclusion: ${check_run.conclusion}. Skipping.`);
+                continue;
+              }
+
+              annotations = await github.rest.checks.listAnnotations({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                check_run_id: check_run.id
+              })
+
+              preemption_annotation = annotations.data.find(function(annotation) {
+                return annotation.annotation_level == 'failure' &&
+                       annotation.message.match(PREEMPTION_REGEX) != null;
+              });
+              if (preemption_annotation != null) {
+                log(`Found preemption message: ${preemption_annotation.message}`);
+                preemptions.push(check_run);
+                break;
+              }
+
+              failure_annotation = annotations.data.find(function(annotation) {
+                return annotation.annotation_level == 'failure' &&
+                       annotation.message.match(FAILURE_REGEX) != null;
+              });
+              if (failure_annotation != null) {
+                log(`Found legitimate failure annotation: ${failure_annotation.message}`);
+                legitimate_failures.push(check_run);
+                break;
+              }
+            }
+
+            if (preemptions) {
+              log('Found some preempted jobs');
+              if (legitimate_failures) {
+                log('Also found some legitimate failures, so not restarting the workflow.');
+              } else {
+                log('Did not find any legitimate failures. Restarting workflow.');
+                await github.rest.actions.reRunWorkflowFailedJobs({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  run_id: context.payload.workflow_run.id
+                })
+              }
+            } else {
+              log('Did not find any preempted jobs. Not restarting the workflow.');
+            }