Merge pull request #75 from GitHubSecurityLab/improveci

Always run tests.

Author: michaelnebel

.github/scripts/pr-tests.sh

@@ -55,9 +55,61 @@ jobs:
       - name: Test Queries
         if: steps.changes.outputs.src == 'true'
         env:
-          GITHUB_TOKEN: ${{ github.token }}
+          RUNNER_TEMP: ${{ runner.temp }}
+        shell: python
         run: |
-          ./.github/scripts/pr-tests.sh ${{ github.event.number }} ${{ matrix.language }}
+          import os
+          import sys
+          import subprocess
+          from pathlib import Path
+
+          def print_error(fmt, *args):
+            print(f"::error::{fmt}", *args)
+
+          def print_error_and_fail(fmt, *args):
+            print_error(fmt, args)
+            sys.exit(1)
+
+          runner_temp = os.environ['RUNNER_TEMP']
+
+          test_root = Path('${{ github.workspace }}', '${{ matrix.language }}', 'test')
+          print(f"Executing tests found (recursively) in the directory '{test_root}'")
+          files_to_close = []
+          try:
+            # Runners have 4 cores, so split the tests into 4 "slices", and run one per thread
+            num_slices = 4
+            procs = []
+
+            for slice in range(1, num_slices+1):
+              test_report_path = os.path.join(runner_temp, "${{ matrix.language }}", f"test_report_slice_{slice}_of_{num_slices}.json")
+              os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
+              test_report_file = open(test_report_path, 'w')
+              files_to_close.append(test_report_file)
+              procs.append(subprocess.Popen(["codeql", "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", test_root], stdout=test_report_file, stderr=subprocess.PIPE))
+
+            for p in procs:
+              _, err = p.communicate()
+              if p.returncode != 0:
+                if p.returncode == 122:
+                  # Failed because a test case failed, so just print the regular output.
+                  # This will allow us to proceed to validate-test-results, which will fail if
+                  # any test cases failed
+                  print(f"{err.decode()}")
+                else:
+                  # Some more serious problem occurred, so print and fail fast
+                  print_error_and_fail(f"Failed to run tests with return code {p.returncode}\n{err.decode()}")
+          finally:
+            for file in files_to_close:
+              file.close()
+
+      - name: Upload test results
+        if: steps.changes.outputs.src == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.language }}-test-results
+          path: |
+            ${{ runner.temp }}/${{ matrix.language }}/test_report_slice_*.json
+          if-no-files-found: error
 
       - name: Compile / Check Suites & Packs
         if: steps.changes.outputs.src == 'true'
@@ -66,6 +118,39 @@ jobs:
         run: |
           ./.github/scripts/pr-suites-packs.sh ${{ github.event.number }} ${{ matrix.language }}
 
+  validate-test-results:
+    name: Validate test results
+    needs: compile-and-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if compile-and-test job failed to complete, if so fail
+        if: ${{ needs.compile-and-test.result == 'failure' }}
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Test run job failed')
+
+      - name: Collect test results
+        uses: actions/download-artifact@v4
+
+      - name: Validate test results
+        run: |
+          if [[ ! -n "$(find . -name 'test_report_*' -print -quit)" ]]; then
+            echo "No test results found"
+            exit 0
+          fi
+
+          for json_report in *-test-results/test_report_*
+          do
+            jq --raw-output '"PASS \(map(select(.pass == true)) | length)/\(length)'" $json_report\"" "$json_report"
+          done
+          FAILING_TESTS=$(jq --raw-output '.[] | select(.pass == false)' *-test-results/test_report_*.json)
+          if [[ ! -z "$FAILING_TESTS" ]]; then
+            echo "ERROR: The following tests failed:"
+            echo $FAILING_TESTS | jq .
+            exit 1
+          fi
+  
   extensions:
     runs-on: ubuntu-latest