Rebase onto #134765 and address review comments

Created using spr 1.3.6-beta.1

Author: pcc

.ci/compute-projects.sh

@@ -194,6 +194,17 @@ def _compute_runtime_check_targets(runtimes_to_test: Set[str]) -> Set[str]:
 def _get_modified_projects(modified_files: list[str]) -> Set[str]:
     modified_projects = set()
     for modified_file in modified_files:
+        path_parts = pathlib.Path(modified_file).parts
+        # Exclude files in the docs directory. They do not impact an test
+        # targets and there is a separate workflow used for ensuring the
+        # documentation builds.
+        if len(path_parts) > 2 and path_parts[1] == "docs":
+            continue
+        # Exclude files for the gn build. We do not test it within premerge
+        # and changes occur often enough that they otherwise take up
+        # capacity.
+        if len(path_parts) > 3 and path_parts[:3] == ("llvm", "utils", "gn"):
+            continue
         modified_projects.add(pathlib.Path(modified_file).parts[0])
     return modified_projects
 

.ci/compute_projects.py

@@ -170,6 +170,24 @@ def test_exclude_runtiems_in_projects(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
+    def test_exclude_docs(self):
+        env_variables = compute_projects.get_env_variables(
+            ["llvm/docs/CIBestPractices.rst"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
+    def test_exclude_gn(self):
+        env_variables = compute_projects.get_env_variables(
+            ["llvm/utils/gn/build/BUILD.gn"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
 
 if __name__ == "__main__":
     unittest.main()

.ci/compute_projects_test.py

@@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
 echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
-. ./.ci/compute-projects.sh
-
 # Project specific pipelines.
 
 # If libc++ or one of the runtimes directories changed.
@@ -73,20 +71,16 @@ fi
 # needs while letting them run on the infrastructure provided by LLVM.
 
 # Figure out which projects need to be built on each platform
-all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-modified_projects="$(keep-modified-projects ${all_projects})"
-
-linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
+linux_projects=${projects_to_build}
+linux_check_targets=${project_check_targets}
+linux_runtimes=${runtimes_to_build}
+linux_runtime_check_targets=${runtimes_check_targets}
 
-linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
+windows_projects=${projects_to_build}
+windows_check_targets=${project_check_targets}
 
-windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then

.ci/generate-buildkite-pipeline-premerge

@@ -0,0 +1,57 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Script to generate a build report for buildkite."""
+
+import argparse
+import os
+import subprocess
+
+import generate_test_report_lib
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "title", help="Title of the test report, without Markdown formatting."
+    )
+    parser.add_argument("context", help="Annotation context to write to.")
+    parser.add_argument("return_code", help="The build's return code.", type=int)
+    parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+    args = parser.parse_args()
+
+    # All of these are required to build a link to download the log file.
+    env_var_names = [
+        "BUILDKITE_ORGANIZATION_SLUG",
+        "BUILDKITE_PIPELINE_SLUG",
+        "BUILDKITE_BUILD_NUMBER",
+        "BUILDKITE_JOB_ID",
+    ]
+    buildkite_info = {k: v for k, v in os.environ.items() if k in env_var_names}
+    if len(buildkite_info) != len(env_var_names):
+        buildkite_info = None
+
+    report, style = generate_test_report_lib.generate_report_from_files(
+        args.title, args.return_code, args.junit_files, buildkite_info
+    )
+
+    if report:
+        p = subprocess.Popen(
+            [
+                "buildkite-agent",
+                "annotate",
+                "--context",
+                args.context,
+                "--style",
+                style,
+            ],
+            stdin=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        # The report can be larger than the buffer for command arguments so we send
+        # it over stdin instead.
+        _, err = p.communicate(input=report)
+        if p.returncode:
+            raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}")

.ci/generate_test_report_buildkite.py

@@ -0,0 +1,23 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Script to generate a build report for Github."""
+
+import argparse
+
+import generate_test_report_lib
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "title", help="Title of the test report, without Markdown formatting."
+    )
+    parser.add_argument("return_code", help="The build's return code.", type=int)
+    parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+    args = parser.parse_args()
+
+    report, _ = generate_test_report_lib.generate_report_from_files(
+        args.title, args.return_code, args.junit_files, None
+    )
+
+    print(report)