Merge remote-tracking branch 'origin/main' into vplan-irphi

Author: fhahn

.ci/metrics/metrics.py

@@ -1,6 +1,8 @@
 import collections
 import datetime
+import dateutil
 import github
+import json
 import logging
 import os
 import requests
@@ -41,7 +43,7 @@
 # This means we essentially have a list of workflows sorted by creation date,
 # and that's all we can deduce from it. So for each iteration, we'll blindly
 # process the last N workflows.
-GITHUB_WORKFLOWS_MAX_PROCESS_COUNT = 1000
+GITHUB_WORKFLOWS_MAX_PROCESS_COUNT = 2000
 # Second reason for the cut: reaching a workflow older than X.
 # This means we will miss long-tails (exceptional jobs running for more than
 # X hours), but that's also the case with the count cutoff above.
@@ -53,6 +55,18 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
+# Lists the BuildKite jobs we want to track. Maps the BuildKite job name to
+# the metric name in Grafana. This is important not to lose metrics history
+# if the workflow name changes.
+BUILDKITE_WORKFLOW_TO_TRACK = {
+    ":linux: Linux x64": "buildkite_linux",
+    ":windows: Windows x64": "buildkite_windows",
+}
+
+# Number of builds to fetch per page. Since we scrape regularly, this can
+# remain small.
+BUILDKITE_GRAPHQL_BUILDS_PER_PAGE = 50
+
 @dataclass
 class JobMetrics:
     job_name: str
@@ -70,6 +84,180 @@ class GaugeMetric:
     time_ns: int
 
 
+def buildkite_fetch_page_build_list(
+    buildkite_token: str, after_cursor: str = None
+) -> list[dict[str, str]]:
+    """Fetches a page of the build list using the GraphQL BuildKite API.
+
+    Returns the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE last running/queued builds,
+    or the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE running/queued builds
+    older than the one pointer by |after_cursor| if provided.
+    The |after_cursor| value is taken from the previous page returned by the
+    API.
+
+    Args:
+      buildkite_token: the secret token to authenticate GraphQL requests.
+      after_cursor: cursor after which to start the page fetch.
+
+    Returns:
+      The most recent builds after cursor (if set) with the following format:
+      [
+        {
+            "cursor": <value>,
+            "number": <build-number>,
+        }
+      ]
+    """
+
+    BUILDKITE_GRAPHQL_QUERY = """
+    query OrganizationShowQuery {{
+      organization(slug: "llvm-project") {{
+        pipelines(search: "Github pull requests", first: 1) {{
+          edges {{
+            node {{
+              builds (state: [CANCELING, CREATING, FAILING, RUNNING], first: {PAGE_SIZE}, after: {AFTER}) {{
+                edges {{
+                  cursor
+                  node {{
+                    number
+                  }}
+                }}
+              }}
+            }}
+          }}
+        }}
+      }}
+    }}
+    """
+    query = BUILDKITE_GRAPHQL_QUERY.format(
+        PAGE_SIZE=BUILDKITE_GRAPHQL_BUILDS_PER_PAGE,
+        AFTER="null" if after_cursor is None else '"{}"'.format(after_cursor),
+    )
+    query = json.dumps({"query": query})
+    url = "https://graphql.buildkite.com/v1"
+    headers = {
+        "Authorization": "Bearer " + buildkite_token,
+        "Content-Type": "application/json",
+    }
+    data = requests.post(url, data=query, headers=headers).json()
+    # De-nest the build list.
+    if "errors" in data:
+        logging.info("Failed to fetch BuildKite jobs: {}".format(data["errors"]))
+        return []
+    builds = data["data"]["organization"]["pipelines"]["edges"][0]["node"]["builds"][
+        "edges"
+    ]
+    # Fold cursor info into the node dictionnary.
+    return [{**x["node"], "cursor": x["cursor"]} for x in builds]
+
+
+def buildkite_get_build_info(build_number: str) -> dict:
+    """Returns all the info associated with the provided build number.
+
+    Note: for unknown reasons, graphql returns no jobs for a given build,
+    while this endpoint does, hence why this uses this API instead of graphql.
+
+    Args:
+      build_number: which build number to fetch info for.
+
+    Returns:
+      The info for the target build, a JSON dictionnary.
+    """
+
+    URL = "https://buildkite.com/llvm-project/github-pull-requests/builds/{}.json"
+    return requests.get(URL.format(build_number)).json()
+
+
+def buildkite_get_incomplete_tasks(buildkite_token: str) -> list:
+    """Returns all the running/pending BuildKite builds.
+
+    Args:
+     buildkite_token: the secret token to authenticate GraphQL requests.
+     last_cursor: the cursor to stop at if set. If None, a full page is fetched.
+    """
+    output = []
+    cursor = None
+    while True:
+        page = buildkite_fetch_page_build_list(buildkite_token, cursor)
+        if len(page) == 0:
+            break
+        cursor = page[-1]["cursor"]
+        output += page
+    return output
+
+
+def buildkite_get_metrics(
+    buildkite_token: str, previously_incomplete: set[int]
+) -> (list[JobMetrics], set[int]):
+    """Returns a tuple with:
+
+    - the metrics recorded for newly completed workflow jobs.
+    - the set of workflow still running now.
+
+    Args:
+      buildkite_token: the secret token to authenticate GraphQL requests.
+        previously_incomplete: the set of running workflows the last time this
+        function was called.
+    """
+
+    running_builds = buildkite_get_incomplete_tasks(buildkite_token)
+    incomplete_now = set([x["number"] for x in running_builds])
+    output = []
+
+    for build_id in previously_incomplete:
+        if build_id in incomplete_now:
+            continue
+
+        info = buildkite_get_build_info(build_id)
+        metric_timestamp = dateutil.parser.isoparse(info["finished_at"])
+        for job in info["jobs"]:
+            # This workflow is not interesting to us.
+            if job["name"] not in BUILDKITE_WORKFLOW_TO_TRACK:
+                continue
+
+            # Don't count canceled jobs.
+            if job["canceled_at"]:
+                continue
+
+            created_at = dateutil.parser.isoparse(job["created_at"])
+            scheduled_at = dateutil.parser.isoparse(job["scheduled_at"])
+            started_at = dateutil.parser.isoparse(job["started_at"])
+            finished_at = dateutil.parser.isoparse(job["finished_at"])
+
+            job_name = BUILDKITE_WORKFLOW_TO_TRACK[job["name"]]
+            queue_time = (started_at - scheduled_at).seconds
+            run_time = (finished_at - started_at).seconds
+            status = bool(job["passed"])
+
+            # Grafana will refuse to ingest metrics older than ~2 hours, so we
+            # should avoid sending historical data.
+            metric_age_mn = (
+                datetime.datetime.now(datetime.timezone.utc) - metric_timestamp
+            ).total_seconds() / 60
+            if metric_age_mn > GRAFANA_METRIC_MAX_AGE_MN:
+                logging.warning(
+                    f"Job {job['name']} from workflow {build_id} dropped due"
+                    + f" to staleness: {metric_age_mn}mn old."
+                )
+                continue
+
+            metric_timestamp_ns = int(metric_timestamp.timestamp()) * 10**9
+            workflow_id = build_id
+            workflow_name = "Github pull requests"
+            output.append(
+                JobMetrics(
+                    job_name,
+                    queue_time,
+                    run_time,
+                    status,
+                    metric_timestamp_ns,
+                    workflow_id,
+                    workflow_name,
+                )
+            )
+
+    return output, incomplete_now
+
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:
@@ -168,8 +356,24 @@ def github_get_metrics(
             created_at = job.created_at
             started_at = job.started_at
             completed_at = job.completed_at
-            queue_time = started_at - created_at
-            run_time = completed_at - started_at
+
+            # GitHub API can return results where the started_at is slightly
+            # later then the created_at (or completed earlier than started).
+            # This would cause a -23h59mn delta, which will show up as +24h
+            # queue/run time on grafana.
+            if started_at < created_at:
+                logging.info(
+                    "Workflow {} started before being created.".format(task.id)
+                )
+                queue_time = datetime.timedelta(seconds=0)
+            else:
+                queue_time = started_at - created_at
+            if completed_at < started_at:
+                logging.info("Workflow {} finished before starting.".format(task.id))
+                run_time = datetime.timedelta(seconds=0)
+            else:
+                run_time = completed_at - started_at
+
             if run_time.seconds == 0:
                 continue
 
@@ -179,7 +383,7 @@ def github_get_metrics(
                 datetime.datetime.now(datetime.timezone.utc) - completed_at
             ).total_seconds() / 60
             if metric_age_mn > GRAFANA_METRIC_MAX_AGE_MN:
-                logging.info(
+                logging.warning(
                     f"Job {job.id} from workflow {task.id} dropped due"
                     + f" to staleness: {metric_age_mn}mn old."
                 )
@@ -276,23 +480,33 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
 def main():
     # Authenticate with Github
     github_auth = Auth.Token(os.environ["GITHUB_TOKEN"])
+    buildkite_token = os.environ["BUILDKITE_TOKEN"]
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
 
     # The last workflow this script processed.
     # Because the Github queries are broken, we'll simply log a 'processed'
     # bit for the last COUNT_TO_PROCESS workflows.
     gh_last_workflows_seen_as_completed = set()
+    # Stores the list of pending/running builds in BuildKite we need to check
+    # at the next iteration.
+    bk_incomplete = set()
 
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
     while True:
         github_object = Github(auth=github_auth)
         github_repo = github_object.get_repo("llvm/llvm-project")
 
-        metrics, gh_last_workflows_seen_as_completed = github_get_metrics(
+        gh_metrics, gh_last_workflows_seen_as_completed = github_get_metrics(
             github_repo, gh_last_workflows_seen_as_completed
         )
+
+        bk_metrics, bk_incomplete = buildkite_get_metrics(
+            buildkite_token, bk_incomplete
+        )
+
+        metrics = gh_metrics + bk_metrics
         upload_metrics(metrics, grafana_metrics_userid, grafana_api_key)
         logging.info(f"Uploaded {len(metrics)} metrics")
 

.ci/metrics/requirements.lock.txt

@@ -247,10 +247,18 @@ pynacl==1.5.0 \
     --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
     --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
     # via pygithub
+python-dateutil==2.9.0.post0 \
+    --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
+    --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
+    # via -r ./requirements.txt
 requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
     # via pygithub
+six==1.17.0 \
+    --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
+    --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
+    # via python-dateutil
 typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8

.ci/metrics/requirements.txt

@@ -1 +1,2 @@
 pygithub==2.5.0
+python-dateutil==2.9.0.post0

.ci/monolithic-linux.sh

@@ -90,23 +90,6 @@ if [[ "${runtimes}" != "" ]]; then
   INSTALL_DIR="${BUILD_DIR}/install"
   mkdir -p ${RUNTIMES_BUILD_DIR}
 
-  echo "--- cmake runtimes C++03"
-
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-      -D LIBCXX_TEST_PARAMS="std=c++03" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++03" \
-      -D LLVM_LIT_ARGS="${lit_args}"
-
-  echo "--- ninja runtimes C++03"
-
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
-
   echo "--- cmake runtimes C++26"
 
   rm -rf "${RUNTIMES_BUILD_DIR}"
@@ -127,7 +110,11 @@ if [[ "${runtimes}" != "" ]]; then
 
   echo "--- cmake runtimes clang modules"
 
-  rm -rf "${RUNTIMES_BUILD_DIR}"
+  # We don't need to do a clean build of runtimes, because LIBCXX_TEST_PARAMS
+  # and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
+  # propagates without a clean build. Other that those two variables, builds
+  # are supposed to be the same.
+
   cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
       -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
       -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
@@ -140,6 +127,6 @@ if [[ "${runtimes}" != "" ]]; then
       -D LLVM_LIT_ARGS="${lit_args}"
 
   echo "--- ninja runtimes clang modules"
-  
+
   ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
 fi

.github/workflows/build-ci-container.yml

@@ -27,9 +27,9 @@ jobs:
           # The arch names should match the names used on dockerhub.
           # See https://github.com/docker-library/official-images#architectures-other-than-amd64
           - arch: amd64
-            runs-on: depot-ubuntu-22.04-16
+            runs-on: depot-ubuntu-24.04-16
           - arch: arm64v8
-            runs-on: depot-ubuntu-22.04-arm-16
+            runs-on: depot-ubuntu-24.04-arm-16
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

.github/workflows/commit-access-greeter.yml

@@ -15,7 +15,7 @@ jobs:
     if: >-
       github.repository_owner == 'llvm' &&
       github.event.label.name == 'infra:commit-access-request'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
         with:

.github/workflows/commit-access-review.yml

@@ -12,7 +12,7 @@ permissions:
 jobs:
   commit-access-review:
     if: github.repository_owner == 'llvm'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Fetch LLVM sources
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

.github/workflows/docs.yml

@@ -65,7 +65,7 @@ jobs:
           fetch-depth: 1
       - name: Get subprojects that have doc changes
         id: docs-changed-subprojects
-        uses: tj-actions/changed-files@dcc7a0cba800f454d79fff4b993e8c3555bcc0a8 # v45.0.7
+        uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1
         with:
           files_yaml: |
             llvm: