Auto merge of rust-lang#112407 - tgross35:ci-docs-publish, r=Mark-Simulacrum

Publish docs as github artifacts during CI

Discussed here: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Building.20docs.20for.20PR.20CI

The goal is to make docs available for download after CI runs on PRs, for easy review of API changes.

Notes:

- Currently this only captures library documentation (`core`, `alloc`, `std`, `test`, `proc_macro`)
- You can't see artifacts until the entire workflow run has completed actions/upload-artifact#53
- There is currently a generic file name `ci-artifacts`. No way to customize this based on contained files unfortunately actions/upload-artifact#349

You can find the results at the bottom of the CI "summary" page:

<img width="379" alt="image" src="https://github.com/rust-lang/rust/assets/13724985/d3748e59-242c-40f8-9f54-82177b9b481b">

Author: bors

.github/workflows/ci.yml

@@ -38,6 +38,7 @@ jobs:
     env:
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
       CACHE_DOMAIN: ci-caches.rust-lang.org
@@ -143,6 +144,17 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
         if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
@@ -156,6 +168,7 @@ jobs:
     env:
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       DEPLOY_BUCKET: rust-lang-ci2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@@ -557,6 +570,17 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
         if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
@@ -571,6 +595,7 @@ jobs:
       DIST_TRY_BUILD: 1
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       DEPLOY_BUCKET: rust-lang-ci2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@@ -672,6 +697,17 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
         if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -45,6 +45,9 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
            python3 ../x.py test --stage 0 src/tools/compiletest && \
            python3 ../x.py test --stage 0 core alloc std test proc_macro && \
            # Build both public and internal documentation.
+           RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library && \
+           mkdir -p /checkout/obj/staging/doc && \
+           cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \
            RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 compiler && \
            RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library/test && \
            /scripts/validate-toolstate.sh && \

src/ci/github-actions/ci.yml

@@ -34,6 +34,8 @@ x--expand-yaml-anchors--remove:
   - &shared-ci-variables
     CI_JOB_NAME: ${{ matrix.name }}
     CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+    # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+    HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
   - &public-variables
     SCCACHE_BUCKET: rust-lang-ci-sccache2
@@ -229,6 +231,20 @@ x--expand-yaml-anchors--remove:
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
         <<: *step
 
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        <<: *step
+
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          # name is set in previous step
+          name: ${{ env.DOC_ARTIFACT_NAME }}
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        <<: *step
+
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:

src/ci/scripts/create-doc-artifacts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Compress doc artifacts and name them based on the commit, or the date if
+# commit is not available.
+
+set -euox pipefail
+
+# Try to get short commit hash, fallback to date
+if [ -n "$HEAD_SHA" ]; then
+    short_rev=$(echo "${HEAD_SHA}" | cut -c1-8)
+else
+    short_rev=$(git rev-parse --short HEAD || date -u +'%Y-%m-%dT%H%M%SZ')
+fi
+
+# Try to get branch, fallback to none
+branch=$(git branch --show-current || echo)
+
+if [ -n "$branch" ]; then
+    branch="${branch}-"
+fi
+
+if [ "${GITHUB_EVENT_NAME:=none}" = "pull_request" ]; then
+    pr_num=$(echo "$GITHUB_REF_NAME" | cut -d'/' -f1)
+    name="doc-${pr_num}-${short_rev}"
+else
+    name="doc-${branch}${short_rev}"
+fi
+
+
+if [ -d "obj/staging/doc" ]; then
+    mkdir -p obj/artifacts/doc
+
+    # Level 12 seems to give a good tradeoff of time vs. space savings
+    ZSTD_CLEVEL=12 ZSTD_NBTHREADS=4 \
+    tar --zstd -cf "obj/artifacts/doc/${name}.tar.zst" -C obj/staging/doc .
+
+    ls -lh obj/artifacts/doc
+fi
+
+# Set this environment variable for future use if running in CI
+if [ -n "$GITHUB_ENV" ]; then
+    echo "DOC_ARTIFACT_NAME=${name}" >> "$GITHUB_ENV"
+fi