fix: Filter linkchecker output (#2406)

Author: untitaker

.github/workflows/test.yml

@@ -88,21 +88,5 @@ jobs:
       - name: yarn install
         if: steps.cache.outputs.cache-hit != 'true'
         run: yarn install
-      - name: Install httpserver
-        run: volta install http-server
-      - name: Install linkcheck
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |-
-          wget -nc https://github.com/filiph/linkcheck/releases/download/v2.0.12/linkcheck-linux-x64.exe -O ./linkcheck
-          chmod +x ./linkcheck
-      - name: Build
-        run: yarn build
-      - name: Run linkcheck (search for "HTTP 404" in logs to get to errors)
-        run: |-
-          http-server public/ > /dev/null &
-          printf "Waiting for server to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null "http://localhost:8080"); do printf '.'; sleep 0.5; done'
-
-          # exitcode=1 -- only warnings (about broken anchors)
-          # exitcode=2 -- hard 404s
-          # We only fail the build if there are hard 404s
-          ./linkcheck http://localhost:8080 || [ $? -eq 1 ]
+      - name: yarn linkcheck
+        run: yarn linkcheck

.gitignore

@@ -77,3 +77,6 @@ static/_platforms/
 
 # generated nginx file
 nginx.out.conf
+
+# linkchecker binary
+linkcheck

bin/linkchecker

@@ -0,0 +1,77 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+LINKCHECK_VERSION=2.0.12
+
+# This linkchecker written in Dart is the fastest we've found and finds the
+# most links. It is still very slow, requires a separate HTTP server, and is
+# barely configurable wrt output verbosity. Hence this script exists.
+#
+# Considering lack of performant, correct alternatives we *assume* that
+# building our own would be hard.
+
+if [ "$(uname)" = "Darwin" ]; then
+    url="https://github.com/filiph/linkcheck/releases/download/v$LINKCHECK_VERSION/linkcheck-mac-x64"
+else
+    url="https://github.com/filiph/linkcheck/releases/download/v$LINKCHECK_VERSION/linkcheck-linux-x64.exe"
+fi
+
+if [ ! -f ./linkcheck ]; then
+    echo ">>> Downloading linkcheck"
+    wget -nc "$url" -O ./linkcheck
+    chmod +x ./linkcheck
+fi
+
+echo ">>> Building docs"
+yarn build
+
+echo ">>> Starting HTTP server"
+yarn http-server public/ > /dev/null &
+httpserver_pid=$!
+trap "kill -9 $httpserver_pid" SIGINT SIGTERM EXIT
+
+echo ">>> Waiting for server to be up"
+timeout 60 bash -c 'until $(curl -Isf -o /dev/null "http://localhost:8080"); do printf '.'; sleep 0.5; done'
+
+# exitcode=1 -- only warnings (about broken anchors)
+# exitcode=2 -- hard 404s
+# We only fail the build if there are hard 404s
+echo ">>> Checking some links (started at $(date))"
+echo "    This step takes 5-10 minutes and does not produce intermediate output."
+echo "    If your fans are spinning it's working."
+(./linkcheck http://localhost:8080 || [ $? -eq 1 ]) | \
+
+    # Post-process the output and remove all anchor warnings
+    grep -v "(HTTP 200 but missing anchor)" | \
+
+    # Clean up output some more, launch in subshell because of namespace
+    # pollution (and because I want to keep the bash-isms to a minimum)
+    (
+        prev_line=""
+        while IFS= read line; do
+            # XXX: Bashism
+            if [ -z "$line" ] && [[ "$prev_line" = http://localhost:8080/* ]]; then
+                # We are in a section for which we report nothing (because we grepped
+                # away anchor warnings):
+                # 
+                # http://...  <-- prev_line
+                #             <-- line
+                # http://...
+
+                true
+            else
+                echo "$prev_line"
+            fi
+
+            prev_line="$line"
+        done
+
+        # Since the loop always outputs the prev_line, we need to fix the off-by-one error here
+        echo "$prev_line"
+    ) | \
+
+    # Remove repeated blank lines
+    cat -s
+
+kill -9 $httpserver_pid

package.json

@@ -90,7 +90,8 @@
     "prettier:fix:all": "prettier --write \"./{src,plugins}/**/*.{md,mdx,js,jsx}\"",
     "start": "yarn run develop",
     "format": "prettier --write \"src/**/*.js\"",
-    "test": "jest"
+    "test": "jest",
+    "linkcheck": "bin/linkchecker"
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^5.11.4",
@@ -106,6 +107,7 @@
     "eslint-plugin-react": "^7.20.6",
     "est": "^0.2.1-alpha",
     "gh-pages": "^2.2.0",
+    "http-server": "^0.12.3",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^26.4.2",
     "jest-dom": "^4.0.0",

yarn.lock

@@ -4102,6 +4102,11 @@ base@^0.11.1:
     mixin-deep "^1.2.0"
     pascalcase "^0.1.1"
 
+basic-auth@^1.0.3:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-1.1.0.tgz#45221ee429f7ee1e5035be3f51533f1cdfd29884"
+  integrity sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=
+
 [email protected]:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16"
@@ -5210,7 +5215,7 @@ colorette@^1.2.1:
   resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b"
   integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==
 
-colors@^1.1.2:
+colors@^1.1.2, colors@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
   integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
@@ -5526,6 +5531,11 @@ cors@^2.8.5:
     object-assign "^4"
     vary "^1"
 
+corser@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87"
+  integrity sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=
+
 cosmiconfig@^5.0.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a"
@@ -6601,6 +6611,16 @@ ecc-jsbn@~0.1.1:
     jsbn "~0.1.0"
     safer-buffer "^2.1.0"
 
+ecstatic@^3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/ecstatic/-/ecstatic-3.3.2.tgz#6d1dd49814d00594682c652adb66076a69d46c48"
+  integrity sha512-fLf9l1hnwrHI2xn9mEDT7KIi22UDqA2jaCwyCbSUJh9a1V+LEUSL/JO/6TIz/QyuBURWUHrFL5Kg2TtO1bkkog==
+  dependencies:
+    he "^1.1.1"
+    mime "^1.6.0"
+    minimist "^1.1.0"
+    url-join "^2.0.5"
+
 [email protected]:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
@@ -9542,6 +9562,11 @@ hastscript@^5.0.0:
     property-information "^5.0.0"
     space-separated-tokens "^1.0.0"
 
+he@^1.1.1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
+  integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
+
 header-case@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/header-case/-/header-case-1.0.1.tgz#9535973197c144b09613cd65d317ef19963bd02d"
@@ -9767,7 +9792,7 @@ [email protected]:
     lodash "^4.17.11"
     micromatch "^3.1.10"
 
-http-proxy@^1.17.0, http-proxy@^1.18.1:
+http-proxy@^1.17.0, http-proxy@^1.18.0, http-proxy@^1.18.1:
   version "1.18.1"
   resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549"
   integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==
@@ -9776,6 +9801,22 @@ http-proxy@^1.17.0, http-proxy@^1.18.1:
     follow-redirects "^1.0.0"
     requires-port "^1.0.0"
 
+http-server@^0.12.3:
+  version "0.12.3"
+  resolved "https://registry.yarnpkg.com/http-server/-/http-server-0.12.3.tgz#ba0471d0ecc425886616cb35c4faf279140a0d37"
+  integrity sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==
+  dependencies:
+    basic-auth "^1.0.3"
+    colors "^1.4.0"
+    corser "^2.0.1"
+    ecstatic "^3.3.2"
+    http-proxy "^1.18.0"
+    minimist "^1.2.5"
+    opener "^1.5.1"
+    portfinder "^1.0.25"
+    secure-compare "3.0.1"
+    union "~0.5.0"
+
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -12389,7 +12430,7 @@ mime-types@^2.1.12, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24:
   dependencies:
     mime-db "1.44.0"
 
-[email protected], mime@^1.3.4:
+[email protected], mime@^1.3.4, mime@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
@@ -13154,6 +13195,11 @@ open@^6.4.0:
   dependencies:
     is-wsl "^1.1.0"
 
+opener@^1.5.1:
+  version "1.5.2"
+  resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598"
+  integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==
+
 opentracing@^0.14.4:
   version "0.14.4"
   resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.4.tgz#a113408ea740da3a90fde5b3b0011a375c2e4268"
@@ -13903,7 +13949,7 @@ pnp-webpack-plugin@^1.6.4:
   dependencies:
     ts-pnp "^1.1.6"
 
-portfinder@^1.0.26:
+portfinder@^1.0.25, portfinder@^1.0.26:
   version "1.0.28"
   resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778"
   integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==
@@ -14561,7 +14607,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
   integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==
 
-qs@^6.5.2:
+qs@^6.4.0, qs@^6.5.2:
   version "6.9.4"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687"
   integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==
@@ -15887,6 +15933,11 @@ section-matter@^1.0.0:
     extend-shallow "^2.0.1"
     kind-of "^6.0.0"
 
+[email protected]:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3"
+  integrity sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=
+
 seek-bzip@^1.0.5:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.6.tgz#35c4171f55a680916b52a07859ecf3b5857f21c4"
@@ -17794,6 +17845,13 @@ union-value@^1.0.0:
     is-extendable "^0.1.1"
     set-value "^2.0.1"
 
+union@~0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075"
+  integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==
+  dependencies:
+    qs "^6.4.0"
+
 uniq@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff"
@@ -18060,6 +18118,11 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
+url-join@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/url-join/-/url-join-2.0.5.tgz#5af22f18c052a000a48d7b82c5e9c2e2feeda728"
+  integrity sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=
+
 url-loader@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-1.1.2.tgz#b971d191b83af693c5e3fea4064be9e1f2d7f8d8"