Split checking hydra eval on buildkite (#1633)

Author: hamishmack

.buildkite/pipeline.yml

@@ -74,13 +74,39 @@ steps:
     agents:
       system: x86_64-linux
 
-  - label: 'Check that jobset will evaluate in Hydra'
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 0 and 1'
     command:
       - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-      - ./check-hydra.sh 0
-      - ./check-hydra.sh 1
-      - ./check-hydra.sh 2
-      - ./check-hydra.sh 3
+      - ./check-hydra.sh --arg ifdLevel 0
+      - ./check-hydra.sh --arg ifdLevel 1
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 2'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - ./check-hydra.sh --arg ifdLevel 2
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 8.10'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 6 x == \"ghc810\"'"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 9.2'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 5 x == \"ghc92\"'"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and not (ghc 8.10 or ghc 9.2)'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: !(__substring 0 6 x == \"ghc810\" || __substring 0 5 x == \"ghc92\")'"
     agents:
       system: x86_64-linux
 

release.nix

@@ -2,6 +2,7 @@
 # on a machine with e.g. no way to build the Darwin IFDs you need!
 { supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]
 , ifdLevel ? 3
+, include ? (compiler-nix-name: true)
 , checkMaterialization ? false }:
 
 let
@@ -16,7 +17,7 @@ let
   lib = genericPkgs.lib;
   ci = import ./ci.nix { inherit supportedSystems ifdLevel checkMaterialization; restrictEval = true; };
   allJobs = stripAttrsForHydra (filterDerivations ci);
-  latestJobs = {
+  latestJobs = lib.optionalAttrs (include "ghc8107") {
     # All the jobs are included in the `requiredJobs`, but the ones
     # added here will also included without aggregation, making it easier
     # to find a failing test.  Keep in mind though that adding too many
@@ -32,7 +33,7 @@ let
         let nixpkgsJobs = allJobs.${nixpkgsVer};
         in lib.concatMap (compiler-nix-name:
           let ghcJobs = nixpkgsJobs.${compiler-nix-name};
-          in builtins.concatMap (platform:
+          in lib.optionals (include compiler-nix-name) (builtins.concatMap (platform:
             let platformJobs = ghcJobs.${platform};
             in builtins.map (crossPlatform: {
               name = "required-${nixpkgsVer}-${compiler-nix-name}-${platform}-${crossPlatform}";
@@ -42,7 +43,7 @@ let
                 constituents = lib.collect (d: lib.isDerivation d) platformJobs.${crossPlatform};
               };
            }) (names platformJobs)
-         ) (names ghcJobs)
+         ) (names ghcJobs))
         ) (names nixpkgsJobs)
       ) (names allJobs));
 in traceNames "job " (latestJobs // requiredJobs // {

scripts/check-hydra.nix

@@ -10,12 +10,12 @@ writeScript "check-hydra.sh" ''
   export PATH="${makeBinPath [ coreutils time gnutar gzip hydra-unstable jq gitMinimal ]}"
   export NIX_PATH=
 
-  echo '~~~ Evaluating release.nix with --arg ifdLevel '$1
+  echo '~~~ Evaluating release.nix with' "$@"
   command time --format '%e' -o eval-time.txt \
       hydra-eval-jobs \
       --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
       --arg supportedSystems '[ builtins.currentSystem ]' \
-      --arg ifdLevel $1 \
+      "$@" \
       -I $(realpath .) release.nix > eval.json
   EVAL_EXIT_CODE="$?"
   if [ "$EVAL_EXIT_CODE" != 0 ]