Add evalSystem and evalPackages project args (#1546)

This adds a way to specify the `evalSystem` or `evalPackages` explicitly when calling the `project` functions.

Currently if we want to make a `flake` that supports multiple systems we have few options:

* Require builders for all the supported systems (even just for `nix flake show`).

* Pass `--impure` so that haskell.nix can see `builtins.currentSystem` to set up `pkgs.evalPackages` to use that.  Unfortunately this prevents nix from caching some of the work it does and often results in it recalculating for each supported system when it would otherwise be cached and take no time at all.

* Add an overlay to replace `evalPackages`.  This works, but it is not straight forward.

* Materialize the nix files for the project.

This change allows `evalSystem = "x86_64-linux";` to be passed telling `haskell.nix` to run `cabal` and `nix-tools` on that system.  The user will have to have a builder for that system, but does not need to have builders for the others (unless building outputs for them).

Author: hamishmack

.buildkite/pipeline.yml

@@ -44,3 +44,9 @@ steps:
       - nix build .#roots.x86_64-linux --option allow-import-from-derivation false
     agents:
       system: x86_64-linux
+
+  - label: 'Check that evaluation of hydra jobs works without using remote builders'
+    command:
+      - nix-instantiate release.nix -A required-unstable-ghc8107-x86_64-darwin-native --show-trace --builders ''
+    agents:
+      system: x86_64-linux

build.nix

@@ -8,8 +8,9 @@ in
 { nixpkgs ? haskellNix.sources.nixpkgs-2111
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
+, evalPackages ? import nixpkgs nixpkgsArgs
 , nixpkgsForHydra ? haskellNix.sources.nixpkgs-2105
-, pkgsForHydra ? import nixpkgsForHydra nixpkgsArgs
+, pkgsForHydra ? import nixpkgsForHydra (nixpkgsArgs // { inherit (pkgs) system; })
 , ifdLevel ? 1000
 , compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
 }:
@@ -19,12 +20,13 @@ let
   buildHaskell = pkgs.buildPackages.haskell-nix;
   tool = buildHaskell.tool;
 in rec {
-  tests = import ./test/default.nix { inherit pkgs ifdLevel compiler-nix-name; };
+  tests = import ./test/default.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
 
   tools = pkgs.lib.optionalAttrs (ifdLevel >= 3) (
     pkgs.recurseIntoAttrs ({
-      cabal-latest = tool compiler-nix-name "cabal" "latest";
+      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; };
       hlint-latest = tool compiler-nix-name "hlint" {
+        inherit evalPackages;
         version = {
             "ghc865" = "3.2.8";
             "ghc882" = "3.3.6";
@@ -33,7 +35,7 @@ in rec {
           }.compiler-nix-name or "latest";
       };
     } // pkgs.lib.optionalAttrs (!__elem compiler-nix-name ["ghc921" "ghc922" "ghc923"]) {
-      hls-latest = tool compiler-nix-name "haskell-language-server" "latest";
+      hls-latest = tool compiler-nix-name "haskell-language-server" { inherit evalPackages; };
     })
   );
 
@@ -85,7 +87,7 @@ in rec {
   # These are pure parts of maintainer-script so they can be built by hydra
   # and added to the cache to speed up buildkite.
   maintainer-script-cache = pkgs.recurseIntoAttrs (
-      (pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+      (pkgs.lib.optionalAttrs (pkgsForHydra.system == "x86_64-linux") {
         inherit (maintainer-scripts) check-hydra;
       })
     // (pkgs.lib.optionalAttrs (ifdLevel > 2) {

builder/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs, compiler }:
+{ pkgs, buildPackages, evalPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs, compiler }:
 
 let
   # Builds a single component of a package.
@@ -46,6 +46,7 @@ let
   hoogleLocal = let
     nixpkgsHoogle = import (pkgs.path + /pkgs/development/haskell-modules/hoogle.nix);
   in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc8107" "hoogle" {
+        inherit evalPackages;
         version = "5.0.18.3";
         index-state = pkgs.haskell-nix.internalHackageIndexState;
       }
@@ -66,7 +67,7 @@ let
 
   # Same as haskellPackages.shellFor in nixpkgs.
   shellFor = haskellLib.weakCallPackage pkgs ./shell-for.nix {
-    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib buildPackages compiler;
+    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib buildPackages evalPackages compiler;
     inherit (buildPackages) glibcLocales;
   };
 

builder/shell-for.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, buildPackages, compiler }:
+{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, buildPackages, evalPackages, compiler }:
 
 { # `packages` function selects packages that will be worked on in the shell itself.
   # These packages will not be built by `shellFor`, but their
@@ -133,7 +133,12 @@ let
     # inherit (hsPkgs) hoogle;
   } // (
     lib.optionalAttrs (args ? tools && args.tools ? hoogle) {
-      hoogle = buildPackages.haskell-nix.tool compiler.nix-name "hoogle" args.tools.hoogle;
+      hoogle = buildPackages.haskell-nix.hackage-tool (
+        haskellLib.versionOrModToMods args.tools.hoogle ++ [{
+          name = "hoogle";
+          compiler-nix-name = compiler.nix-name;
+          inherit evalPackages;
+        }]);
     }
   ));
 
@@ -147,7 +152,7 @@ in
     nativeBuildInputs = [ ghcEnv ]
       ++ nativeBuildInputs
       ++ mkDrvArgs.nativeBuildInputs or []
-      ++ lib.attrValues (buildPackages.haskell-nix.tools compiler.nix-name tools)
+      ++ lib.attrValues (buildPackages.haskell-nix.tools evalPackages compiler.nix-name tools)
       # If this shell is a cross compilation shell include
       # wrapper script for running cabal build with appropriate args.
       # Includes `--with-compiler` in case the `cabal.project` file has `with-compiler:` in it.

changelog.md

@@ -1,6 +1,32 @@
 This file contains a summary of changes to Haskell.nix and `nix-tools`
 that will impact users.
 
+## Jul 27, 2022
+* Removed reliance on `builtins.currentSystem`.  It was used it to provide
+  `pkgs.evalPackages` via an overlay that it used to run derivations
+  used in imports from derivation (IFDs).
+
+  These derivations are now run on `buildPackages` by default.
+
+  Passsing `evalPackages` to a project function will change where all the
+  derivations used in IFDs are run for that project (including shell tools):
+    evalPackages = import nixpkgs haskellNix.nixpkgsArgs;
+
+  Passing `evalSystem` instead will use create a suitable `nixpkgs` using `pkgs.path`
+  and `pkgs.overlay`:
+    evalSystem = "x86_64-linux";
+  or
+    evalSystem = builtins.currentSystem;
+
+  The `haskellLib.cleanGit` function is also affected by this change.  If you are cross
+  compiling and using `cleanGit` you should probably do something like:
+    pkgs = import nixpkgs haskellNix.nixpkgsArgs;
+    evalPackages = import nixpkgs (haskellNix.nixpkgsArgs // { system = evalSystem; });
+    p = pkgs.pkgsCross.mingwW64.haskell-nix.cabalProject {
+      inherit evalPackages;
+      src = evalPackages.haskell-nix.haskellLib.cleanGit { src = ./.; };
+    };
+
 ## Feb 16, 2022
 * Removed lookupSha256 argument from project functions.
   Pass a `sha256map` instead.

ci.nix

@@ -78,12 +78,11 @@
 in
 dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
   let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
-      # We need this for generic nixpkgs stuff at the right version
-      genericPkgs = import pinnedNixpkgsSrc {};
-  in dimension "GHC version" (compilerNixNames nixpkgsName genericPkgs) (compiler-nix-name: {runTests}:
-    dimension "System" (systems nixpkgsName genericPkgs compiler-nix-name) (systemName: system:
+      evalPackages = import pinnedNixpkgsSrc nixpkgsArgs;
+  in dimension "GHC version" (compilerNixNames nixpkgsName evalPackages) (compiler-nix-name: {runTests}:
+    dimension "System" (systems nixpkgsName evalPackages compiler-nix-name) (systemName: system:
       let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
-          build = import ./build.nix { inherit pkgs ifdLevel compiler-nix-name; };
+          build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
           platformFilter = platformFilterGeneric pkgs system;
       in filterAttrsOnlyRecursive (_: v: platformFilter v && !(isDisabled v)) ({
         # Native builds
@@ -96,14 +95,14 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
         } // pkgs.lib.optionalAttrs (ifdLevel >= 1) {
           iserv-proxy = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "iserv-proxy:exe:iserv-proxy";
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
-          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).getComponent "exe:hello";
+          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit evalPackages compiler-nix-name; }).getComponent "exe:hello";
         });
       }
       //
-      dimension "Cross system" (crossSystems nixpkgsName genericPkgs compiler-nix-name system) (crossSystemName: crossSystem:
+      dimension "Cross system" (crossSystems nixpkgsName evalPackages compiler-nix-name system) (crossSystemName: crossSystem:
         # Cross builds
         let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system crossSystem; });
-            build = import ./build.nix { inherit pkgs ifdLevel compiler-nix-name; };
+            build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
         in pkgs.recurseIntoAttrs (pkgs.lib.optionalAttrs (ifdLevel >= 1) ({
             roots = pkgs.haskell-nix.roots' compiler-nix-name ifdLevel;
             ghc = pkgs.buildPackages.haskell-nix.compiler."${compiler-nix-name}";

flake.nix

@@ -59,7 +59,7 @@
     };
   };
 
-  outputs = { self, nixpkgs, nixpkgs-unstable, nixpkgs-2105, flake-utils, ... }@inputs:
+  outputs = { self, nixpkgs, nixpkgs-unstable, nixpkgs-2105, nixpkgs-2111, nixpkgs-2205, flake-utils, ... }@inputs:
     let compiler = "ghc923";
       config = import ./config.nix;
     in {
@@ -101,6 +101,12 @@
             };
             pkgs = import nixpkgs
               (nixpkgsArgs // { localSystem = { inherit system; }; });
+            pkgs-2105 = import nixpkgs-2105
+              (nixpkgsArgs // { localSystem = { inherit system; }; });
+            pkgs-2111 = import nixpkgs-2111
+              (nixpkgsArgs // { localSystem = { inherit system; }; });
+            pkgs-2205 = import nixpkgs-2205
+              (nixpkgsArgs // { localSystem = { inherit system; }; });
             pkgs-unstable = import nixpkgs-unstable
               (nixpkgsArgs // { localSystem = { inherit system; }; });
             hix = import ./hix/default.nix { inherit pkgs; };

hix/project.nix

@@ -56,20 +56,20 @@ let
         nixpkgsArgs = config.haskellNix.nixpkgsArgs // {
           overlays = config.haskellNix.nixpkgsArgs.overlays ++ config.overlays;
         };
-        pkgs = import config.nixpkgs config.nixpkgsArgs;
-        project = config.pkgs.haskell-nix.project [
+        _module.args.pkgs = import config.nixpkgs config.nixpkgsArgs;
+        project = pkgs.haskell-nix.project [
             (import ../modules/hix-project.nix)
             userDefaults
             projectDefaults
             commandArgs'
-            {
+            ({config, ...}: {
               src =
                 if __pathExists (toString (src.origSrcSubDir or src) + "/.git")
-                  then config.pkgs.haskell-nix.haskellLib.cleanGit {
+                  then config.evalPackages.haskell-nix.haskellLib.cleanGit {
                     inherit src name;
                   }
                   else src;
-            }
+            })
           ];
       })
     ];

lib/cabal-project-parser.nix

@@ -122,7 +122,7 @@ let
   # This works because `cabal configure` does not include any of the `/nix/sore/`
   # paths in the `plan.json` (so materialized plan-nix will still work as expeced).
   # See tests/unit.nix for examples of input and output.
-  parseRepositoryBlock = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
+  parseRepositoryBlock = evalPackages: cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
     let
       lines = pkgs.lib.splitString "\n" block;
       # The first line will contain the repository name.
@@ -138,7 +138,7 @@ let
       # The $HOME/.cabal/packages/${name} after running `cabal v2-update` to download the repository
       repoContents = if inputMap ? ${attrs.url}
         # If there is an input use it to make `file:` url and create a suitable `.cabal/packages/${name}` directory
-        then pkgs.evalPackages.runCommand name ({
+        then evalPackages.runCommand name ({
           nativeBuildInputs = [ cabal-install ];
           preferLocalBuild = true;
         }) ''
@@ -155,9 +155,9 @@ let
             cabal v2-update ${name}
             cp -r $HOME/.cabal/packages/${name} $out
         ''
-        else pkgs.evalPackages.runCommand name ({
-          nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
-          LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
+        else evalPackages.runCommand name ({
+          nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ];
+          LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
           LANG = "en_US.UTF-8";
           preferLocalBuild = true;
         } // pkgs.lib.optionalAttrs (sha256 != null) {
@@ -175,15 +175,15 @@ let
               ${pkgs.lib.optionalString (attrs ? key-threshold) "key-threshold: ${attrs.key-threshold}"}
             EOF
 
-            export SSL_CERT_FILE=${pkgs.evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
+            export SSL_CERT_FILE=${evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
             cabal v2-update ${name}
             cp -r $HOME/.cabal/packages/${name} $out
         '';
       # Output of hackage-to-nix
       hackage = import (
-        pkgs.evalPackages.runCommand ("hackage-to-nix-" + name) {
-          nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
-          LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
+        evalPackages.runCommand ("hackage-to-nix-" + name) {
+          nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ];
+          LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
           LANG = "en_US.UTF-8";
           preferLocalBuild = true;
         } ''
@@ -196,11 +196,11 @@ let
       };
     };
 
-  parseRepositories = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
+  parseRepositories = evalPackages: cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
     let
       # This will leave the name of repository in the first line of each block
       blocks = pkgs.lib.splitString "\nrepository " ("\n" + projectFile);
-      repoBlocks = builtins.map (parseRepositoryBlock cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
+      repoBlocks = builtins.map (parseRepositoryBlock evalPackages cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
     in {
       extra-hackages = pkgs.lib.lists.map (block: block.hackage) repoBlocks;
       repos = pkgs.lib.lists.foldl' (x: block: x // block.repo) {} repoBlocks;