Merge branch 'master' into avieth/misc-patches

Author: avieth

.buildkite/pipeline.yml

@@ -1,16 +1,112 @@
 steps:
-  - label: 'Run tests with ghc8107'
-    command: "./test/tests.sh ghc8107"
+  - label: 'Run tests with ghc8107: Running the nix-build tests...'
+    command: "./test/tests.sh ghc8107 nix-build"
     agents:
       system: x86_64-linux
 
-  - label: 'Check that jobset will evaluate in Hydra'
+  - label: 'Run tests with ghc8107: Running the unit tests...'
+    command: "./test/tests.sh ghc8107 unit-tests"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that a nix-shell works for runghc...'
+    command: "./test/tests.sh ghc8107 runghc"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that a nix-shell works for cabal...'
+    command: "./test/tests.sh ghc8107 cabal"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that a nix-shell works for cabal (doExactConfig component)...'
+    command: "./test/tests.sh ghc8107 cabal-doExactConfig"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that a nix-shell works for a project with test-suite build-tools and benchmarks...'
+    command: "./test/tests.sh ghc8107 tests-benchmarks"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that a nix-shell works for a multi-target project...'
+    command: "./test/tests.sh ghc8107 multi-target"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking shellFor works for a cabal project, multiple packages...'
+    command: "./test/tests.sh ghc8107 shellFor-single-package"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking shellFor works for a cabal project, single package...y'
+    command: "./test/tests.sh ghc8107 shellFor-multiple-package"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking shellFor works for a cabal project, single package...'
+    command: "./test/tests.sh ghc8107 shellFor-hoogle"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking shellFor does not depend on given packages...y'
+    command: "./test/tests.sh ghc8107 shellFor-not-depends"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking the maintainer scripts...y'
+    command: "./test/tests.sh ghc8107 maintainer-scripts"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that plan construction works with extra Hackages...'
+    command: "./test/tests.sh ghc8107 plan-extra-hackages"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: Checking that package with extra Hackages can be build...'
+    command: "./test/tests.sh ghc8107 build-extra-hackages"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run tests with ghc8107: End-2-end test of hix project initialization and flakes development shell ...'
+    command: "./test/tests.sh ghc8107 hix"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 0 and 1'
     command:
       - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-      - ./check-hydra.sh 0
-      - ./check-hydra.sh 1
-      - ./check-hydra.sh 2
-      - ./check-hydra.sh 3
+      - ./check-hydra.sh --arg ifdLevel 0
+      - ./check-hydra.sh --arg ifdLevel 1
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 2'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - ./check-hydra.sh --arg ifdLevel 2
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 8.10'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 6 x == \"ghc810\"'"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 9.2'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 5 x == \"ghc92\"'"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that jobset will evaluate in Hydra at ifdLevel 3 and not (ghc 8.10 or ghc 9.2)'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
+      - "./check-hydra.sh --arg ifdLevel 3 --arg include 'x: !(__substring 0 6 x == \"ghc810\" || __substring 0 5 x == \"ghc92\")'"
     agents:
       system: x86_64-linux
 
@@ -41,7 +137,7 @@ steps:
 
   - label: 'Check that the haskell.nix roots do not require IFDs'
     command:
-      - nix build .#roots.x86_64-linux --option allow-import-from-derivation false
+      - nix build .#roots.x86_64-linux --accept-flake-config --option allow-import-from-derivation false
     agents:
       system: x86_64-linux
 
@@ -50,3 +146,10 @@ steps:
       - nix-instantiate release.nix -A required-unstable-ghc8107-x86_64-darwin-native --show-trace --builders ''
     agents:
       system: x86_64-linux
+
+  - label: 'Check hix -- run github:haskell/cabal/3.8#cabal-install:exe:cabal -- --version'
+    command:
+      - "HIX_DIR=$(mktemp -d) nix run .#hix --accept-flake-config -- run github:haskell/cabal/3.8#cabal-install:exe:cabal --accept-flake-config --override-input haskellNix . -- --version"
+    agents:
+      system: x86_64-linux
+

README.md

@@ -9,17 +9,17 @@ its dependencies into Nix code.
 ## Documentation
 
 - [Introduction](https://input-output-hk.github.io/haskell.nix/index.html)
-- [Getting Started](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started.html)
-- [Troubleshooting](https://input-output-hk.github.io/haskell.nix/troubleshooting.html)
+- [Getting Started](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started)
+- [Troubleshooting](https://input-output-hk.github.io/haskell.nix/troubleshooting)
 - Explore the documentation from there to find further topics.
 
 ## Help! Something isn't working
 
 The #1 problem that people have when using `haskell.nix` is that they find themselves building GHC.
 This should not happen, but you *must* follow the `haskell.nix` setup instructions properly to avoid it.
 If you find this happening to you, please check that you have followed the 
-[getting started instructions](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started.html#setting-up-the-binary-cache) and
-consult the corresponding [troubleshooting section](https://input-output-hk.github.io/haskell.nix/troubleshooting.html#why-am-i-building-ghc).
+[getting started instructions](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started#setting-up-the-binary-cache) and
+consult the corresponding [troubleshooting section](https://input-output-hk.github.io/haskell.nix/troubleshooting#why-am-i-building-ghc).
 
 The troubleshooting documentation also contains some help for other common issues.
 Please give it a read before asking on IRC or opening an issue.

build.nix

@@ -5,7 +5,7 @@
 let
   haskellNix = (import ./default.nix {});
 in
-{ nixpkgs ? haskellNix.sources.nixpkgs-2111
+{ nixpkgs ? haskellNix.sources.nixpkgs-2205
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
 , evalPackages ? import nixpkgs nixpkgsArgs
@@ -34,7 +34,6 @@ in rec {
             "ghc884" = "3.3.6";
           }.compiler-nix-name or "latest";
       };
-    } // pkgs.lib.optionalAttrs (!__elem compiler-nix-name ["ghc921" "ghc922" "ghc923" "ghc924"]) {
       hls-latest = tool compiler-nix-name "haskell-language-server" { inherit evalPackages; };
     })
   );
@@ -77,10 +76,14 @@ in rec {
       ];
     };
     check-materialization-concurrency = pkgs.buildPackages.callPackage ./scripts/check-materialization-concurrency/check.nix {};
-    check-path-support = pkgs.buildPackages.callPackage ./scripts/check-path-support.nix {
-      # TODO remove this when nixpkgs-2205 is released and used for `pkgs`
-      # check-path-support fails unless we have nix 2.4 or newer.
-      inherit (import haskellNix.sources.nixpkgs-unstable {}) nix;
+    # Forcing nixpkgs-unstable here because this test makes a script
+    # that when run will build `aeson` (used by `tests/cabal-simple`)
+    # and we currently do not build that on hydra for nixpkgs-2205 (used by `pkgs`).
+    # Using nixpkgs-unstable should allow buildkite to find what it needs
+    # in the hydra cache when it runs the script.
+    check-path-support = (import haskellNix.sources.nixpkgs-unstable nixpkgsArgs)
+        .buildPackages.callPackage ./scripts/check-path-support.nix {
+      inherit compiler-nix-name;
     };
   };
 

builder/comp-builder.nix

@@ -239,7 +239,7 @@ let
     (lib.concatMap (c: if c.isHaskell or false
       then builtins.attrValues (c.components.exes or {})
       else [c]) build-tools) ++
-    lib.optional (pkgconfig != []) buildPackages.pkgconfig;
+    lib.optional (pkgconfig != []) buildPackages.cabalPkgConfigWrapper;
 
   # Unfortunately, we need to wrap ghc commands for cabal builds to
   # work in the nix-shell. See ../doc/removing-with-package-wrapper.md.
@@ -370,7 +370,7 @@ let
          frameworks # Frameworks will be needed at link time
       # Not sure why pkgconfig needs to be propagatedBuildInputs but
       # for gi-gtk-hs it seems to help.
-      ++ builtins.concatLists pkgconfig
+      ++ map pkgs.lib.getDev (builtins.concatLists pkgconfig)
       ++ lib.optionals (stdenv.hostPlatform.isWindows)
         (lib.flatten component.libs
         ++ map haskellLib.dependToLib component.depends);

builder/default.nix

@@ -17,7 +17,7 @@ let
     # in the native case, it would be the same in the cross case however
     # we *really* want to build the Setup.hs on the build machine and not
     # have the stdenv confuse it with the target/host env.
-    inherit (buildPackages) stdenv pkgconfig;
+    inherit (buildPackages) stdenv;
     inherit buildPackages;
     inherit haskellLib nonReinstallablePkgs makeSetupConfigFiles;
   };

builder/hspkg-builder.nix

@@ -53,7 +53,7 @@ let
       ||
         # These are the dependencies of `Cabal`
         !builtins.elem package.identifier.name
-          ["nix-tools" "alex" "happy" "hscolour" "Cabal" "bytestring" "aeson" "time"
+          ["nix-tools" "alex" "happy" "hscolour" "Cabal" "Cabal-syntax" "bytestring" "aeson" "time"
            "filepath" "base-compat-batteries" "base-compat" "unix" "directory" "transformers"
            "containers" "binary" "mtl" "text" "process" "parsec"];
 

builder/make-config-files.nix

@@ -11,7 +11,9 @@ let
 
   flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
     echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags
-    echo "${field}: ${lib.concatStringsSep " " xs}" >> $out/cabal.config
+    ${lib.concatStrings (map (x: ''
+      echo "${field}: ${x}" >> $out/cabal.config
+    '') xs)}
   '';
 
   target-pkg = "${ghc.targetPrefix}ghc-pkg";
@@ -127,8 +129,11 @@ let
     done
 
     ${ # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
-      flagsAndConfig "package-db" ["clear" "$out/${packageCfgDir}"]
+       # However in `cabal.config` `cabal` requires `global` to be first.
+      flagsAndConfig "package-db" ["clear"]
     }
+    echo "package-db: global" >> $out/cabal.config
+    ${ flagsAndConfig "package-db" ["$out/${packageCfgDir}"] }
 
     echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags
 

builder/setup-builder.nix

@@ -1,4 +1,4 @@
-{ pkgs, stdenv, lib, buildPackages, haskellLib, ghc, nonReinstallablePkgs, hsPkgs, makeSetupConfigFiles, pkgconfig }@defaults:
+{ pkgs, stdenv, lib, buildPackages, haskellLib, ghc, nonReinstallablePkgs, hsPkgs, makeSetupConfigFiles }@defaults:
 
 let self =
 { component, package, name, src, enableDWARF ? false, flags ? {}, revision ? null, patches ? [], defaultSetupSrc
@@ -38,7 +38,7 @@ let
     (lib.concatMap (c: if c.isHaskell or false
       then builtins.attrValues (c.components.exes or {})
       else [c]) component.build-tools) ++
-    lib.optional (component.pkgconfig != []) pkgconfig;
+    lib.optional (component.pkgconfig != []) buildPackages.cabalPkgConfigWrapper;
 
   drv =
     stdenv.mkDerivation ({

builder/shell-for.nix

@@ -78,10 +78,12 @@ let
     ++ lib.optionals packageSetupDeps (map (p: p.setup.config)
          (lib.filter (p: p.buildType != "Simple") selectedPackages));
 
-  name = if lib.length selectedPackages == 1
+  identifierName = if lib.length selectedPackages == 1
     then "ghc-shell-for-${(lib.head selectedPackages).identifier.name}"
     else "ghc-shell-for-packages";
 
+  name = if (mkDrvArgs.name or null) == null then identifierName else mkDrvArgs.name;
+
   # We need to remove any dependencies which would bring in selected components (see above).
   packageInputs = removeSelectedInputs (lib.concatMap (cfg: cfg.depends) selectedConfigs)
     ++ additionalPackages;
@@ -93,11 +95,14 @@ let
   #
   # Also, we take care to keep duplicates out of the list, otherwise we may see
   # "Argument list too long" errors from bash when entering a shell.
-  uniqueInputs = inputList: builtins.listToAttrs (builtins.map (x: lib.nameValuePair (x.name) x) inputList);
-  unionComponentInputs = inputsSoFar: c:
-    inputsSoFar // uniqueInputs c.buildInputs // uniqueInputs c.propagatedBuildInputs;
-  systemInputs' = lib.lists.foldl unionComponentInputs {} selectedComponents;
-  systemInputs = removeSelectedInputs (builtins.attrValues systemInputs');
+  #
+  # Version of `lib.unique` that should be fast if the name attributes are unique
+  uniqueWithName = list:
+    lib.concatMap lib.unique (
+      builtins.attrValues (
+        builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") list));
+  allSystemInputs = lib.concatMap (c: c.buildInputs ++ c.propagatedBuildInputs) selectedComponents;
+  systemInputs = removeSelectedInputs (uniqueWithName allSystemInputs);
 
   nativeBuildInputs = removeSelectedInputs
     (lib.concatMap (c: c.executableToolDepends) selectedComponents);
@@ -111,14 +116,14 @@ let
     doExactConfig = false;
   };
   configFiles = makeConfigFiles {
-    fullName = args.name or name;
-    identifier.name = name;
+    fullName = name;
+    identifier.name = identifierName;
     inherit component enableDWARF;
     chooseDrv = p: if withHaddock && p ? haddock then p.haddock else p;
   };
   ghcEnv = ghcForComponent {
     inherit configFiles;
-    componentName = name;
+    componentName = identifierName;
     postInstall = lib.optionalString withHoogle' ''
       ln -s ${hoogleIndex}/bin/hoogle $out/bin
     '';
@@ -152,7 +157,7 @@ let
   mkDrvArgs = builtins.removeAttrs args ["packages" "components" "additional" "withHoogle" "tools"];
 in
   mkShell (mkDrvArgs // {
-    name = mkDrvArgs.name or name;
+    inherit name;
 
     buildInputs = systemInputs
       ++ mkDrvArgs.buildInputs or [];

ci.nix

@@ -10,7 +10,6 @@
   inherit (import ./ci-lib.nix { inherit pkgs; }) dimension platformFilterGeneric filterAttrsOnlyRecursive;
   inherit (pkgs.haskell-nix) sources;
   nixpkgsVersions = {
-    "R2111" = "nixpkgs-2111";
     "R2205" = "nixpkgs-2205";
     "unstable" = "nixpkgs-unstable";
   };
@@ -30,17 +29,14 @@
     # cabal-install and nix-tools plans.  When removing a ghc version
     # from here (so that is no longer cached) also remove ./materialized/ghcXXX.
     # Update supported-ghc-versions.md to reflect any changes made here.
-    nixpkgs.lib.optionalAttrs (nixpkgsName == "R2111") {
+    nixpkgs.lib.optionalAttrs (nixpkgsName == "R2205") {
       ghc865 = false;
-      ghc8107 = true;
-    } // nixpkgs.lib.optionalAttrs (nixpkgsName == "R2205") {
-      ghc865 = false;
-      ghc8107 = true;
+      ghc8107 = false;
     } // nixpkgs.lib.optionalAttrs (nixpkgsName == "unstable") {
       ghc865 = false;
       ghc884 = false; # Native version is used to boot 9.0.1
       ghc8107 = true;
-      ghc902 = true;
+      ghc902 = false;
       ghc924 = true;
     });
   systems = nixpkgsName: nixpkgs: compiler-nix-name: nixpkgs.lib.genAttrs (

default.nix

@@ -1,16 +1,19 @@
 {...}@args:
 
 let
+  pins = (__fromJSON (__readFile ./flake.lock)).nodes;
+  nixpkgsPin = pins.nixpkgs-2205.locked;
+  flakeCompatPin = pins.flake-compat.locked;
   nixpkgsSrc =
     builtins.fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/110a2c9ebbf5d4a94486854f18a37a938cfacbbb.tar.gz";
-      sha256 = "sha256-leWXLchbAbqOlLT6tju631G40SzQWPqaAXQG3zH1Imw=";
+      url = "https://github.com/NixOS/nixpkgs/archive/${nixpkgsPin.rev}.tar.gz";
+      sha256 = nixpkgsPin.narHash;
     };
   pkgs = args.pkgs or (import nixpkgsSrc {});
   flake-compat =
     pkgs.fetchzip {
-      url = "https://github.com/edolstra/flake-compat/archive/5523c47f13259b981c49b26e28499724a5125fd8.tar.gz";
-      sha256 = "sha256-7IySNHriQjzOZ88DDk6VDPf1GoUaOrOeUdukY62o52o=";
+      url = "https://github.com/input-output-hk/flake-compat/archive/${flakeCompatPin.rev}.tar.gz";
+      sha256 = flakeCompatPin.narHash;
     };
   self = import flake-compat {
     # We bypass flake-compat's rootSrc cleaning by evading its detection of this as a git