Merge remote-tracking branch 'origin/master' into hpc-patch

# Conflicts:
#	lib/cover-project.nix

Author: hamishmack

.buildkite/pipeline.yml

@@ -38,3 +38,15 @@ steps:
     command:
       - nix-build build.nix -A maintainer-scripts.check-path-support --argstr compiler-nix-name ghc8107 -o check-path-support.sh
       - ./check-path-support.sh
+
+  - label: 'Check that the haskell.nix roots do not require IFDs'
+    command:
+      - nix build .#roots.x86_64-linux --option allow-import-from-derivation false
+    agents:
+      system: x86_64-linux
+
+  - label: 'Check that evaluation of hydra jobs works without using remote builders'
+    command:
+      - nix-instantiate release.nix -A required-unstable-ghc8107-x86_64-darwin-native --show-trace --builders ''
+    agents:
+      system: x86_64-linux

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -6,6 +6,9 @@ docs/reference/modules.md
 result*
 /nt
 
+# https://github.com/nix-community/nix-direnv
+.direnv
+
 # Created by https://www.gitignore.io/api/haskell,emacs,vim
 
 ### Emacs ###

build.nix

@@ -8,8 +8,9 @@ in
 { nixpkgs ? haskellNix.sources.nixpkgs-2111
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
+, evalPackages ? import nixpkgs nixpkgsArgs
 , nixpkgsForHydra ? haskellNix.sources.nixpkgs-2105
-, pkgsForHydra ? import nixpkgsForHydra nixpkgsArgs
+, pkgsForHydra ? import nixpkgsForHydra (nixpkgsArgs // { inherit (pkgs) system; })
 , ifdLevel ? 1000
 , compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
 }:
@@ -19,14 +20,21 @@ let
   buildHaskell = pkgs.buildPackages.haskell-nix;
   tool = buildHaskell.tool;
 in rec {
-  tests = import ./test/default.nix { inherit pkgs ifdLevel compiler-nix-name; };
+  tests = import ./test/default.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
 
   tools = pkgs.lib.optionalAttrs (ifdLevel >= 3) (
     pkgs.recurseIntoAttrs ({
-      cabal-latest = tool compiler-nix-name "cabal" "latest";
-    } // pkgs.lib.optionalAttrs (!__elem compiler-nix-name ["ghc901" "ghc902" "ghc921" "ghc922"]) {
-      hls-latest = tool compiler-nix-name "haskell-language-server" "latest";
-      hlint-latest = tool compiler-nix-name "hlint" (if compiler-nix-name == "ghc865" then "3.2.7" else "latest");
+      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; };
+      hlint-latest = tool compiler-nix-name "hlint" {
+        inherit evalPackages;
+        version = {
+            "ghc865" = "3.2.8";
+            "ghc882" = "3.3.6";
+            "ghc883" = "3.3.6";
+            "ghc884" = "3.3.6";
+          }.compiler-nix-name or "latest";
+      };
+      hls-latest = tool compiler-nix-name "haskell-language-server" { inherit evalPackages; };
     })
   );
 
@@ -78,7 +86,7 @@ in rec {
   # These are pure parts of maintainer-script so they can be built by hydra
   # and added to the cache to speed up buildkite.
   maintainer-script-cache = pkgs.recurseIntoAttrs (
-      (pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+      (pkgs.lib.optionalAttrs (pkgsForHydra.system == "x86_64-linux") {
         inherit (maintainer-scripts) check-hydra;
       })
     // (pkgs.lib.optionalAttrs (ifdLevel > 2) {

builder/comp-builder.nix

@@ -50,6 +50,7 @@ let self =
 , writeHieFiles ? component.writeHieFiles
 
 , ghcOptions ? component.ghcOptions
+, contentAddressed ? component.contentAddressed
 
 # Options for Haddock generation
 , doHaddock ? component.doHaddock  # Enable haddock and hoogle generation
@@ -99,9 +100,10 @@ let
     && !stdenv.hostPlatform.isMusl
     && builtins.compareVersions defaults.ghc.version "8.10.2" >= 0;
 
-  ghc = if enableDWARF then defaults.ghc.dwarf else
-        if smallAddressSpace then defaults.ghc.smallAddressSpace else defaults.ghc;
-  setup = if enableDWARF then drvArgs.setup.dwarf else drvArgs.setup;
+  ghc = (if enableDWARF then (x: x.dwarf) else (x: x)) (
+        (if smallAddressSpace then (x: x.smallAddressSpace) else (x: x)) defaults.ghc);
+  setup = (if enableDWARF then (x: x.dwarf) else (x: x)) (
+        (if smallAddressSpace then (x: x.smallAddressSpace) else (x: x)) drvArgs.setup);
 
   # TODO fix cabal wildcard support so hpack wildcards can be mapped to cabal wildcards
   canCleanSource = !(cabal-generator == "hpack" && !(package.cleanHpack or false));
@@ -315,7 +317,13 @@ let
     componentDrv = drv;
   };
 
-  drv = stdenv.mkDerivation (commonAttrs // {
+  contentAddressedAttrs = lib.optionalAttrs contentAddressed {
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  };
+                    
+  drv = stdenv.mkDerivation (commonAttrs // contentAddressedAttrs // {
     pname = nameOnly;
     inherit (package.identifier) version;
 
@@ -348,6 +356,14 @@ let
       description = package.synopsis or "";
       license = haskellLib.cabalToNixpkgsLicense package.license;
       platforms = if platforms == null then lib.platforms.all else platforms;
+    } // lib.optionalAttrs (haskellLib.isExecutableType componentId) {
+      # Set main executable name for executable components, so that `nix run` in
+      # nix flakes will work correctly. When not set, `nix run` would (typically
+      # erroneously) deduce the executable name from the derivation name and
+      # attempt to run, for example,
+      # `/nix/store/...-project-exe-app-0.1.0.0/bin/project-exe-app` instead of
+      # `/nix/store/...-project-exe-app-0.1.0.0/bin/app`.
+      mainProgram = exeName;
     };
 
     propagatedBuildInputs =
@@ -404,7 +420,17 @@ let
       (lib.optionalString stdenv.hostPlatform.isWindows ''
         export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
       '') +
-      (if stdenv.hostPlatform.isGhcjs then ''
+      # The following could be refactored but would lead to many rebuilds
+
+      # In case of content addressed components we need avoid parallel building (passing -j1)
+      # in order to have a deterministic output and therefore avoid potential situations
+      # where the binary cache becomes useless
+      # See also https://gitlab.haskell.org/ghc/ghc/-/issues/12935
+      (if contentAddressed then ''
+        runHook preBuild
+        $SETUP_HS build ${haskellLib.componentTarget componentId} -j1 ${lib.concatStringsSep " " setupBuildFlags}
+        runHook postBuild
+      '' else if stdenv.hostPlatform.isGhcjs then ''
         runHook preBuild
         # https://gitlab.haskell.org/ghc/ghc/issues/9221
         $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}

builder/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs, compiler }:
+{ pkgs, buildPackages, evalPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs, compiler }:
 
 let
   # Builds a single component of a package.
@@ -46,6 +46,7 @@ let
   hoogleLocal = let
     nixpkgsHoogle = import (pkgs.path + /pkgs/development/haskell-modules/hoogle.nix);
   in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc8107" "hoogle" {
+        inherit evalPackages;
         version = "5.0.18.3";
         index-state = pkgs.haskell-nix.internalHackageIndexState;
       }
@@ -66,7 +67,7 @@ let
 
   # Same as haskellPackages.shellFor in nixpkgs.
   shellFor = haskellLib.weakCallPackage pkgs ./shell-for.nix {
-    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib buildPackages compiler;
+    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib buildPackages evalPackages compiler;
     inherit (buildPackages) glibcLocales;
   };
 

builder/hspkg-builder.nix

@@ -39,41 +39,28 @@ let
 
   defaultSetupSrc = if stdenv.hostPlatform.isGhcjs then ./Setup.ghcjs.hs else ./Setup.hs;
 
-  # Get the Cabal lib used to build `cabal-install`.
-  # To avoid infinite recursion we have to leave this out for packages
-  # needed to build `cabal-install`.
-  # We always do this for ghcjs as the patched version of Cabal is needed.
-  cabalLibDepends = lib.optional (
-    stdenv.hostPlatform.isGhcjs || (
-        builtins.elem compiler-nix-name["ghc865" "ghc884"]
-      &&
-        !builtins.elem package.identifier.name
-          ["nix-tools" "alex" "happy" "hscolour" "Cabal" "bytestring" "aeson" "time"
-           "filepath" "base-compat-batteries" "base-compat" "unix" "directory" "transformers"
-           "containers" "binary" "mtl" "text" "process" "parsec"]
-      )
-    )
-    buildPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name}.project.hsPkgs.Cabal.components.library;
+  # This is the `Cabal` library that was built for `cabal-install` to use.
+  # It makes sense to use this version (when possible) because it will match the behavior of
+  # building with `cabal-install` (including fixes that may not be in the
+  # version of Cabal bundled with GHC).
+  cabalFromCabalInstall = buildPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name}.project.hsPkgs.Cabal.components.library;
 
-  # This logic is needed so that we don't get duplicate packages if we
-  # add a custom Cabal package to the dependencies.  That way custom
-  # setups won't complain about e.g. binary from the Cabal dependencies
-  # and binary from the global package-db.
-  nonReinstallablePkgs = if (
-    stdenv.hostPlatform.isGhcjs || (
-        builtins.elem compiler-nix-name["ghc865" "ghc884"]
-      &&
+  # Check there is no chance we are building `cabalFromCabalInstall`.  Using `cabalFromCabalInstall`
+  # to build itseld would cause infinite recursion.
+  useCabalFromCabalInstall =
+        # `cabalFromCabalInstall` is not cross compiled
+        stdenv.buildPlatform != stdenv.hostPlatform
+      ||
+        # These are the dependencies of `Cabal`
         !builtins.elem package.identifier.name
           ["nix-tools" "alex" "happy" "hscolour" "Cabal" "bytestring" "aeson" "time"
            "filepath" "base-compat-batteries" "base-compat" "unix" "directory" "transformers"
-           "containers" "binary" "mtl" "text" "process" "parsec"]
-      )
-    ) then [] else null;
+           "containers" "binary" "mtl" "text" "process" "parsec"];
 
   defaultSetup = setup-builder ({
     name = "${ghc.targetPrefix}default-Setup";
     component = {
-      depends = config.setup-depends ++ cabalLibDepends;
+      depends = config.setup-depends ++ lib.optional useCabalFromCabalInstall cabalFromCabalInstall;
       libs = [];
       frameworks = [];
       doExactConfig = false;
@@ -107,7 +94,13 @@ let
       cat ${defaultSetupSrc} > $out/Setup.hs
     '';
     inherit defaultSetupSrc;
-  } // (if nonReinstallablePkgs == null then {} else { inherit nonReinstallablePkgs; }));
+  } // lib.optionalAttrs useCabalFromCabalInstall {
+    # This is needed so that we don't get duplicate packages when we
+    # add a custom Cabal package to the dependencies.  That way custom
+    # setups won't complain about e.g. binary from the Cabal dependencies
+    # and binary from the global package-db.
+    nonReinstallablePkgs = [];
+  });
 
   # buildPackages.runCommand "default-Setup" { nativeBuildInputs = [(ghc.passthru.buildGHC or ghc)]; } ''
   #   cat ${defaultSetupSrc} > Setup.hs
@@ -138,7 +131,7 @@ in rec {
   checks = pkgs.recurseIntoAttrs (builtins.mapAttrs
     (_: d: haskellLib.check d)
       (lib.filterAttrs (_: d: d.config.doCheck) components.tests));
-  inherit (package) identifier detailLevel isLocal isProject;
+  inherit (package) identifier detailLevel isLocal isProject buildType;
   inherit setup cabalFile;
   isHaskell = true;
   inherit src;

builder/make-config-files.nix

@@ -11,7 +11,9 @@ let
 
   flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
     echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags
-    echo "${field}: ${lib.concatStringsSep " " xs}" >> $out/cabal.config
+    ${lib.concatStrings (map (x: ''
+      echo "${field}: ${x}" >> $out/cabal.config
+    '') xs)}
   '';
 
   target-pkg = "${ghc.targetPrefix}ghc-pkg";
@@ -91,7 +93,8 @@ let
           (map (p: "${lib.getBin p}/bin")
                (lib.flatten component.libs ++ lib.concatLists component.pkgconfig));
       "extra-include-dirs" = map (p: "${lib.getDev p}/include") (lib.flatten component.libs);
-      "extra-framework-dirs" = map (p: "${p}/Library/Frameworks") component.frameworks;
+      "extra-framework-dirs" = lib.optionals (stdenv.hostPlatform.isDarwin)
+        (map (p: "${p}/Library/Frameworks") component.frameworks);
     })}
 
     ghc=${ghc}
@@ -126,8 +129,11 @@ let
     done
 
     ${ # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
-      flagsAndConfig "package-db" ["clear" "$out/${packageCfgDir}"]
+       # However in `cabal.config` `cabal` requires `global` to be first.
+      flagsAndConfig "package-db" ["clear"]
     }
+    echo "package-db: global" >> $out/cabal.config
+    ${ flagsAndConfig "package-db" ["$out/${packageCfgDir}"] }
 
     echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags
 

builder/setup-builder.nix

@@ -8,9 +8,13 @@ let self =
 , preInstall ? component.preInstall , postInstall ? component.postInstall
 , cleanSrc ? haskellLib.cleanCabalComponent package component "setup" src
 , nonReinstallablePkgs ? defaults.nonReinstallablePkgs
+, smallAddressSpace ? false
 }@drvArgs:
 
 let
+  ghc = (if enableDWARF then (x: x.dwarf) else (x: x)) (
+        (if smallAddressSpace then (x: x.smallAddressSpace) else (x: x)) defaults.ghc);
+
   cleanSrc' = haskellLib.rootAndSubDir cleanSrc;
 
   fullName = "${name}-setup";
@@ -53,6 +57,7 @@ let
         cleanSrc = cleanSrc';
         inherit configFiles;
         dwarf = self (drvArgs // { enableDWARF = true; });
+        smallAddressSpace = self (drvArgs // { smallAddressSpace = true; });
       };
 
       meta = {
@@ -71,13 +76,12 @@ let
         for f in Setup.hs Setup.lhs; do
           if [ -f $f ]; then
             echo Compiling package $f
-            ghc $f -threaded '' + (if includeGhcPackage then "-package ghc " else "")
-                + ''-package-db ${configFiles}/${configFiles.packageCfgDir} --make -o ./Setup
-            setup=$(pwd)/Setup
+            ghc $f -threaded ${if includeGhcPackage then "-package ghc " else ""
+                }-package-db ${configFiles}/${configFiles.packageCfgDir} --make -o ./Setup
           fi
         done
         [ -f ./Setup ] || (echo Failed to build Setup && exit 1)
-        runHook preBuild
+        runHook postBuild
       '';
 
       installPhase = ''

builder/shell-for.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, buildPackages, compiler }:
+{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, buildPackages, evalPackages, compiler }:
 
 { # `packages` function selects packages that will be worked on in the shell itself.
   # These packages will not be built by `shellFor`, but their
@@ -71,14 +71,19 @@ let
   removeSelectedInputs =
     lib.filter (input: !(isSelectedComponent input));
 
-  # The configs of all the selected components
+  # The configs of all the selected components.
+  # This excludes the `setup` dependencies of `Simple` packages, because
+  # `cabal-install` does not build a `Setup` executable for `Simple` packages.
   selectedConfigs = map (c: c.config) selectedComponents
-    ++ lib.optionals packageSetupDeps (map (p: p.setup.config) selectedPackages);
+    ++ lib.optionals packageSetupDeps (map (p: p.setup.config)
+         (lib.filter (p: p.buildType != "Simple") selectedPackages));
 
-  name = if lib.length selectedPackages == 1
+  identifierName = if lib.length selectedPackages == 1
     then "ghc-shell-for-${(lib.head selectedPackages).identifier.name}"
     else "ghc-shell-for-packages";
 
+  name = if (mkDrvArgs.name or null) == null then identifierName else mkDrvArgs.name;
+
   # We need to remove any dependencies which would bring in selected components (see above).
   packageInputs = removeSelectedInputs (lib.concatMap (cfg: cfg.depends) selectedConfigs)
     ++ additionalPackages;
@@ -101,14 +106,14 @@ let
     doExactConfig = false;
   };
   configFiles = makeConfigFiles {
-    fullName = args.name or name;
-    identifier.name = name;
+    fullName = name;
+    identifier.name = identifierName;
     inherit component enableDWARF;
     chooseDrv = p: if withHaddock && p ? haddock then p.haddock else p;
   };
   ghcEnv = ghcForComponent {
     inherit configFiles;
-    componentName = name;
+    componentName = identifierName;
     postInstall = lib.optionalString withHoogle' ''
       ln -s ${hoogleIndex}/bin/hoogle $out/bin
     '';
@@ -130,21 +135,26 @@ let
     # inherit (hsPkgs) hoogle;
   } // (
     lib.optionalAttrs (args ? tools && args.tools ? hoogle) {
-      hoogle = buildPackages.haskell-nix.tool compiler.nix-name "hoogle" args.tools.hoogle;
+      hoogle = buildPackages.haskell-nix.hackage-tool (
+        haskellLib.versionOrModToMods args.tools.hoogle ++ [{
+          name = "hoogle";
+          compiler-nix-name = compiler.nix-name;
+          inherit evalPackages;
+        }]);
     }
   ));
 
   mkDrvArgs = builtins.removeAttrs args ["packages" "components" "additional" "withHoogle" "tools"];
 in
   mkShell (mkDrvArgs // {
-    name = mkDrvArgs.name or name;
+    inherit name;
 
     buildInputs = systemInputs
       ++ mkDrvArgs.buildInputs or [];
     nativeBuildInputs = [ ghcEnv ]
       ++ nativeBuildInputs
       ++ mkDrvArgs.nativeBuildInputs or []
-      ++ lib.attrValues (buildPackages.haskell-nix.tools compiler.nix-name tools)
+      ++ lib.attrValues (buildPackages.haskell-nix.tools' evalPackages compiler.nix-name tools)
       # If this shell is a cross compilation shell include
       # wrapper script for running cabal build with appropriate args.
       # Includes `--with-compiler` in case the `cabal.project` file has `with-compiler:` in it.