Fixes and optimize nix-tools usage

Author: hamishmack

builder/comp-builder.nix

@@ -371,19 +371,18 @@ let
       # Not sure why pkgconfig needs to be propagatedBuildInputs but
       # for gi-gtk-hs it seems to help.
       ++ builtins.concatLists pkgconfig
+      ++ configFiles.libDeps
       ++ lib.optionals (stdenv.hostPlatform.isWindows)
-        (lib.flatten component.libs
-        ++ map haskellLib.dependToLib component.depends);
+        (lib.flatten component.libs);
 
     buildInputs = lib.optionals (!stdenv.hostPlatform.isWindows)
-      (lib.flatten component.libs
-      ++ map haskellLib.dependToLib component.depends);
+        (lib.flatten component.libs);
 
     nativeBuildInputs =
       [ghc buildPackages.removeReferencesTo]
       ++ executableToolDepends;
 
-    outputs = ["out" "configFiles"]
+    outputs = ["out"]
       ++ (lib.optional enableSeparateDataOutput "data")
       ++ (lib.optional keepSource "source")
       ++ (lib.optional writeHieFiles "hie");
@@ -403,15 +402,12 @@ let
       '') + commonAttrs.prePatch;
 
     configurePhase = ''
-      echo A ${name}
+      configFiles=$(mktemp -d)
       ${configFiles.script}
-      echo B ${name}
       wrappedGhc=$(mktemp -d)
-      echo C ${name}
       ${shellWrappers.script}
-      echo D ${name}
       PATH=$wrappedGhc/bin:$PATH
-      echo E ${name}
+
       runHook preConfigure
       echo Configure flags:
       printf "%q " ${finalConfigureFlags}
@@ -602,4 +598,4 @@ let
   // lib.optionalAttrs (hardeningDisable != [] || stdenv.hostPlatform.isMusl) {
     hardeningDisable = hardeningDisable ++ lib.optional stdenv.hostPlatform.isMusl "pie";
   });
-in drv // { configFiles = drv.configFiles // configFiles; }; in self)
+in drv; in self)

builder/make-config-files.nix

@@ -49,21 +49,11 @@ let
   libDir         = "lib/${ghcCommand}-${ghc.version}";
   packageCfgDir  = "${libDir}/package.conf.d";
 
-  # Filters out only library packages that for this GHC target
-  # TODO investigate why this is needed
-  # TODO find out why p ? configFiles helps (for instance for `R1909.aarch64-unknown-linux-gnu.tests.cabal-22.run.x86_64-linux`)
   libDeps = map chooseDrv (
     (if enableDWARF then (x: map (p: p.dwarf or p) x) else x: x)
     ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
-    (lib.filter (p: (p ? configFiles) && p.configFiles.targetPrefix == ghc.targetPrefix)
-    (map getLibComponent component.depends)))
+    (map haskellLib.dependToLib component.depends))
   );
-  cfgFiles =
-    let xs = map
-      (p: "${p.configFiles}")
-      libDeps;
-    in lib.concatStringsSep "\" \"" xs;
-  libs = lib.concatMapStringsSep "\" \"" (p: "${p}") libDeps;
   script = ''
     ${target-pkg} init $configFiles/${packageCfgDir}
 
@@ -86,8 +76,8 @@ let
       done
     ''}
 
-    for l in "${cfgFiles}"; do
-      if [ -n "$l" ]; then
+    for l in "''${pkgsHostTarget[@]}"; do
+      if [ -d "$l/${packageCfgDir}" ]; then
         files=("$l/${packageCfgDir}/"*.conf)
         if (( ''${#files[@]} )); then
           cp -f "''${files[@]}" $configFiles/${packageCfgDir}
@@ -97,8 +87,8 @@ let
         fi
       fi
     done
-    for l in "${libs}"; do
-      if [ -n "$l" ]; then
+    for l in "''${pkgsHostTarget[@]}"; do
+      if [ -d "$l/package.conf.d" ]; then
         files=("$l/package.conf.d/"*.conf)
         if (( ''${#files[@]} )); then
           cp -f "''${files[@]}" $configFiles/${packageCfgDir}
@@ -133,11 +123,15 @@ let
       echo "allow-older: ${identifier.name}:*" >> $configFiles/cabal.config
     ''}
 
-    for p in ${lib.concatStringsSep " " libDeps}; do
-      cat $p/envDep >> $configFiles/ghc-environment
+    for p in $propagatedBuildInputs; do
+      if [ -e $p/envDep ]; then
+        cat $p/envDep >> $configFiles/ghc-environment
+      fi
       ${ lib.optionalString component.doExactConfig ''
-        cat $p/exactDep/configure-flags >> $configFiles/configure-flags
-        cat $p/exactDep/cabal.config >> $configFiles/cabal.config
+        if [ -d $p/exactDep ]; then
+          cat $p/exactDep/configure-flags >> $configFiles/configure-flags
+          cat $p/exactDep/cabal.config >> $configFiles/cabal.config
+        fi
       ''}
     done
     for p in ${lib.concatStringsSep " " (lib.remove "ghc" nonReinstallablePkgs')}; do
@@ -211,7 +205,7 @@ let
   '');
 in {
   inherit (ghc) targetPrefix;
-  inherit script drv ghcCommand ghcCommandCaps libDir packageCfgDir component;
+  inherit script libDeps drv ghcCommand ghcCommandCaps libDir packageCfgDir component;
   # Use ''${pkgroot} relative paths so that we can relocate the package database
   # along with referenced packages and still have it work on systems with
   # or without nix installed.

builder/setup-builder.nix

@@ -46,7 +46,9 @@ let
       src = cleanSrc'.root;
       buildInputs = component.libs
         ++ component.frameworks
-        ++ builtins.concatLists component.pkgconfig;
+        ++ builtins.concatLists component.pkgconfig
+        ++ configFiles.libDeps
+        ++ [ghc]; # Needs to be a build input so that the boot libraries are included
       nativeBuildInputs = [ghc] ++ executableToolDepends;
 
       passthru = {
@@ -69,6 +71,9 @@ let
 
       phases = ["unpackPhase" "patchPhase" "buildPhase" "installPhase"];
       buildPhase = ''
+        configFiles=$(mktemp -d)
+        mkdir -p $configFiles
+        ${configFiles.script}
         runHook preBuild
         if [[ ! -f ./Setup.hs  && ! -f ./Setup.lhs ]]; then
           cat ${defaultSetupSrc} > Setup.hs
@@ -77,7 +82,7 @@ let
           if [ -f $f ]; then
             echo Compiling package $f
             ghc $f -threaded ${if includeGhcPackage then "-package ghc " else ""
-                }-package-db ${configFiles.drv}/${configFiles.packageCfgDir} --make -o ./Setup
+                }-package-db $configFiles/${configFiles.packageCfgDir} --make -o ./Setup
           fi
         done
         [ -f ./Setup ] || (echo Failed to build Setup && exit 1)

lib/call-cabal-project-to-nix.nix

@@ -65,6 +65,7 @@ in
                             # any plutus-apps input being used for a
                             # package.
 , evalPackages
+, supportHpack ? false      # Run hpack on package.yaml files with no .cabal file
 , ...
 }@args:
 
@@ -453,7 +454,10 @@ let
   } // pkgs.lib.optionalAttrs (checkMaterialization != null) {
     inherit checkMaterialization;
   }) (evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
-    nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg cabal-install evalPackages.rsync evalPackages.gitMinimal ];
+    nativeBuildInputs = [
+      nix-tools.project.hsPkgs.nix-tools.components.exes.plan-to-nix
+      dummy-ghc dummy-ghc-pkg cabal-install evalPackages.rsync evalPackages.gitMinimal ]
+      ++ pkgs.lib.optional supportHpack nix-tools.project.hsPkgs.hpack.components.exes.hpack;
     # Needed or stack-to-nix will die on unicode inputs
     LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
     LANG = "en_US.UTF-8";
@@ -510,12 +514,20 @@ let
         if [ -e "$cabalFile" ]; then
           echo Ignoring $hpackFile as $cabalFile exists
         else
+          ${
           # warning: this may not generate the proper cabal file.
           # hpack allows globbing, and turns that into module lists
-          # without the source available (we cleaneSourceWith'd it),
+          # without the source available (we cleanSourceWith'd it),
           # this may not produce the right result.
-          echo No .cabal file found, running hpack on $hpackFile
-          hpack $hpackFile
+          if supportHpack
+            then '' 
+              echo No .cabal file found, running hpack on $hpackFile
+              hpack $hpackFile
+            ''
+            else ''
+              echo WARNING $hpackFile has no .cabal file and `supportHpack` was not set.
+            ''
+          }
         fi
       done
       )

overlays/bootstrap.nix

@@ -980,7 +980,7 @@ in {
           final.buildPackages.buildPackages.gitMinimal
           final.buildPackages.buildPackages.nix-prefetch-git ];
     in
-      final.buildPackages.symlinkJoin {
+      (final.buildPackages.symlinkJoin {
         name = "nix-tools";
         paths = exes;
         buildInputs = [ final.buildPackages.makeWrapper ];
@@ -992,7 +992,7 @@ in {
             wrapProgram "$out/bin/$prog" --prefix PATH : "${final.lib.makeBinPath tools}"
           done
         '';
-      };
+      }) // { inherit project; };
 
     # Memoize the cabal-install and nix-tools derivations by adding:
     #   haskell-nix.cabal-install.ghcXXX

overlays/default-setup.nix

@@ -37,7 +37,7 @@ let
       # version of Cabal bundled with GHC).
       cabalFromCabalInstall = final.buildPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name}.project.hsPkgs.Cabal.components.library;
 
-     in  ghc // rec {
+     in ghc // rec {
       defaultSetup = final.lib.mapAttrs (_: useCabalFromCabalInstall: setup-builder ({
         name = "${ghc.targetPrefix}default-Setup";
         component = {

overlays/haskell.nix

@@ -193,7 +193,7 @@ final: prev: {
                 name = "01-index.tar.gz-at-${at}";
                 url = "https://hackage.haskell.org/01-index.tar.gz";
                 downloadToTemp = true;
-                postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
+                postFetch = "${nix-tools.project.hsPkgs.nix-tools.components.exes.truncate-index}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
 
                 outputHashAlgo = "sha256";
                 outputHash = sha256;