Merge branch 'master' of github.com:input-output-hk/haskell.nix into circuithub

Author: ocharles

.github/workflows/pipeline.yml

@@ -227,13 +227,20 @@ jobs:
       - name: "Check that the haskell.nix roots do not require IFDs"
         run: nix build .#roots.x86_64-linux --accept-flake-config --option allow-import-from-derivation false
 
-  hydra-without-remote-builders:
+  hydra-without-remote-builders-ghc8107:
     runs-on: [self-hosted, linux]
     steps:
       - uses: actions/checkout@v4
-      - name: "Check that evaluation of hydra jobs works without using remote builders"
+      - name: "Check that evaluation of hydra jobs works without using remote builders for GHC 8.10.7"
         run: nix path-info --derivation .#requiredJobs.x86_64-darwin.required-unstable-ghc8107-native --show-trace --builders ''
 
+  hydra-without-remote-builders-ghc9101:
+    runs-on: [self-hosted, linux]
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Check that evaluation of hydra jobs works without using remote builders for GHC 9.10.1"
+        run: nix path-info --derivation .#requiredJobs.x86_64-darwin.required-unstable-ghc9101-native --show-trace --builders ''
+
   hix-cabal:
     runs-on: [self-hosted, linux]
     steps:

BUGLOG

@@ -0,0 +1,86 @@
+This file contains bugs we find while working on haskell.nix.  The format is as
+follow:
+<separator: 80 * '-'>
+YYYY-MM-DD: nix-job name
+
+<error>
+
+<discussion>
+
+--------------------------------------------------------------------------------
+2024-04-09 x86_64-linux.R2305.ghc8107.mingwW64.ghc
+
+/build/ghc62733_0/ghc_1.s:50:0: error:
+     Error: CFI instruction used without previous .cfi_startproc
+   |
+50 |         .cfi_escape 0x16, 0x07, 0x04, 0x77, 152, 65
+   | ^
+`x86_64-w64-mingw32-cc' failed in phase `Assembler'. (Exit code: 1)
+make[1]: *** [rts/ghc.mk:325: rts/dist/build/StgCRun.o] Error 1
+
+The source for this is
+> https://github.com/ghc/ghc/blob/1f02b7430b2fbab403d7ffdde9cfd006e884678e/rts/StgCRun.c#L433
+
+It appears that GCC C17 12.2.0 does _not_ emit .cfi_startproc / .cfi_endprocs
+whereas GCC C17 13.2.0 _does_. Specificall x86_64-w64-mingw32-cc. So this might
+be a cross compilation issue.
+
+The -g is hardcoded in
+https://github.com/ghc/ghc/blob/1f02b7430b2fbab403d7ffdde9cfd006e884678e/mk/config.mk.in#L361
+
+Turns out, this was disabled for anything but linux in https://github.com/ghc/ghc/commit/5b08e0c06e038448a63aa9bd7f163b23d824ba4b,
+hence we backport that patch to GHC-8.10 when targeting windows (to prevent mass rebuilds for
+other archs).
+
+--------------------------------------------------------------------------------
+2024-04-10 x86_64-linux.R2305.ghc902.mingwW64.ghc
+
+make[1]: *** [utils/hsc2hs/ghc.mk:22: utils/hsc2hs/dist-install/build/tmp/hsc2hs.exe] Error 1
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x2a): relocation truncated to fit: R_X86_64_32S against `.text'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x46): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.data'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x8b): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x95): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0xe3): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0xed): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x13b): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x145): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x193): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x19d): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x1eb): additional relocation overflows omitted from the output
+
+We notice `fake`, which is GHC failing to provide .file identifier in the source.
+We also see lots of R_X64_64_32S relocations, which are signed 32bit relocations.
+These fall with ASLR and high entropy base images from later binutils.
+
+The underlying issue is that GHC emits _absolute_ label loads (mov $... reg), instead
+of %rpi or other relative loads. This then leads to the linker emitting 32bit
+absolute relocation. With the final image being potentially loaded into high memory
+(e.g. dynamic base, and the base image being set to some high address), the linker
+starts falling over itself, because it simply can't resolve those absolute addresses
+in the 32bit slots.
+
+This was fixed in GHC upstream in https://gitlab.haskell.org/ghc/ghc/-/merge_requests/7449,
+while the patch in haskell.nix is a bit more pedestrian and just sets PIC on windows to
+always be on, and then uses the PIC pipeline.
+
+--------------------------------------------------------------------------------
+2024-06-18 x86_64-linux.unstable.ghc9101.ucrt64.tests.th-dlls-minimal.build
+
+0024:err:seh:call_stack_handlers invalid frame 00007FFFFF68EF18 (0000000000022000-0000000000220000)
+0024:err:seh:NtRaiseException Exception frame is not in stack limits => unable to dispatch exception.
+iserv-proxy: {handle: <socket: 11>}: GHCi.Message.remoteCall: end of file
+
+This is due to GHC mislinking GNU import libraries (dll.a). What happens is that
+GHC ends up creating GOT entries for function calls instead of PLT entries. The
+loader/linker in GHC for Windows has logic to lazy load .dll's as referenced. For
+this symbols get a dependency symbol attached, this could be a symbol indicating
+the DLL that needs to be loaded. While walking the dependencies to find the dll to
+load (or in some cases just the dependent symbol, not the dll), we override the
+symbol type with the one of the dependent symbol.  This however means we'll
+override the type of a symbol with the DATA type each time the symbol leads to a
+dllInstance to be loaded. Subsequently we end up creating a GOT entry instead of
+a PLT entry for the symbol, irrepsective of the original symbol being a code or
+data symbol. If code symbols end up getting GOT stubs, we see the above crash as
+the control flow jumps to the location of the stub, and instead of a PLT/jump
+island just lands in the address of the target symbol, which is in most cases
+non-sensical machine code.

build.nix

@@ -62,10 +62,10 @@ in rec {
         inherit evalPackages;
         src = pkgs.haskell-nix.sources."hls-2.2";
       };
-    } // pkgs.lib.optionalAttrs (ghcFromTo "9.0" "9.9") {
-      "hls-28" = tool compiler-nix-name "haskell-language-server" {
+    } // pkgs.lib.optionalAttrs (ghcFromTo "9.0" "9.11") {
+      "hls-29" = tool compiler-nix-name "haskell-language-server" {
         inherit evalPackages;
-        src = pkgs.haskell-nix.sources."hls-2.8";
+        src = pkgs.haskell-nix.sources."hls-2.9";
       };
     })
   );

builder/default.nix

@@ -13,7 +13,7 @@
 #
 # tl;dr: the builder must not re-introduce any reference to the build plan.
 
-{ pkgs, buildPackages, evalPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, nonReinstallablePkgs, hsPkgs, compiler }:
+{ pkgs, buildPackages, pkgsBuildBuild, evalPackages, stdenv, lib, haskellLib, ghc, compiler-nix-name, fetchurl, nonReinstallablePkgs, hsPkgs, compiler }:
 
 let
   # Builds a single component of a package.
@@ -64,13 +64,7 @@ let
       then pkgs.path
       else pkgs.haskell-nix.sources.nixpkgs-2205;
     nixpkgsHoogle = import (nixpkgs + /pkgs/development/haskell-modules/hoogle.nix);
-  in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc928" "hoogle" {
-        inherit evalPackages;
-        version = "5.0.18.3";
-        # index-state = pkgs.haskell-nix.internalHackageIndexState;
-        index-state = "2023-06-05T00:00:00Z";
-      }
-    }:
+  in { packages ? [], hoogle }:
     let
       haskellPackages = {
         # For musl we can use haddock from the buildGHC
@@ -85,7 +79,7 @@ let
 
   # Same as haskellPackages.shellFor in nixpkgs.
   shellFor = haskellLib.weakCallPackage pkgs ./shell-for.nix {
-    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib buildPackages evalPackages compiler;
+    inherit hsPkgs ghcForComponent makeConfigFiles hoogleLocal haskellLib pkgsBuildBuild evalPackages compiler;
     inherit (buildPackages) glibcLocales;
   };
 

builder/shell-for.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, buildPackages, evalPackages, compiler }:
+{ lib, stdenv, mkShell, glibcLocales, pkgconfig, ghcForComponent, makeConfigFiles, hsPkgs, hoogleLocal, haskellLib, pkgsBuildBuild, evalPackages, compiler }:
 
 { # `packages` function selects packages that will be worked on in the shell itself.
   # These packages will not be built by `shellFor`, but their
@@ -142,21 +142,17 @@ let
       pname = p.identifier.name;
       haddockDir = p.haddockDir;
     };
-  in hoogleLocal ({
+  in hoogleLocal {
     packages = map docPackage (haskellLib.flatLibDepends component);
 
-    # Need to add hoogle to hsPkgs.
-    # inherit (hsPkgs) hoogle;
-  } // (
-    lib.optionalAttrs (args ? tools && args.tools ? hoogle) {
-      hoogle = buildPackages.haskell-nix.hackage-tool (
-        haskellLib.versionOrModToMods args.tools.hoogle ++ [{
+    hoogle = pkgsBuildBuild.haskell-nix.hackage-tool (
+        lib.optionals (args ? tools && args.tools ? hoogle) (haskellLib.versionOrModToMods args.tools.hoogle)
+      ++ [{
           name = "hoogle";
           compiler-nix-name = compiler.nix-name;
           inherit evalPackages;
         }]);
-    }
-  ));
+    };
 
   mkDrvArgs = builtins.removeAttrs args ["packages" "components" "additional" "withHoogle" "tools"];
 in
@@ -168,12 +164,12 @@ in
     nativeBuildInputs = [ ghcEnv.drv ]
       ++ nativeBuildInputs
       ++ mkDrvArgs.nativeBuildInputs or []
-      ++ lib.attrValues (buildPackages.haskell-nix.tools' evalPackages compiler.nix-name tools)
+      ++ lib.attrValues (pkgsBuildBuild.haskell-nix.tools' evalPackages compiler.nix-name tools)
       # If this shell is a cross compilation shell include
       # wrapper script for running cabal build with appropriate args.
       # Includes `--with-compiler` in case the `cabal.project` file has `with-compiler:` in it.
       ++ lib.optional (ghcEnv.targetPrefix != "") (
-            buildPackages.writeShellScriptBin "${ghcEnv.targetPrefix}cabal" ''
+            pkgsBuildBuild.writeShellScriptBin "${ghcEnv.targetPrefix}cabal" ''
               exec cabal \
                 --with-ghc=${ghcEnv.targetPrefix}ghc \
                 --with-compiler=${ghcEnv.targetPrefix}ghc \

ci.nix

@@ -22,17 +22,22 @@
     "R2211" = inputs.nixpkgs-2211;
     "R2305" = inputs.nixpkgs-2305;
     "R2311" = inputs.nixpkgs-2311;
+    "R2405" = inputs.nixpkgs-2405;
     "unstable" = inputs.nixpkgs-unstable;
   };
 
   nixpkgsArgs = {
     # set checkMaterialization as per top-level argument
     overlays = [
       haskellNix.overlay
-      (_final: prev: {
+      (final: prev: {
         haskell-nix = prev.haskell-nix // {
           inherit checkMaterialization;
+          extraPkgconfigMappings = prev.haskell-nix.extraPkgconfigMappings or {} // {
+            "libsodium" = [ "libsodium-18" ];
+          };
         };
+        libsodium-18 = (final.callPackage (inputs.nixpkgs-2311 + "/pkgs/development/libraries/libsodium") {}).overrideAttrs (_: { dontDisableStatic = true; });
       })
     ];
     # Needed for dwarf tests
@@ -56,7 +61,7 @@
       # cabal-install and nix-tools plans.  When removing a ghc version
       # from here (so that is no longer cached) also remove ./materialized/ghcXXX.
       # Update supported-ghc-versions.md to reflect any changes made here.
-      nixpkgs.lib.optionalAttrs (nixpkgsName == "R2311") {
+      nixpkgs.lib.optionalAttrs (nixpkgsName == "R2405") {
         ghc94 = false;
         ghc96 = false;
         ghc98 = false;
@@ -120,8 +125,6 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: pinnedNixpkgsSrc:
           ghc = pkgs.buildPackages.haskell-nix.compiler.${compiler-nix-name};
         } // pkgs.lib.optionalAttrs runTests {
           inherit (build) tests tools maintainer-scripts maintainer-script-cache;
-        } // pkgs.lib.optionalAttrs (ifdLevel >= 2) {
-          inherit (pkgs.haskell-nix.iserv-proxy-exes.${compiler-nix-name}) iserv-proxy;
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
           hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit evalPackages compiler-nix-name; }).getComponent "exe:hello";
         });

compiler/ghc/default.nix

@@ -251,7 +251,9 @@ let
       compiler-nix-name =
         if builtins.compareVersions ghc-version "9.4.7" < 0
           then "ghc928"
-          else "ghc962";
+        else if buildPackages.haskell.compiler ? ghc964
+          then "ghc964"
+        else "ghc962";
     in
     buildPackages.haskell-nix.tool compiler-nix-name "hadrian" {
       compilerSelection = p: p.haskell.compiler;
@@ -272,7 +274,9 @@ let
           then ../../materialized/${compiler-nix-name}/hadrian-ghc981
         else if builtins.compareVersions ghc-version "9.9" < 0
           then ../../materialized/${compiler-nix-name}/hadrian-ghc98
-        else ../../materialized/${compiler-nix-name}/hadrian-ghc99;
+        else if builtins.compareVersions ghc-version "9.11" < 0
+          then ../../materialized/${compiler-nix-name}/hadrian-ghc910
+        else null;
       modules = [{
         reinstallableLibGhc = false;
         # Apply the patches in a way that does not require using something
@@ -652,7 +656,7 @@ stdenv.mkDerivation (rec {
     # We could add `configured-src` as an output of the ghc derivation, but
     # having it as its own derivation means it can be accessed quickly without
     # building GHC.
-    raw-src = stdenv.mkDerivation {
+    raw-src = evalPackages: evalPackages.stdenv.mkDerivation {
       name = name + "-raw-src";
       inherit
         version
@@ -701,11 +705,16 @@ stdenv.mkDerivation (rec {
           ${hadrian}/bin/hadrian ${hadrianArgs} _build/stage0/compiler/build/$a
           cp _build/stage0/compiler/build/$a compiler/GHC/Builtin/$a
         done
-      '' + lib.optionalString stdenv.isDarwin ''
+      '' + lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
         substituteInPlace mk/system-cxx-std-lib-1.0.conf \
           --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
         find . -name 'system*.conf*'
         cat mk/system-cxx-std-lib-1.0.conf
+      '' + lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+        substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+          --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+        find . -name 'system*.conf*'
+        cat mk/system-cxx-std-lib-1.0.conf
       '' + lib.optionalString (installStage1 && stdenv.targetPlatform.isMusl) ''
         substituteInPlace hadrian/cfg/system.config \
           --replace 'cross-compiling       = YES' \
@@ -786,11 +795,16 @@ stdenv.mkDerivation (rec {
     export XATTR=$(mktemp -d)/nothing
   '';
 } // lib.optionalAttrs useHadrian {
-  postConfigure = lib.optionalString stdenv.isDarwin ''
+  postConfigure = lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
     substituteInPlace mk/system-cxx-std-lib-1.0.conf \
       --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
     find . -name 'system*.conf*'
     cat mk/system-cxx-std-lib-1.0.conf
+  '' + lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+    substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+      --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+    find . -name 'system*.conf*'
+    cat mk/system-cxx-std-lib-1.0.conf
   '' + lib.optionalString (installStage1 && !haskell-nix.haskellLib.isCrossTarget && stdenv.targetPlatform.isMusl) ''
     substituteInPlace hadrian/cfg/system.config \
       --replace 'cross-compiling       = YES' \
@@ -850,12 +864,18 @@ stdenv.mkDerivation (rec {
         ${hadrian}/bin/hadrian ${hadrianArgs} binary-dist-dir
         cd _build/bindist/ghc-*
         ./configure --prefix=$out ${lib.concatStringsSep " " configureFlags}
-        ${lib.optionalString stdenv.isDarwin ''
+        ${lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
           substituteInPlace mk/system-cxx-std-lib-1.0.conf \
             --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
           substituteInPlace lib/package.conf.d/system-cxx-std-lib-1.0.conf \
             --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
         ''}
+        ${lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+          substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+            --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+          substituteInPlace lib/package.conf.d/system-cxx-std-lib-1.0.conf \
+            --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+        ''}
         mkdir -p utils
         cp -r ../../../utils/completion utils
         make install

default.nix

@@ -88,6 +88,12 @@ self // {
   pkgs-2305 = import self.inputs.nixpkgs-2305 (nixpkgsArgs // {
     localSystem = { inherit system; };
   });
+  pkgs-2311 = import self.inputs.nixpkgs-2311 (nixpkgsArgs // {
+    localSystem = { inherit system; };
+  });
+  pkgs-2405 = import self.inputs.nixpkgs-2405 (nixpkgsArgs // {
+    localSystem = { inherit system; };
+  });
   pkgs-unstable = import self.inputs.nixpkgs-unstable (nixpkgsArgs // {
     localSystem = { inherit system; };
   });