Merge pull request #12 from input-output-hk/builder-haddock

Dev environments

Author: rvl

.travis.yml

@@ -19,8 +19,4 @@ before_script:
 script:
 - nix-env -if https://github.com/cachix/cachix/tarball/master --extra-substituters https://cachix.cachix.org --trusted-public-keys 'cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cachix.cachix.org-1:eWNHQldwUO7G2VkjpnjDbWwy4KQ/HNxht7H4SSoMckM='
 - cachix use nix-tools
-- nix-build
-  -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/${CHANNEL}.tar.gz
-  test/default.nix
-  --cores 0
-  -j2
+- NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/${CHANNEL}.tar.gz NIX_BUILD_ARGS="--cores 0 --max-jobs 2" ./test/tests.sh

builder/comp-builder.nix

@@ -1,4 +1,4 @@
-{ stdenv, buildPackages, ghc, lib, pkgconfig, writeText, runCommand, haskellLib, nonReinstallablePkgs }:
+{ stdenv, buildPackages, ghc, lib, pkgconfig, writeText, runCommand, haskellLib, nonReinstallablePkgs, withPackage }:
 
 { componentId
 , component
@@ -16,8 +16,9 @@
 , preBuild ? null, postBuild ? null
 , preCheck ? null, postCheck ? null
 , preInstall ? null, postInstall ? null
+, shellHook ? null
 
-, doCheck ? component.doCheck || componentId.ctype == "test"
+, doCheck ? component.doCheck || haskellLib.isTest componentId
 , doCrossCheck ? component.doCrossCheck || false
 , dontPatchELF ? true
 , dontStrip ? true
@@ -27,7 +28,9 @@
 }:
 
 let
-  fullName = "${name}-${componentId.ctype}-${componentId.cname}";
+  fullName = if haskellLib.isAll componentId
+    then "${name}-all"
+    else "${name}-${componentId.ctype}-${componentId.cname}";
 
   flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
     echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags
@@ -44,18 +47,30 @@ let
     in map ({val,...}: val) closure;
 
   exactDep = pdbArg: p: ''
-    if id=$(${ghc.targetPrefix}ghc-pkg -v0 ${pdbArg} field ${p} id --simple-output); then
+    if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
       echo "--dependency=${p}=$id" >> $out/configure-flags
     fi
-    if ver=$(${ghc.targetPrefix}ghc-pkg -v0 ${pdbArg} field ${p} version --simple-output); then
+    if ver=$(target-pkg ${pdbArg} field ${p} version --simple-output); then
       echo "constraint: ${p} == $ver" >> $out/cabal.config
       echo "constraint: ${p} installed" >> $out/cabal.config
     fi
   '';
 
+  envDep = pdbArg: p: ''
+    if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
+      echo "package-id $id" >> $out/ghc-environment
+    fi
+  '';
+
   configFiles = runCommand "${fullName}-config" { nativeBuildInputs = [ghc]; } (''
     mkdir -p $out
-    ${ghc.targetPrefix}ghc-pkg -v0 init $out/package.conf.d
+
+    # Calls ghc-pkg for the target platform
+    target-pkg() {
+      ${ghc.targetPrefix}ghc-pkg "$@"
+    }
+
+    target-pkg init $out/package.conf.d
 
     ${lib.concatStringsSep "\n" (lib.mapAttrsToList flagsAndConfig {
       "extra-lib-dirs" = map (p: "${lib.getLib p}/lib") component.libs;
@@ -67,18 +82,26 @@ let
     # Note: we need to use --global-package-db with ghc-pkg to prevent it
     #       from looking into the implicit global package db when registering the package.
     ${lib.concatMapStringsSep "\n" (p: ''
-      ${ghc.targetPrefix}ghc-pkg -v0 describe ${p} | ${ghc.targetPrefix}ghc-pkg -v0 --force --global-package-db $out/package.conf.d register - || true
+      target-pkg describe ${p} | target-pkg --force --global-package-db $out/package.conf.d register - || true
     '') nonReinstallablePkgs}
 
     ${lib.concatMapStringsSep "\n" (p: ''
-      ${ghc.targetPrefix}ghc-pkg -v0 --package-db ${p}/package.conf.d dump | ${ghc.targetPrefix}ghc-pkg -v0 --force --package-db $out/package.conf.d register -
+      target-pkg --package-db ${p}/package.conf.d dump | target-pkg --force --package-db $out/package.conf.d register -
     '') flatDepends}
 
     # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
     ${flagsAndConfig "package-db" ["clear" "$out/package.conf.d"]}
 
     echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags
 
+    # Provide a GHC environment file
+    cat > $out/ghc-environment <<EOF
+    clear-package-db
+    package-db $out/package.conf.d
+    EOF
+    ${lib.concatMapStringsSep "\n" (p: envDep "--package-db ${p.components.library}/package.conf.d" p.identifier.name) component.depends}
+    ${lib.concatMapStringsSep "\n" (envDep "") (lib.remove "ghc" nonReinstallablePkgs)}
+
   '' + lib.optionalString component.doExactConfig ''
     echo "--exact-configuration" >> $out/configure-flags
     echo "allow-newer: ${package.identifier.name}:*" >> $out/cabal.config
@@ -115,14 +138,14 @@ let
       sed -i "s,dynamic-library-dirs: .*,dynamic-library-dirs: $dynamicLinksDir," $f
     done
   '' + ''
-    ${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d recache
+    target-pkg --package-db $out/package.conf.d recache
   '' + ''
-    ${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d check
+    target-pkg --package-db $out/package.conf.d check
   '');
 
   finalConfigureFlags = lib.concatStringsSep " " (
     [ "--prefix=$out"
-      "${componentId.ctype}:${componentId.cname}"
+      "${haskellLib.componentTarget componentId}"
       "$(cat ${configFiles}/configure-flags)"
       # GHC
       "--with-ghc=${ghc.targetPrefix}ghc"
@@ -145,6 +168,16 @@ let
       ++ component.configureFlags
   );
 
+  executableToolDepends = lib.concatMap (c: if c.isHaskell or false
+      then builtins.attrValues (c.components.exes or {})
+      else [c]) component.build-tools;
+
+  # Unfortunately, we need to wrap ghc commands for cabal builds to
+  # work in the nix-shell. See ../doc/removing-with-package-wrapper.md.
+  shellWrappers = withPackage {
+    inherit package configFiles;
+  };
+
 in stdenv.mkDerivation ({
   name = fullName;
 
@@ -154,6 +187,7 @@ in stdenv.mkDerivation ({
     inherit (package) identifier;
     config = component;
     inherit configFiles;
+    env = shellWrappers;
   };
 
   meta = {
@@ -167,6 +201,7 @@ in stdenv.mkDerivation ({
   };
 
   CABAL_CONFIG = configFiles + /cabal.config;
+  GHC_ENVIRONMENT = configFiles + /ghc-environment;
   LANG = "en_US.UTF-8";         # GHC needs the locale configured during the Haddock phase.
   LC_ALL = "en_US.UTF-8";
 
@@ -179,9 +214,7 @@ in stdenv.mkDerivation ({
   nativeBuildInputs =
     [ghc]
     ++ lib.optional (component.pkgconfig != []) pkgconfig
-    ++ lib.concatMap (c: if c.isHaskell or false
-      then builtins.attrValues (c.components.exes or {})
-      else [c]) component.build-tools;
+    ++ executableToolDepends;
 
   SETUP_HS = setup + /bin/Setup;
 
@@ -217,12 +250,12 @@ in stdenv.mkDerivation ({
   installPhase = ''
     runHook preInstall
     $SETUP_HS copy ${lib.concatStringsSep " " component.setupInstallFlags}
-    ${lib.optionalString (haskellLib.isLibrary componentId) ''
+    ${lib.optionalString (haskellLib.isLibrary componentId || haskellLib.isAll componentId) ''
       $SETUP_HS register --gen-pkg-config=${name}.conf
       ${ghc.targetPrefix}ghc-pkg -v0 init $out/package.conf.d
       ${ghc.targetPrefix}ghc-pkg -v0 --package-db ${configFiles}/package.conf.d -f $out/package.conf.d register ${name}.conf
     ''}
-    ${lib.optionalString (componentId.ctype == "test") ''
+    ${lib.optionalString (haskellLib.isTest componentId || haskellLib.isAll componentId) ''
       mkdir -p $out/${name}
       if [ -f "dist/build/${componentId.cname}/${componentId.cname}" ]; then
         cp dist/build/${componentId.cname}/${componentId.cname} $out/${name}/
@@ -233,6 +266,11 @@ in stdenv.mkDerivation ({
     ''}
     runHook postInstall
   '';
+
+  shellHook = ''
+    export PATH="${shellWrappers}/bin:$PATH"
+    ${toString shellHook}
+  '';
 }
 # patches can (if they like) depend on the version and revision of the package.
 // lib.optionalAttrs (patches != []) { patches = map (p: if builtins.isFunction p then p { inherit (package.identifier) version; inherit revision; } else p) patches; }

builder/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, writeText, runCommand, pkgconfig, nonReinstallablePkgs }:
+{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, writeText, runCommand, pkgconfig, nonReinstallablePkgs, withPackage }:
 
 { flags
 , package
@@ -22,6 +22,8 @@
 , preInstall
 , postInstall
 
+, shellHook
+
 , ...
 }@config:
 
@@ -67,11 +69,12 @@ let
       '';
     };
 
-  comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix { inherit ghc haskellLib nonReinstallablePkgs; };
+  comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix { inherit ghc haskellLib nonReinstallablePkgs withPackage; };
 
   buildComp = componentId: component: comp-builder {
     inherit componentId component package name src flags setup cabalFile patches revision
             preUnpack postUnpack preConfigure postConfigure preBuild postBuild preCheck postCheck preInstall postInstall
+            shellHook
             ;
   };
 

builder/with-package-wrapper.nix

@@ -0,0 +1,77 @@
+# This is a simplified version of the ghcWithPackages wrapper in
+# nixpkgs, adapted to work with the package database of a single
+# component.
+{ lib, stdenv, ghc, runCommand, lndir, makeWrapper
+}:
+
+{ package
+, configFiles
+, postBuild ? ""
+}:
+
+let
+  isGhcjs       = ghc.isGhcjs or false;
+  ghcCommand'    = if isGhcjs then "ghcjs" else "ghc";
+  ghcCommand = "${ghc.targetPrefix}${ghcCommand'}";
+  ghcCommandCaps= lib.toUpper ghcCommand';
+  libDir        = "$out/lib/${ghcCommand}-${ghc.version}";
+  docDir        = "$out/share/doc/ghc/html";
+  packageCfgDir = "${libDir}/package.conf.d";
+
+in runCommand "${ghc.name}-with-${package.identifier.name}" {
+  preferLocalBuild = true;
+  passthru = {
+    inherit (ghc) version meta;
+    baseGhc = ghc;
+    inherit package;
+  };
+} (
+  ''
+    . ${makeWrapper}/nix-support/setup-hook
+
+    # Start with a ghc...
+    mkdir -p $out/bin
+    ${lndir}/bin/lndir -silent ${ghc} $out
+
+    # ...and replace package database with the one from target package config.
+    rm -rf ${libDir}
+    mkdir -p ${libDir}
+    ln -s ${configFiles}/package.conf.d ${packageCfgDir}
+
+    # Wrap compiler executables with correct env variables.
+    # The NIX_ variables are used by the patched Paths_ghc module.
+    # The GHC_ENVIRONMENT variable forces ghc to use the build
+    # dependencies of the component.
+
+    for prg in ${ghcCommand} ${ghcCommand}i ${ghcCommand}-${ghc.version} ${ghcCommand}i-${ghc.version} runghc runhaskell; do
+      if [[ -x "${ghc}/bin/$prg" ]]; then
+        rm -f $out/bin/$prg
+        makeWrapper ${ghc}/bin/$prg $out/bin/$prg                           \
+          --set "NIX_${ghcCommandCaps}"        "$out/bin/${ghcCommand}"     \
+          --set "NIX_${ghcCommandCaps}PKG"     "$out/bin/${ghcCommand}-pkg" \
+          --set "NIX_${ghcCommandCaps}_DOCDIR" "${docDir}"                  \
+          --set "NIX_${ghcCommandCaps}_LIBDIR" "${libDir}"                  \
+          --set "${ghcCommandCaps}_ENVIRONMENT" "${configFiles}/ghc-environment"
+      fi
+    done
+
+    # Point ghc-pkg to the package database of the component using the
+    # GHC_PACKAGE_PATH variable.
+
+    for prg in ${ghcCommand}-pkg ${ghcCommand}-pkg-${ghc.version}; do
+      if [[ -x "${ghc}/bin/$prg" ]]; then
+        rm -f $out/bin/$prg
+        makeWrapper ${ghc}/bin/$prg $out/bin/$prg \
+          --set "${ghcCommandCaps}_PACKAGE_PATH" "${configFiles}/package.conf.d"
+      fi
+    done
+
+    # fixme: check if this is needed
+    # haddock was referring to the base ghc, https://github.com/NixOS/nixpkgs/issues/36976
+    if [[ -x "${ghc}/bin/haddock" ]]; then
+      rm -f $out/bin/haddock
+      makeWrapper ${ghc}/bin/haddock $out/bin/haddock    \
+        --set "NIX_${ghcCommandCaps}_LIBDIR" "${libDir}"
+    fi
+  ''
+)

doc/removing-with-package-wrapper.md

@@ -0,0 +1,66 @@
+# `ghcWithPackages` wrapper removal
+
+The current [Nixpkgs Haskell infrastructure][nixpkgs-haskell] and `haskell.nix` both
+provide a `ghcWithPackages` derivation which contains shell script
+wrappers that wrap `ghc` and `ghc-pkg`.
+
+In the Nixpkgs Haskell infrastructure, the wrapper scripts are used
+for building Haskell packages. However, in `haskell.nix`, the wrappers
+are only used for development environments.
+
+The wrapper scripts provide a `ghc` command that "knows" about the
+package set and has all Haskell package dependencies available to it.
+
+We would like to remove the wrapper scripts, but it's currently not
+possible to configure all build tools using environment variables
+alone.
+
+## Plain `ghc`
+
+When using `ghc` or `ghci` by itself, the `GHC_ENVIRONMENT` variable
+can point to a configuration file containing an exact package
+set. This works quite well.
+
+## `ghc-pkg`
+
+The package tool `ghc-pkg` does not recognize `GHC_ENVIRONMENT`, but
+does recognize a `GHC_PACKAGE_PATH` pointing to a `package.conf.d`.
+
+This works well. However, the `cabal` command will refuse to start if
+`GHC_PACKAGE_PATH` is set.
+
+## `Setup.hs`
+
+When invoking `Setup.hs configure`, the package database is provided
+with the `--package-db` argument and exact dependencies in the package
+set can be provided as `--dependency` arguments.
+
+The `haskell.nix` component builder uses `Setup.hs` with these
+command-line options to build Haskell packages.
+
+## `cabal new-build`
+
+Cabal-install will observe the `CABAL_CONFIG` environment variable,
+which points to a cabal config file. This config file can provide a
+`package-db` value, but it can't specify exact versions of packages.
+
+Cabal is designed to solve dependencies, not simply take the package
+set which is given to it.
+
+Therefore, `cabal` does not use `GHC_ENVIRONMENT`, but instead creates
+its own environment file. It will not accept `--dependency` arguments.
+
+As far as I know, the best way to force `cabal` to take a pre-computed
+package set is to use a `new-freeze` file. However there is no
+environment variable (or config file entry) which can specify a path
+to a freeze file.
+
+Specifying a `package-db` path in the cabal config file is not enough
+for it to successfully resolve dependencies.
+
+As mentioned before, `cabal` does not work when `GHC_PACKAGE_PATH` is
+set. The best way to work around this is to wrap `ghc` and `ghc-pkg`
+in shell scripts.
+
+
+[nixpkgs-haskell]: https://nixos.org/nixpkgs/manual/#users-guide-to-the-haskell-infrastructure