Cross comp. & callCabalProjectToNix/callStackToNix (#202)

* Cross comp. & callCabalProjectToNix/callStackToNix

This fixes an issue where the these IFD functions fail because they try
to build and run a cross compiled nix-tools on the build system.

It may also help if there are cross compilation issues with
setup-depends (they also access the buildPackages).

* Align stack-to-nix ifd with plan-to-nix ifd

This also fixes the restricted/pure eval issues with callStackToNix.

I've also made the tests run in restricted mode to catch these kinds of
issues.

Author: hamishmack

call-stack-to-nix.nix

@@ -62,7 +62,13 @@ let
   };
   stackage = import stackageSrc;
 
-  packages = self: ({
+  packages = pkgs: self: (rec {
+    inherit pkgs; # Make pkgs available (it is the underlying nixpkgs)
+
+     # Packages built to run on the build platform, not the host platform
+    buildPackages = pkgs.buildPackages.lib.makeScope pkgs.buildPackages.newScope
+      (packages pkgs.buildPackages);
+
     # Utility functions for working with the component builder.
     haskellLib = let hl = import ./lib { inherit (pkgs) lib; haskellLib = hl; }; in hl;
 
@@ -125,17 +131,20 @@ let
     haskellPackages = self.snapshots."lts-13.26";
 
     # Programs for generating Nix expressions from Cabal and Stack
-    # files. We need to make sure we build this from the buildPackages,
-    # we never want to actually cross compile nix-tools on it's own.
-    nix-tools = pkgs.buildPackages.callPackage ./nix-tools {
+    # files. This version of nix-tools may be cross compiled.
+    # We probably never want to actually cross compile nix-tools on
+    # it's own.
+    nix-tools-cross-compiled = pkgs.callPackage ./nix-tools {
       inherit fetchExternal cleanSourceHaskell;
-      hpack = pkgs.buildPackages.haskell.lib.justStaticExecutables
-        (pkgs.buildPackages.haskellPackages.hpack);
+      hpack = pkgs.haskell.lib.justStaticExecutables
+        (pkgs.haskellPackages.hpack);
       inherit (self) mkCabalProjectPkgSet;
     };
-
-    # Function to call stackToNix
-    callStackToNix = self.callPackage ./call-stack-to-nix.nix {};
+    # While `nix-tools-cross-compiled` may be cross compiled,
+    # getting it from `buildPackages` we should get
+    # nix-tools suitable for running on the build system.
+    nix-tools = buildPackages.nix-tools-cross-compiled;
+    # TODO perhaps there is a cleaner way to get a suitable nix-tools.
 
     # Snapshots of Hackage and Stackage, converted to Nix expressions,
     # regularly updated.
@@ -180,15 +189,22 @@ let
 
     update-index-state-hashes = self.callPackage ./scripts/update-index-state-hashes.nix {};
 
+    # Function to call stackToNix
+    callStackToNix = import ./lib/call-stack-to-nix.nix {
+      pkgs = buildPackages.pkgs;
+      inherit (buildPackages.pkgs) runCommand;
+      inherit (buildPackages) nix-tools;
+    };
+
     # Takes a haskell src directory runs cabal new-configure and plan-to-nix.
     # Resulting nix files are added to nix-plan subdirectory.
-    callCabalProjectToNix = import ./lib/cabalProjectToNix.nix {
+    callCabalProjectToNix = import ./lib/call-cabal-project-to-nix.nix {
       index-state-hashes = import indexStateHashesPath;
-      inherit (self) dotCabal;
-      inherit pkgs;
-      inherit (pkgs) runCommand cabal-install ghc symlinkJoin cacert;
-      inherit (pkgs.haskellPackages) hpack;
-      inherit (self) nix-tools;
+      inherit (buildPackages) dotCabal;
+      pkgs = buildPackages.pkgs; # buildPackages;
+      inherit (buildPackages.pkgs.haskellPackages) hpack;
+      inherit (buildPackages.pkgs) runCommand cabal-install ghc symlinkJoin cacert;
+      inherit (buildPackages) nix-tools;
     };
 
     # References to the unpacked sources, for caching in a Hydra jobset.
@@ -198,4 +214,4 @@ let
   });
 
 in
-  pkgs.lib.makeScope pkgs.newScope packages
+  pkgs.lib.makeScope pkgs.newScope (packages pkgs)

default.nix

@@ -17,7 +17,7 @@ let
         type == "directory" ||
         pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ];
     };
-  plan = runCommand "plan" {
+  plan = runCommand "plan-to-nix-pkgs" {
     nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];
   } ''
     tmp=$(mktemp -d)
@@ -31,7 +31,9 @@ let
     find . -name package.yaml -exec hpack "{}" \;
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
-    HOME=${dotCabal { inherit index-state; sha256 = index-sha256; }} cabal new-configure
+    HOME=${dotCabal { inherit index-state; sha256 = index-sha256; }} cabal new-configure \
+        --with-ghc=${ghc.targetPrefix}ghc \
+        --with-ghc-pkg=${ghc.targetPrefix}ghc-pkg
 
     export LANG=C.utf8 # Needed or stack-to-nix will die on unicode inputs
     mkdir -p $out
@@ -76,7 +78,7 @@ in
   #   # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
   #   buildInputs = [ plan src ];
   # }
-  runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
+  runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
     mkdir $out
     # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
     rsync -a "${src}/" "$out/"

lib/cabalProjectToNix.nix renamed to lib/call-cabal-project-to-nix.nix

@@ -0,0 +1,38 @@
+/* The function obtained when this is applied to a package set calls
+ * the stack-to-nix tool on a supplied source set and then
+ * imports the resulting pkgs.nix. The application of this function
+ * to a source path can thus be used directly as the input to mkStackPackageSet
+ *
+ * see also `call-cabal-project-to-nix`!
+ */
+{ runCommand, nix-tools, pkgs }:
+{ src, stackYaml ? null }:
+let 
+  stack = runCommand "stack-to-nix-pkgs" {
+    nativeBuildInputs = [ nix-tools pkgs.nix-prefetch-git ];
+  } ''
+    export LANG=C.utf8 # Needed or stack-to-nix will die on unicode inputs
+    mkdir -p $out
+
+    (cd $out && stack-to-nix --stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml} -o .)
+
+    # We need to strip out any references to $src, as those won't
+    # be accessable in restricted mode.
+    for nixf in $(find $out -name "*.nix" -type f); do
+      substituteInPlace $nixf --replace "${src}" "."
+    done
+
+    # move pkgs.nix to default.nix ensure we can just nix `import` the result.
+    mv $out/pkgs.nix $out/default.nix
+  '';
+in 
+runCommand "stack-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
+  mkdir $out
+  # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
+  rsync -a "${src}/" "$out/"
+  rsync -a ${stack}/ $out/
+  # Rsync will have made $out read only and that can cause problems when
+  # nix sandboxing is enabled (since it can prevent nix from moving the directory
+  # out of the chroot sandbox).  
+  chmod +w $out
+''

lib/call-stack-to-nix.nix

@@ -0,0 +1,11 @@
+with import <nixpkgs> {};
+let this = import ./. {}; in
+stdenv.mkDerivation rec {
+  name = "env";
+  env = buildEnv { name = name; paths = buildInputs; };
+  buildInputs = [
+    this.ghc
+    this.cabal-install
+    this.nix-tools
+  ];
+}

nixpkgs/hackage-src.json

@@ -4,13 +4,13 @@ with stdenv.lib;
 
 let
   pkgSet = mkStackPkgSet {
-    stack-pkgs = callStackToNix { src = ./.; };
+    stack-pkgs = import (callStackToNix {
+      src = ../stack-simple;
+    });
     pkg-def-extras = [];
     modules = [];
   };
-
   packages = pkgSet.config.hsPkgs;
-
 in
   stdenv.mkDerivation {
     name = "callStackToNix-test";

shell.nix

@@ -18,8 +18,8 @@ in pkgs.recurseIntoAttrs {
   stack-simple = haskell.callPackage ./stack-simple {};
   snapshots = haskell.callPackage ./snapshots {};
   shell-for = haskell.callPackage ./shell-for {};
-  # callStackToNix = haskell.callPackage ./callStackToNix {};
-  # callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
+  callStackToNix = haskell.callPackage ./call-stack-to-nix {};
+  callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
 
   # Run unit tests with: nix-instantiate --eval --strict -A unit.tests
   # An empty list means success.

test/callStackToNix/.gitignore renamed to test/call-stack-to-nix/.gitignore

@@ -12,7 +12,11 @@ rm -rvf */cabal.project.local */.ghc.environment* */dist */dist-newstyle */.stac
 echo >& 2
 
 printf "*** Running the nix-build tests...\n" >& 2
-nix build $NIX_BUILD_ARGS --no-link --keep-going -f ./default.nix
+nix build $NIX_BUILD_ARGS \
+   -I . -I .. \
+   --option restrict-eval true \
+   --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
+   --no-link --keep-going -f default.nix
 echo >& 2
 
 printf "*** Running the unit tests... " >& 2