Call cabal project to nix (now with hackage-truncate)

default.nix

@@ -38,11 +38,6 @@ let
   # overridden with NIX_PATH.
   fetchExternal = import ./lib/fetch-external.nix;
 
-  mkHackageIndex = indexState: import ./lib/hackageIndex.nix {
-    inherit (pkgs) runCommand cabal-install;
-    inherit indexState;
-  };
-
   # All packages from Hackage as Nix expressions
   hackage = import (fetchExternal {
     name     = "hackage-exprs-source";
@@ -132,10 +127,44 @@ let
     # Make this handy overridable fetch function available.
     inherit fetchExternal;
 
+    # Produce a fixed output derivation from a moving target (hackage index tarball)
+    hackageTarball = { index-state, sha256 }:
+      pkgs.runCommand "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}" {
+        nativeBuildInputs = [ pkgs.curl ];
+        outputHashAlgo = "sha256";
+        outputHash = sha256;
+        # We'll use fetchurl's result in an env var ...
+        HACKAGE_INDEX = builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz";
+        # ... and mark that impure.  That way we can
+        # ensure the sore path stays the same and doesn't
+        # depend on the fetchurl result path.
+        impureEnvVars = [ "HACKAGE_INDEX" ];
+      }
+      ''
+      ${self.nix-tools}/bin/truncate-index -o $out -i $HACKAGE_INDEX -s ${index-state}
+      '';
+
+    mkLocalHackageRepo = import ./mk-local-hackage-repo { inherit (self) hackageTarball; inherit pkgs; };
+
+    dotCabal = { index-state, sha256 }@args:
+      pkgs.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.cabal-install ]; } ''
+        mkdir -p $out/.cabal
+        cat <<EOF > $out/.cabal/config
+        repository cached
+          url: file:${self.mkLocalHackageRepo args}
+          secure: True
+          root-keys:
+          key-threshold: 0
+        EOF
+        mkdir -p $out/.cabal/packages/cached
+        HOME=$out cabal new-update cached
+      '';
+
+
     # Takes a haskell src directory runs cabal new-configure and plan-to-nix.
     # Resulting nix files are added to nix-plan subdirectory.
     callCabalProjectToNix = import ./lib/cabalProjectToNix.nix {
-      inherit mkHackageIndex;
+      inherit (self) dotCabal;
       inherit pkgs;
       inherit (pkgs) runCommand cabal-install ghc;
       inherit (pkgs.haskellPackages) hpack;

lib/cabalProjectToNix.nix

@@ -1,7 +1,7 @@
-{ mkHackageIndex, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }:
+{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }:
 let defaultGhc = ghc;
     defaultCabalInstall = cabal-install;
-in { hackageIndexState, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }:
+in { index-state, index-sha256 ? import ./index-state-hashes.nix index-state, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }:
 let
   cabalFiles =
     pkgs.lib.cleanSourceWith {
@@ -14,7 +14,7 @@ let
          # cabal-install versions before 2.4 will generate insufficient plan information.
          then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix"
          else runCommand "plan" {
-    nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync ];
+    nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];
   } ''
     tmp=$(mktemp -d)
     cd $tmp
@@ -25,7 +25,7 @@ let
     # without the source available (we cleaneSourceWith'd it),
     # this may not produce the right result.
     find . -name package.yaml -exec hpack "{}" \;
-    HOME=${mkHackageIndex hackageIndexState} cabal new-configure
+    HOME=${dotCabal { inherit index-state; sha256 = index-sha256; }} cabal new-configure
 
     export LANG=C.utf8 # Needed or stack-to-nix will die on unicode inputs
     mkdir -p $out
@@ -54,9 +54,25 @@ let
     mv $out/pkgs.nix $out/default.nix
   '';
 in
-  runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.xorg.lndir pkgs.rsync ]; } ''
+  # TODO: We really want this (symlinks) instead of copying the source over each and
+  #       every time.  However this will not work with sandboxed builds.  They won't
+  #       have access to `plan` or `src` paths.  So while they will see all the
+  #       links, they won't be able to read any of them.
+  #
+  #       We should be able to fix this if we propagaed the build inputs properly.
+  #       As we are `import`ing the produced nix-path here, we seem to be losing the
+  #       dependencies though.
+  #
+  #       I guess the end-result is that ifd's don't work well with symlinks.
+  #
+  # symlinkJoin {
+  #   name = "plan-and-src";
+  #   # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
+  #   buildInputs = [ plan src ];
+  # }
+  runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
     mkdir $out
     # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
-    lndir -silent "${src}" "$out"
+    rsync -a "${src}/" "$out/"
     rsync -a ${plan}/ $out/
   ''

lib/index-state-hashes.nix

@@ -0,0 +1,51 @@
+dt: {
+  "2019-04-01-00T00:00:00Z" = "a46ed96a1aa0510d1b54d508ae14dc6e460ff89ae41f0bd075aef87ba8c8ae73 ";
+  "2019-04-02-00T00:00:00Z" = "56dca813015122e0d633cb53bf8d11fb0c43a2e518b93cbbf93b0bdbf00e0c99 ";
+  "2019-04-03-00T00:00:00Z" = "b7a162996feec90e9acdff8bfb85a46ccbcafbbf0772739da45aef4e56480198 ";
+  "2019-04-04-00T00:00:00Z" = "6c890e98501539389807c7f7353b40af9bdd9e84a816adaccb79244ab1e506a0 ";
+  "2019-04-05-00T00:00:00Z" = "905b0ae09f4929b19b0885dcef785a45142bd6a7673e26979d6bc7044dedfd29 ";
+  "2019-04-06-00T00:00:00Z" = "2face70f256bc0a556e645f1f843c196411a97c78ded610443f3814b0783e5dc ";
+  "2019-04-07-00T00:00:00Z" = "f837f74181bd81120904c0e72b7c479faf178d1f2172a1442e167e7213b4a433 ";
+  "2019-04-08-00T00:00:00Z" = "86ff75e355e8b72757becb7761a1cd90b45b60f0f4777f8a137f30a27c766d65 ";
+  "2019-04-09-00T00:00:00Z" = "c00dc1146184db0743dce73d7bb53dca0fe75dff6634f61f11a6326984dba4ad ";
+  "2019-04-10-00T00:00:00Z" = "09cbd82c5f4a42e69299564a1494f5c2ff9fefde0663441a1878d351833a278c ";
+  "2019-04-11-00T00:00:00Z" = "072da9339e8c60b5e911659c110b0317a4e7890e30ce77e113edce1c373434d5 ";
+  "2019-04-12-00T00:00:00Z" = "9b14849542745d86529afa0f2cd2db989ea32a22a9a46d56fd3bc659339bdc28 ";
+  "2019-04-13-00T00:00:00Z" = "4aeba1860afc057b3faca85a289313a148a3392b80e5feab2263abfe8f9af155 ";
+  "2019-04-14-00T00:00:00Z" = "63a804787e63db9283b948ccc8ffafb487809efe60151295499e9d95230146fa ";
+  "2019-04-15-00T00:00:00Z" = "ef8607a4bf9e62052a428111b6decb5ef99e6da9dc885bf1bb51e593c4a97f9f ";
+  "2019-04-16-00T00:00:00Z" = "451781d772888fbabc9b3efa5b612972ada8b10d9f1e3e4ef9e13e0055b26d52 ";
+  "2019-04-17-00T00:00:00Z" = "af1169f894221a57d0c19a24ae656e65b37f571256f24586ee288f1848b781a4 ";
+  "2019-04-18-00T00:00:00Z" = "686c07e94d86c3b3bfc7faa98f8e92123cc338a3cccb961aab69cce26cb9b8df ";
+  "2019-04-19-00T00:00:00Z" = "da95cb1cb086671a046db5c1d49e81a641975ac875d87a1760b730fd44d592c3 ";
+  "2019-04-20-00T00:00:00Z" = "1e215bdab78c14262f34e4c49e1f21af4bcb307cd87cd6dcb13a148c38b9e2a1 ";
+  "2019-04-21-00T00:00:00Z" = "0fa42b89bdd55becc51eb6e0697d491d4c2f0205acc929f1f6ec46031ea4e37f ";
+  "2019-04-22-00T00:00:00Z" = "2d8925ef46b1ebb2c5d5a4b2084e24ecc700aece3d459b07b33997e2dae557e0 ";
+  "2019-04-23-00T00:00:00Z" = "b8c1ed659457be7d9b36f6f912d60b4e0960f19675315ede84503829346155d6 ";
+  "2019-04-24-00T00:00:00Z" = "6342a0cdc02a3d4d49cc35a71f3a7f8e50814deb7610440a66837df69f13d6c2 ";
+  "2019-04-25-00T00:00:00Z" = "b0624fb282a7540dc696325d95c81efdfcc00f2ad12738dc6e368609ebeb2258 ";
+  "2019-04-26-00T00:00:00Z" = "8990dba9bf5657b5b263a28ef5f0a94593977fb44e7306a332c011e672c95aa1 ";
+  "2019-04-27-00T00:00:00Z" = "42928f2ebc26e97f265be57a49b57fe0a0f64e55cd7e1a2ac4cafe2923a07011 ";
+  "2019-04-28-00T00:00:00Z" = "1f1062646f08f565f58df6db14293a2f6faceecae5be4938d8ac9c223a9c37cc ";
+  "2019-04-29-00T00:00:00Z" = "895bbce8a69b4fa4f0238dae9fc171eb57e9f56e71d9b31a5bb59f379db5298e ";
+  "2019-04-30-00T00:00:00Z" = "e793c2ee42dc48c738b7e014bdf23d3b5106f0d2a943cc1a3ab6b39e24b092b2 ";
+  "2019-05-01-00T00:00:00Z" = "1bfd413657fc405af03a6be09813a51b4a06a58aca3d3b1309de48b68ab7989e ";
+  "2019-05-02-00T00:00:00Z" = "0c15661a4d602db8bc6ceeb637daeaa3513a97881a0690770bf16e6b5117096f ";
+  "2019-05-03-00T00:00:00Z" = "6ec0bde7ba7efa8878f0a0ecf292a2ee5f439ae1687d00e201f2ad1bdb1f00d9 ";
+  "2019-05-04-00T00:00:00Z" = "e279e2f613a6609874a29192e7e2c8dd4cbb98bc9d4da1ede8a4347f1484fcf5 ";
+  "2019-05-05-00T00:00:00Z" = "6e47ddad859c1a848a45a9cf480be7c00966fd8b1497cf778acad344443165c8 ";
+  "2019-05-06-00T00:00:00Z" = "e67d97b349980006c2041225900d2d7fc672c7ec64fb4e2f1276403a468d72a9 ";
+  "2019-05-07-00T00:00:00Z" = "436d95d1de9c79c28081ad739f026ac1c9629549f4dcc153ec2fbfa57998c072 ";
+  "2019-05-08-00T00:00:00Z" = "c75741834dc028e5e9968ce68d3d3381e22182246750cb6810f7b72e4803b39b ";
+  "2019-05-09-00T00:00:00Z" = "8e65a6b0460b7cdffacab757e86c8c479bacc329523ebb1044dd7eb1e3c51c37 ";
+  "2019-05-10-00T00:00:00Z" = "2cbcd3df939e0def72fc425753bd24bb803031d10bb5a9259f456c6e33d9d68d ";
+  "2019-05-11-00T00:00:00Z" = "f8fd06d5571990bf7787f8aaff9b233f53ae816c6a8b2e3c193556dd376355f4 ";
+  "2019-05-12-00T00:00:00Z" = "4651555d1bee27ee57643e0046eac1ec0b7a37c2da20be25b40b1775fb524402 ";
+  "2019-05-13-00T00:00:00Z" = "3cda810efb958168fe8e01dc67f550b43b052c42c2d481e4941476b9165d2fd4 ";
+  "2019-05-14-00T00:00:00Z" = "6a26a74e2cf1f8cbf110c35967befbfa4c3b941c56d69f33981444a432bba9b6 ";
+  "2019-05-15-00T00:00:00Z" = "617f36845a950ed52192326d016a7c072e53cf62f30a0dcdad552fbf02966110 ";
+  "2019-05-16-00T00:00:00Z" = "b5017246f47d403a5f3ed887bab30f6c39da411d8397477f6962375741a07935 ";
+  "2019-05-17-00T00:00:00Z" = "2b269610ca0cdaeef36d70bd7bbe13a2a3425dda3873c89c9752704796281a78 ";
+  "2019-05-18-00T00:00:00Z" = "d8ec971679393cb24a20035c752f2295e0ee8aabd1177e3902a40a94dd167529 ";
+  "2019-05-19-00T00:00:00Z" = "8a61ec8f6748e92763fd35e36d3fabcde205ff1f8cdf7c08c942a69c6344f040 ";
+}.dt or null

mk-local-hackage-repo/default.nix

@@ -0,0 +1,45 @@
+# Create a local hackare repo, we can use as a repository in a cabal config
+#
+# This will include:
+# - 01-index.tar.gz (the index file)
+# - root.json and
+# - mirrors.json as metadata items.
+# - snapshot.json that records the index, root and mirrors.
+# - timestamp.json that will record the snapshot.json
+#
+# This is all part of The Update Framework (TUF) and the specific implementation
+# cabal-install (via hackage-security) does of it.
+#
+# We will create a completely unsigned bare repository.  Using signing keys within
+# nix would be pointless as we'd have to hardcode them to produce the same output
+# reproducably.
+#
+{ pkgs, hackageTarball }:
+{ index-state, sha256 }@args:
+let index = hackageTarball args; in
+pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.nix ]; } ''
+mkdir -p $out
+export expires="4000-01-01T00:00:00Z"
+
+ln -sf ${index} $out/01-index.tar.gz
+export index_md5=$(nix-hash --flat --type md5 ${index})
+export index_sha256=$(nix-hash --flat --type sha256 ${index})
+export index_length=$(stat --printf="%s" ${index})
+
+substituteAll ${./root.json} $out/root.json
+export root_md5=$(nix-hash --flat --type md5 $out/root.json)
+export root_sha256=$(nix-hash --flat --type sha256 $out/root.json)
+export root_length=$(stat --printf="%s" $out/root.json)
+
+substituteAll ${./mirrors.json} $out/mirrors.json
+export mirrors_md5=$(nix-hash --flat --type md5 $out/mirrors.json)
+export mirrors_sha256=$(nix-hash --flat --type sha256 $out/mirrors.json)
+export mirrors_length=$(stat --printf="%s" $out/mirrors.json)
+
+substituteAll ${./snapshot.json} $out/snapshot.json
+export snapshot_md5=$(nix-hash --flat --type md5 $out/snapshot.json)
+export snapshot_sha256=$(nix-hash --flat --type sha256 $out/snapshot.json)
+export snapshot_length=$(stat --printf="%s" $out/snapshot.json)
+
+substituteAll ${./timestamp.json} $out/timestamp.json
+''

mk-local-hackage-repo/mirrors.json

@@ -0,0 +1,9 @@
+{
+  "signatures": [ ],
+  "signed": {
+    "_type": "Mirrorlist",
+    "expires": "@expires@",
+    "mirrors": [],
+    "version": 1
+  }
+}

mk-local-hackage-repo/root.json

@@ -0,0 +1,34 @@
+{
+  "signatures": [],
+  "signed": {
+    "_type": "Root",
+    "expires": "@expires@",
+    "keys": {
+    },
+    "roles": {
+      "mirrors": {
+        "keyids": [ ],
+        "threshold": 0
+      },
+      "root": {
+        "keyids": [ ],
+        "threshold": 0
+      },
+      "snapshot": {
+        "keyids": [ ],
+        "threshold": 0
+      },
+      "targets": {
+        "keyids": [
+        ],
+        "threshold": 0
+      },
+      "timestamp": {
+        "keyids": [
+        ],
+        "threshold": 0
+      }
+    },
+    "version": 1
+  }
+}

mk-local-hackage-repo/snapshot.json

@@ -0,0 +1,31 @@
+{
+  "signatures": [],
+  "signed": {
+    "_type": "Snapshot",
+    "expires": "@expires@",
+    "meta": {
+      "<repo>/01-index.tar.gz": {
+        "hashes": {
+          "md5": "@index_md5@",
+          "sha256": "@index_sha256@"
+        },
+        "length": @index_length@
+      },
+      "<repo>/root.json": {
+        "hashes": {
+          "md5": "@root_md5@",
+          "sha256": "@root_sha256@"
+        },
+        "length": @root_length@
+      },
+      "<repo>/mirrors.json": {
+        "hashes": {
+          "md5": "@mirrors_md5@",
+          "sha256": "@mirrors_sha256@"
+        },
+        "length": @mirrors_length@
+      }
+    },
+    "version": 1
+  }
+}

mk-local-hackage-repo/timestamp.json

@@ -0,0 +1,17 @@
+{
+  "signatures": [],
+  "signed": {
+    "_type": "Timestamp",
+    "expires": "@expires@",
+    "meta": {
+      "<repo>/snapshot.json": {
+        "hashes": {
+          "md5": "@snapshot_md5@",
+          "sha256": "@snapshot_sha256@"
+        },
+        "length": @snapshot_length@
+      }
+    },
+    "version": 1
+  }
+}

test/call-cabal-project-to-nix/default.nix

@@ -5,15 +5,15 @@ with stdenv.lib;
 let
   pkgSet = mkCabalProjectPkgSet {
     plan-pkgs = import (callCabalProjectToNix {
-      hackageIndexState = "2019-04-24T21:34:04Z";
+      index-state = "2019-04-30T00:00:00Z";
       # reuse the cabal-simple test project
       src = ../cabal-simple;
     });
   };
   packages = pkgSet.config.hsPkgs;
 in
   stdenv.mkDerivation {
-    name = "callStackToNix-test";
+    name = "call-cabal-project-to-nix-test";
 
     buildCommand = ''
       exe="${packages.cabal-simple.components.exes.cabal-simple}/bin/cabal-simple"

test/default.nix

@@ -1,4 +1,5 @@
-{ pkgs ? import nixpkgs {}
+{ system ? builtins.currentSystem
+, pkgs ? import nixpkgs { inherit system; }
 , nixpkgs ? ../nixpkgs
 , haskell ? pkgs.callPackage ../. { }
 }:
@@ -17,8 +18,7 @@ in {
   stack-simple = haskell.callPackage ./stack-simple {};
   callStackToNix = haskell.callPackage ./callStackToNix {};
 
-  # Disabled -- doesn't work in a sandboxed build
-  # callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
+  callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
 
   # Run unit tests with: nix-instantiate --eval --strict -A unit
   # An empty list means success.