Merge pull request #47 from input-output-hk/rvl/hackage-stackage

Provide hackage and stackage from haskell.nix

Author: rvl

README.org

@@ -1,4 +1,34 @@
-* haskell.nix
+* Alternative Haskell Infrastructure for Nixpkgs
 
-This repository contains the runtime system for [[https://github.com/input-output-hk/nix-tools][nix-tools]].
-Please see [[https://github.com/input-output-hk/nix-tools][nix-tools]] for more details.
+[[https://travis-ci.org/input-output-hk/haskell.nix][https://travis-ci.org/input-output-hk/haskell.nix.svg?branch=master]]
+
+=haskell.nix= is an experimental new builder for Haskell packages.
+
+It works by automatically translating your Cabal or Stack project and
+its dependencies into Nix code. Most of your dependencies are already
+translated, so you generally won't have too much generated code.
+
+For full documentation, see https://input-output-hk.github.io/haskell.nix
+
+** Quickstart
+
+This will download and build =nix-tools=.
+
+#+begin_src sh
+nix build -f https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz nix-tools -o nt
+./nt/bin/cabal-to-nix --help
+#+end_src
+
+** Related repos
+
+The =haskell.nix= repository contains the runtime system for building
+Haskell packages in Nix. It depends on other repos, which are:
+
+- [[https://github.com/input-output-hk/nix-tools][nix-tools]] — provides the programs for generating Nix expressions from
+  Haskell projects.
+
+- [[https://github.com/input-output-hk/hackage.nix][hackage.nix]] — the latest contents of the [[https://hackage.haskell.org/][Hackage]] databases,
+  converted to Nix expressions.
+
+- [[https://github.com/input-output-hk/stackage.nix][stackage.nix]] — all of the [[https://www.stackage.org/][Stackage]] snapshots, converted to Nix
+  expressions.

default.nix

@@ -1,3 +1,6 @@
+{ pkgs ? import <nixpkgs> {}
+}:
+
 let
   # pkg-def's may reference boot packages, but those
   # are not guaranteed to be available on hackage, as
@@ -13,19 +16,83 @@ let
                     then filterAttrs (k: _: !(builtins.elem k boot-pkgs)) v
                     else v)
              (pkg-def hackage);
-in
-hackage: let haskell = rec {
+
   # ghc hackage patches.
   # these are patches that turn hackage packages into the same as the ones
   # ghc ships with the supposedly same version. See GHC Track Issue: 16199
   ghcHackagePatches = import ./patches;
 
-  compat = import ./compat.nix;
+  compat = import ./lib/compat.nix;
+
+  # Utility function for downloading a pinned git repo, that can be
+  # overridden with NIX_PATH.
+  fetchExternal = import ./lib/fetch-external.nix;
+
+  # All packages from Hackage as Nix expressions
+  hackage = import (fetchExternal {
+    name     = "hackage-exprs-source";
+    specJSON = ./hackage-src.json;
+    override = "hackage";
+  });
+
+  # The set of all Stackage snapshots
+  stackage = import (fetchExternal {
+    name     = "stackage-snapshot-source";
+    specJSON = ./stackage-src.json;
+    override = "stackage";
+  });
+
+  packages = self: ({
+    # Utility functions for working with the component builder.
+    haskellLib = let hl = import ./lib { inherit (pkgs) lib; haskellLib = hl; }; in hl;
 
-  mkPkgSet
-    = { pkgs, pkg-def, pkg-def-overlays ? [], modules ? [] }@args:
-      import ./package-set.nix (args // { inherit hackage; pkg-def = strip-pkg-def pkgs pkg-def; });
+    # Create a Haskell package set based on a cabal build plan (plan-to-nix)
+    # and Nix expressions representing cabal packages (cabal-to-nix).
+    mkPkgSet =
+      { pkg-def  # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix)
+      , pkg-def-overlays ? [] # Additional packages to augment the Base package set `pkg-def` with.
+      , modules ? []
+      }@args:
 
-  mkNewPkgSet = args: builtins.trace "DEPRECATED: use mkPkgSet instead of mkNewPkgSet" (mkPkgSet args);
+      import ./package-set.nix (args // {
+        inherit hackage pkgs;
+        pkg-def = strip-pkg-def pkgs pkg-def;
+      });
 
-}; in haskell
+    # Create a Haskell package set based on a Stack configuration.
+    mkStackPkgSet =
+      { stack-pkgs  # Path to the output of stack-to-nix
+      , pkg-def-overlays ? []
+      , modules ? []
+      }@args:
+
+      let
+        # The Stackage release referenced in the stack config
+        pkg-def = stackage.${stack-pkgs.resolver};
+        # The compiler referenced in the stack config
+        compiler = (stack-pkgs.overlay hackage).compiler or (pkg-def hackage).compiler;
+      in self.mkPkgSet {
+        inherit pkg-def;
+        pkg-def-overlays = [ stack-pkgs.overlay ] ++ pkg-def-overlays;
+        modules = [ ghcHackagePatches.${compiler.nix-name} ] ++ modules;
+      };
+
+    # Programs for generating Nix expressions from Cabal and Stack
+    # files.
+    nix-tools = self.callPackage ./nix-tools {
+      inherit fetchExternal;
+    };
+
+    # Snapshots of Hackage and Stackage, converted to Nix expressions,
+    # regularly updated.
+    inherit hackage stackage;
+
+    # Scripts for keeping Hackage and Stackage up to date.
+    maintainer-scripts = {
+      update-hackage = self.callPackage ./scripts/update-hackage.nix {};
+      update-stackage = self.callPackage ./scripts/update-stackage.nix {};
+    };
+  });
+
+in
+  pkgs.lib.makeScope pkgs.newScope packages

doc/maintainer-scripts.md

@@ -0,0 +1,18 @@
+# Updating Hackage and Stackage Nix expressions
+
+The [`hackage.nix`](https://github.com/input-output-hk/hackage.nix)
+and [`stackage.nix`](https://github.com/input-output-hk/stackage.nix)
+repos and corresponding files `hackage-src.json` and
+`stackage-src.json` will be regularly and automatically updated using
+scripts in this repo.
+
+To run the updater scripts manually, use:
+
+    nix-build -A maintainer-scripts.update-hackage -o update-hackage.sh
+    ./update-hackage.sh
+
+    nix-build -A maintainer-scripts.update-stackage -o update-stackage.sh
+    ./update-stackage.sh
+
+The scripts will clone the repo, generate the latest data, then
+attempt to push back to the repo and update the source JSON file.

hackage-src.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/input-output-hk/hackage.nix",
+  "rev": "3180384b563ec7c7b46bca86b3ace0f32d04cde8",
+  "date": "2019-02-05T17:32:14+08:00",
+  "sha256": "19ndkn8pivli9plwq0wnx1cj126l89yk7jw9a0dj51ic3b2qhlb2",
+  "fetchSubmodules": false
+}

compat.nix renamed to lib/compat.nix

@@ -0,0 +1,16 @@
+# Provides a function for fetching a GitHub repo from a JSON spec,
+# overridable with the given entry on the NIX_PATH.
+
+let
+  overrideWith = import ./override-with.nix;
+in
+  { name, specJSON, override }:
+  let
+    spec = builtins.fromJSON (builtins.readFile specJSON);
+  in
+    overrideWith override
+      (builtins.fetchTarball {
+        inherit name;
+        url = "${spec.url}/archive/${spec.rev}.tar.gz";
+        inherit (spec) sha256;
+      })

lib/fetch-external.nix

@@ -0,0 +1,9 @@
+# overrideWith - Allow the user to override the given default path
+# with a NIX_PATH entry.
+override: default:
+ let
+   try = builtins.tryEval (builtins.findFile builtins.nixPath override);
+ in if try.success then
+   builtins.trace "using search host <${override}>" try.value
+ else
+   default

lib/override-with.nix

@@ -0,0 +1,15 @@
+{ symlinkJoin, fetchExternal, mkPkgSet }:
+
+let
+  src = fetchExternal {
+    name     = "nix-tools-src";
+    specJSON = ./nix-tools-src.json;
+    override = "nix-tools";
+  };
+
+  hsPkgs = import (src + "/pkgs.nix") { inherit mkPkgSet; };
+in
+  symlinkJoin {
+    name = "nix-tools";
+    paths = builtins.attrValues hsPkgs.nix-tools.components.exes;
+  }

nix-tools/default.nix

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/input-output-hk/nix-tools",
+  "rev": "232e4fde7f942ef234b649144016d5da0f7e2745",
+  "date": "2019-02-11T12:59:38+08:00",
+  "sha256": "1nrm9vcq443isk09z1fmlp8zxnw9p3cx95zbda29s5mky17ky2c0",
+  "fetchSubmodules": false
+}

nix-tools/nix-tools-src.json

@@ -0,0 +1,26 @@
+{ stdenv, writeScript, glibc, coreutils, git, nix-tools, cabal-install, nix-prefetch-git }:
+
+{ name, script }:
+
+with stdenv.lib;
+
+writeScript "update-${name}-nix.sh" ''
+  #!${stdenv.shell}
+
+  set -euo pipefail
+
+  export PATH="${makeBinPath [ coreutils glibc git nix-tools cabal-install nix-prefetch-git ]}"
+
+  ${script}
+
+  git add .
+  git commit --allow-empty -m "Automatic update for $(date)"
+
+  rev=$(git rev-parse HEAD)
+
+  git push
+
+  cd ..
+
+  nix-prefetch-git https://github.com/input-output-hk/${name}.nix.git --rev "$rev" | tee ${name}-src.json
+''

scripts/update-external.nix

@@ -0,0 +1,24 @@
+{ stdenv, writeScript, coreutils, glibc, git, nix-tools, cabal-install, nix-prefetch-git }@args:
+
+import ./update-external.nix args {
+  name = "hackage";
+  script = ''
+    # Make sure the hackage index is recent.
+    cabal new-update
+
+    # Clone or update the Hackage Nix expressions repo.
+    if [ -d hackage.nix ]; then
+      cd hackage.nix
+      git pull --ff-only
+      cd ..
+    else
+      git clone [email protected]:input-output-hk/hackage.nix.git
+    fi
+
+    echo "Running hackage-to-nix..."
+
+    hackage-to-nix hackage.nix
+
+    cd hackage.nix
+  '';
+}

scripts/update-hackage.nix

@@ -0,0 +1,44 @@
+{ stdenv, writeScript, coreutils, glibc, git, nix-tools, cabal-install, nix-prefetch-git }@args:
+
+import ./update-external.nix args {
+  name = "stackage";
+  script = ''
+    # Clone or update the main Stackage Nix expressions repo.
+    # The upstream LTS and Nightly package sets are in submodules, which
+    # should also be updated.
+    if [ -d stackage.nix ]; then
+      cd stackage.nix
+      git pull --ff-only
+      git submodule update --init
+      git submodule foreach git pull origin master
+    else
+      git clone [email protected]:input-output-hk/stackage.nix.git
+      cd stackage.nix
+      git submodule update --init
+    fi
+
+    echo "Running lts-to-nix for all snapshots..."
+
+    # update them all in parallel...
+    N=$(getconf _NPROCESSORS_ONLN)
+    for lts in {lts-haskell,stackage-nightly}/*.yaml
+    do
+        lts-to-nix $lts > $(basename ''${lts%.yaml}.nix) &
+        while [[ $(jobs -r -p | wc -l) -gt $N ]]; do
+        # can't use `wait -n` on older bash versions.
+        # e.g. what ships with macOS High Sierra
+        sleep 1;
+        done
+    done
+    wait
+
+    # update nightlies
+    echo "{" > nightlies.nix;
+    for a in nightly-*.nix; do echo "  \"''${a%%.nix}\" = import ./$a;" >> nightlies.nix; done;
+    echo "}" >> nightlies.nix
+    # update lts
+    echo "{" > ltss.nix;
+    for a in lts-*.nix; do echo "  \"''${a%%.nix}\" = import ./$a;" >> ltss.nix; done;
+    echo "}" >> ltss.nix
+  '';
+}

scripts/update-stackage.nix

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/input-output-hk/stackage.nix",
+  "rev": "2615a4e6b1651215ee400e62fcdcb195062a3d35",
+  "date": "2019-02-05T17:41:44+08:00",
+  "sha256": "08c8lb8x047hndwm1cb2zxixnjmrswfp5y18xp1v79cjqlva0qj6",
+  "fetchSubmodules": false
+}

stackage-src.json

@@ -1,13 +1,9 @@
-{ pkgs
-, haskell
-, stdenv
-}:
+{ mkPkgSet, stdenv }:
 
 with stdenv.lib;
 
 let
-  pkgSet = haskell.mkPkgSet {
-    inherit pkgs;
+  pkgSet = mkPkgSet {
     # generated with:
     #   cabal new-build
     #   plan-to-nix dist-newstyle/cache/plan.json > plan.nix

test/builder-haddock/default.nix

@@ -1,13 +1,9 @@
-{ pkgs
-, haskell
-, stdenv
-}:
+{ stdenv, mkPkgSet }:
 
 with stdenv.lib;
 
 let
-  pkgSet = haskell.mkPkgSet {
-    inherit pkgs;
+  pkgSet = mkPkgSet {
     pkg-def = import ./plan.nix;
     pkg-def-overlays = [
       { project = ./project.nix; }
@@ -32,23 +28,23 @@ in
       "$exe"
 
       printf "checking that executable is dynamically linked to system libraries... " >& 2
-    '' + pkgs.lib.optionalString pkgs.stdenv.isLinux ''
+    '' + optionalString stdenv.isLinux ''
       ldd $exe | grep libpthread
-    '' + pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+    '' + optionalString stdenv.isDarwin ''
       otool -L $exe | grep "libSystem.B"
     '' + ''
       # fixme: posix-specific
       printf "checking that dynamic library is produced... " >& 2
-    '' + pkgs.lib.optionalString pkgs.stdenv.isLinux ''
+    '' + optionalString stdenv.isLinux ''
       sofile=$(find "${packages.project.components.library}" | grep -e '\.so$')
-    '' + pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+    '' + optionalString stdenv.isDarwin ''
       sofile=$(find "${packages.project.components.library}" | grep -e '\.dylib$')
     '' + ''
       echo "$sofile"
       printf "checking that dynamic library is dynamically linked to prim... " >& 2
-    '' + pkgs.lib.optionalString pkgs.stdenv.isLinux ''
+    '' + optionalString stdenv.isLinux ''
       ldd $sofile | grep libHSghc-prim
-    '' + pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+    '' + optionalString stdenv.isDarwin ''
       otool -L $sofile | grep libHSghc-prim
     '' + ''
       touch $out