Buildkite pipeline with stack rebuild

.buildkite/pipeline.yml

@@ -1,5 +1,14 @@
 steps:
   - label: 'nix build'
     command: 'nix-build release.nix'
+  - label: 'stack rebuild'
+    env:
+      AWS_REGION: us-west-1
+      S3_BUCKET: appveyor-ci-cache
+      CACHE_S3_MAX_SIZE: 2500MB
+      STACK_ROOT: "/build/cardano-shell.stack"
+    command:
+      - "nix-build scripts/buildkite -o stack-rebuild"
+      - "./stack-rebuild --build-dir /build/cardano-shell --base-branch develop"
     agents:
       system: x86_64-linux

README.md

@@ -1,5 +1,7 @@
 # cardano-shell
 
+[![Build status](https://badge.buildkite.com/5e4cd5ff2fd87975136914d037c409618deb4d8ed6579f8635.svg)](https://buildkite.com/input-output-hk/cardano-shell)
+
 This is the `cardano-shell` project.
 This project uses Github issues for tracking the project progress.
 

nix/fetch-nixpkgs.nix

@@ -0,0 +1,6 @@
+let
+  spec = builtins.fromJSON (builtins.readFile ./nixpkgs-src.json);
+in builtins.fetchTarball {
+  url = "${spec.url}/archive/${spec.rev}.tar.gz";
+  inherit (spec) sha256;
+}

nix/nixpkgs-src.json

@@ -0,0 +1,6 @@
+{
+  "url": "https://github.com/NixOS/nixpkgs",
+  "rev": "3f5f9eda9d00f076140906f316554735c18c93d4",
+  "sha256": "1rmc879rvkb9dm5bd2a1jwrk3sv8bbispbjkb9dnhma7fqwbm6gg",
+  "fetchSubmodules": false
+}

scripts/buildkite/cache-s3.nix

@@ -0,0 +1,25 @@
+{ stdenv, fetchurl, zlib, gmp }:
+
+stdenv.mkDerivation rec {
+  name = "cache-s3-${version}";
+  version = "v0.1.6";
+  src = fetchurl {
+    url = "https://github.com/fpco/cache-s3/releases/download/${version}/cache-s3-${version}-linux-x86_64.tar.gz";
+    sha256 = "01qm6mg11g6kq3sfnnj1civmda35mbfmp1fym5yvqwbdsmqd0b19";
+  };
+  libPath = stdenv.lib.makeLibraryPath [
+    stdenv.cc.cc.lib
+    zlib
+    gmp
+  ];
+  sourceRoot = ".";
+  buildPhase = "true";
+  installPhase = ''
+    mkdir -p $out/bin
+    install -m 0755 cache-s3 $out/bin
+  '';
+  postFixup = ''
+    patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $out/bin/cache-s3
+    patchelf --set-rpath ${libPath} $out/bin/cache-s3
+  '';
+}

scripts/buildkite/default.nix

@@ -0,0 +1,20 @@
+{ pkgs ? import (import ../../nix/fetch-nixpkgs.nix) { }
+, buildTools ? with pkgs; [ git nix gnumake ]
+}:
+
+with pkgs.lib;
+with pkgs;
+
+let
+  cache-s3 = callPackage ./cache-s3.nix {};
+
+  stackRebuild = runCommand "stack-rebuild" {} ''
+    ${haskellPackages.ghcWithPackages (ps: [ps.turtle ps.safe ps.transformers])}/bin/ghc -o $out ${./rebuild.hs}
+  '';
+
+in
+  writeScript "stack-rebuild-wrapped" ''
+    #!${stdenv.shell}
+    export PATH=${lib.makeBinPath ([ cache-s3 stack gnused coreutils ] ++ buildTools)}
+    exec ${stackRebuild} "$@"
+  ''

scripts/buildkite/rebuild.hs

@@ -0,0 +1,166 @@
+{-# LANGUAGE OverloadedStrings, RecordWildCards, LambdaCase, ScopedTypeVariables #-}
+
+import Turtle hiding (option)
+import Prelude hiding (FilePath)
+import qualified Filesystem.Path.CurrentOS as FP
+import Control.Monad.Trans.Maybe
+import Control.Exception
+import qualified Data.Text as T
+import Safe
+import System.Exit (exitWith)
+import Options.Applicative
+
+data BuildkiteEnv = BuildkiteEnv
+  { bkBuildNum :: Int
+  , bkPipeline :: Text
+  , bkBranch   :: Text
+  } deriving (Show)
+
+data CICacheConfig = CICacheConfig
+  { ccMaxSize    :: Maybe Text
+  , ccBucket     :: Text
+  , ccRegion     :: Maybe Text
+  , ccPrefix     :: Text
+  , ccBranch     :: Text
+  , ccBaseBranch :: Text
+  } deriving (Show)
+
+data RebuildOpts = RebuildOpts
+  { optBuildDirectory :: Maybe FilePath
+  , optBaseBranch :: Text
+  } deriving (Show)
+
+rebuildOpts :: Parser RebuildOpts
+rebuildOpts = RebuildOpts <$> optional buildDir <*> baseBranch
+  where
+    buildDir = option (FP.decodeString <$> str) (long "build-dir" <> metavar "DIR" <> help "Copy sources to directory before building")
+    baseBranch = option str ( long "base-branch" <> value "master" <> showDefault <> help "Fallback base branch for cache-s3" )
+
+parseOpts :: IO RebuildOpts
+parseOpts = execParser opts
+  where opts = info (rebuildOpts <**> helper)
+          ( fullDesc <> progDesc "Build cardano-sl project with stack in Buildkite" )
+
+main :: IO ()
+main = do
+  RebuildOpts{..} <- parseOpts
+  awsCreds
+  bk <- getBuildkiteEnv
+  maybe (pure ()) setupBuildDirectory optBuildDirectory
+  cacheConfig <- getCacheConfig bk optBaseBranch
+  cacheDownloadStep cacheConfig
+  buildResult <- buildStep
+  cacheUploadStep cacheConfig
+  exitWith buildResult
+
+buildStep :: IO ExitCode
+buildStep = do
+  echo "+++ Build and test"
+  build .&&. test
+  where
+    cfg = ["--dump-logs", "--color", "always"]
+    stackBuild args = run "stack" $ cfg ++ ["build", "--fast"] ++ args
+    buildArgs = ["--bench", "--no-run-benchmarks", "--no-haddock-deps"]
+    buildAndTest = stackBuild $ ["--tests"] ++ buildArgs
+    build = stackBuild $ ["--no-run-tests"] ++ buildArgs
+    test = stackBuild ["--test", "--jobs", "1"]
+
+-- buildkite agents have S3 creds installed, but under different names
+awsCreds :: IO ()
+awsCreds = mapM_ (uncurry copy) things
+  where
+    copy src dst = need src >>= maybe (pure ()) (export dst)
+    things = [ ( "BUILDKITE_S3_ACCESS_KEY_ID"
+               , "AWS_ACCESS_KEY_ID")
+             , ( "BUILDKITE_S3_SECRET_ACCESS_KEY"
+               , "AWS_SECRET_ACCESS_KEY")
+             , ( "BUILDKITE_S3_DEFAULT_REGION"
+               , "AWS_DEFAULT_REGION")
+             ]
+
+getBuildkiteEnv :: IO (Maybe BuildkiteEnv)
+getBuildkiteEnv = runMaybeT $ do
+  bkBuildNum <- MaybeT $ needRead "BUILDKITE_BUILD_NUMBER"
+  bkPipeline <- MaybeT $ need "BUILDKITE_PIPELINE_SLUG"
+  bkBranch   <- MaybeT $ need "BUILDKITE_BRANCH"
+  pure BuildkiteEnv{..}
+
+needRead :: Read a => Text -> IO (Maybe a)
+needRead v = (>>= readMay) . fmap T.unpack <$> need v
+
+getCacheConfig :: Maybe BuildkiteEnv -> Text -> IO (Either Text CICacheConfig)
+getCacheConfig Nothing _ = pure (Left "BUILDKITE_* environment variables are not set")
+getCacheConfig (Just BuildkiteEnv{..}) ccBaseBranch = do
+  ccMaxSize <- need "CACHE_S3_MAX_SIZE"
+  ccRegion <- need "AWS_REGION"
+  need "S3_BUCKET" >>= \case
+    Just ccBucket -> pure (Right CICacheConfig{ccBranch=bkBranch, ccPrefix=bkPipeline, ..})
+    Nothing -> pure (Left "S3_BUCKET environment variable is not set")
+
+cacheDownloadStep :: Either Text CICacheConfig -> IO ()
+cacheDownloadStep cacheConfig = do
+  echo "--- CI Cache Download"
+  case cacheConfig of
+    Right cfg -> restoreCICache cfg `catch`
+      \ (ex :: IOException) -> do
+        eprintf ("Failed to download CI cache: "%w%"\nContinuing anyway...\n") ex
+    Left ex -> eprintf ("Not using CI cache because "%s%"\n") ex
+
+cacheUploadStep :: Either Text CICacheConfig -> IO ()
+cacheUploadStep cacheConfig = do
+  echo "--- CI Cache Upload"
+  case cacheConfig of
+    Right cfg -> saveCICache cfg `catch`
+      \ (ex :: IOException) -> do
+        eprintf ("Failed to upload CI cache: "%w%"\n") ex
+    Left _ -> printf "CI cache not configured.\n"
+
+restoreCICache :: CICacheConfig -> IO ()
+restoreCICache cfg = do
+  -- cacheS3 cfg (Just $ ccBaseBranch cfg) "restore stack"
+  cacheS3 cfg (Just $ ccBaseBranch cfg) "restore stack work"
+
+saveCICache :: CICacheConfig -> IO ()
+saveCICache cfg = do
+  -- cacheS3 cfg Nothing "save stack"
+  cacheS3 cfg Nothing "save stack work"
+
+cacheS3 :: CICacheConfig -> Maybe Text -> Text -> IO ()
+cacheS3 CICacheConfig{..} baseBranch cmd = void $ run "cache-s3" args
+  where
+    args = ml maxSize ++ ml regionArg ++
+           [ format ("--bucket="%s) ccBucket
+           , format ("--prefix="%s) ccPrefix
+           , format ("--git-branch="%s) ccBranch
+           , "--suffix=linux"
+           , "-v"
+           , "info"
+           ] ++ cmds ++ ml baseBranchArg
+    baseBranchArg = format ("--base-branch="%s) <$> baseBranch
+    maxSize = format ("--max-size="%s) <$> ccMaxSize
+    regionArg = format ("--region="%s) <$> ccRegion
+    cmds = ("-c":T.words cmd)
+    ml = maybe [] pure
+
+-- cache-s3 needs a build directory that is the same across all
+-- buildkite agents. The build directory option can be used to ensure
+-- this is the case.
+setupBuildDirectory :: FilePath -> IO ()
+setupBuildDirectory buildDir = do
+  exists <- testpath buildDir
+  when exists $ do
+    printf ("Removing old build directory "%fp%"\n") buildDir
+    rmtree buildDir
+  src <- pwd
+  printf ("Copying source tree "%fp%" -> "%fp%"\n") src buildDir
+  cptree src buildDir
+  cd buildDir
+
+run :: Text -> [Text] -> IO ExitCode
+run cmd args = do
+  printf (s%" "%s%"\n") cmd (T.unwords args)
+  res <- proc cmd args empty
+  case res of
+    ExitSuccess -> pure ()
+    ExitFailure code -> eprintf ("error: Command exited with code "%d%"!\nContinuing...\n") code
+  pure res

stack.yaml

@@ -19,7 +19,7 @@
 # resolver: ./custom-snapshot.yaml
 # resolver: https://example.com/snapshots/2018-01-01.yaml
 
-resolver: https://raw.githubusercontent.com/input-output-hk/cardano-prelude/0a32ec92c461e144f981864c97546db11b52e46a/snapshot.yaml
+resolver: https://raw.githubusercontent.com/input-output-hk/cardano-prelude/4b4457e75303ce352223b9723f7771fac6fe0600/snapshot.yaml
 
 # User packages to be built.
 # Various formats can be used as shown in the example below.