Merge remote-tracking branch 'origin/master' into hkm/combined-derivations

Author: hamishmack

.buildkite/pipeline.yml

@@ -41,7 +41,7 @@ steps:
 
   - label: 'Check that the haskell.nix roots do not require IFDs'
     command:
-      - nix build .#roots.x86_64-linux --option allow-import-from-derivation false
+      - yes | nix build .#roots.x86_64-linux --option allow-import-from-derivation false
     agents:
       system: x86_64-linux
 

docs/tutorials/getting-started.md

@@ -110,7 +110,7 @@ in pkgs.haskell-nix.project {
     src = ./.;
   };
   # Specify the GHC version to use.
-  compiler-nix-name = "ghc8102"; # Not required for `stack.yaml` based projects.
+  compiler-nix-name = "ghc8107"; # Not required for `stack.yaml` based projects.
 }
 ```
 

flake.lock

@@ -147,4 +147,14 @@
           ];
         };
     });
+
+  # --- Flake Local Nix Configuration ----------------------------
+  nixConfig = {
+    # This sets the flake to use the IOG nix cache.
+    # Nix should ask for permission before using it,
+    # but remove it here if you do not want it to.
+    extra-substituters = ["https://cache.iog.io"];
+    extra-trusted-public-keys = ["hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="];
+    allow-import-from-derivation = "true";
+  };
 }

flake.nix

@@ -216,6 +216,7 @@ in {
                 ++ fromUntil "8.10"   "9.2"    ./patches/ghc/ghc-8.10-global-unique-counters-in-rts.patch # backport of https://gitlab.haskell.org/ghc/ghc/-/commit/9a28680d2e23e7b25dd7254a439aea31dfae32d5
                 ++ fromUntil "9.2"    "9.3"    ./patches/ghc/ghc-9.2-global-unique-counters-in-rts.patch # backport of https://gitlab.haskell.org/ghc/ghc/-/commit/9a28680d2e23e7b25dd7254a439aea31dfae32d5
                 ++ fromUntil "8.10"   "9.1"    ./patches/ghc/issue-18708.patch              # https://gitlab.haskell.org/ghc/ghc/-/merge_requests/6554
+                ++ fromUntil "8.6.5"  "9.5"    ./patches/ghc/ghc-hpc-response-files.patch   # https://gitlab.haskell.org/ghc/ghc/-/merge_requests/8194
 
                 # the following is a partial reversal of https://gitlab.haskell.org/ghc/ghc/-/merge_requests/4391, to address haskell.nix#1227
                 ++ final.lib.optional (versionAtLeast "8.10" && versionLessThan "9.0" && final.targetPlatform.isAarch64) ./patches/ghc/mmap-next.patch

overlays/bootstrap.nix

@@ -0,0 +1,145 @@
+diff --git a/utils/hpc/Main.hs b/utils/hpc/Main.hs
+index 3f1813f2430f8a69dc8c334621661fdc03157c21..f7617ec6775351cbc3c149a433c4cbe5b47fb4d2 100644
+--- a/utils/hpc/Main.hs
++++ b/utils/hpc/Main.hs
+@@ -1,10 +1,17 @@
++{-# LANGUAGE ScopedTypeVariables, TupleSections #-}
+ -- (c) 2007 Andy Gill
+ 
+ -- Main driver for Hpc
++import Control.Monad (forM, forM_, when)
++import Data.Bifunctor (bimap)
++import Data.List (intercalate, partition, uncons)
++import Data.List.NonEmpty (NonEmpty((:|)))
++import Data.Maybe (catMaybes, isJust)
+ import Data.Version
+ import System.Environment
+ import System.Exit
+ import System.Console.GetOpt
++import System.Directory (doesPathExist)
+ 
+ import HpcFlags
+ import HpcReport
+@@ -16,7 +23,7 @@ import HpcOverlay
+ import Paths_hpc_bin
+ 
+ helpList :: IO ()
+-helpList =
++helpList = do
+      putStrLn $
+            "Usage: hpc COMMAND ...\n\n" ++
+            section "Commands" help ++
+@@ -25,6 +32,15 @@ helpList =
+            section "Coverage Overlays" overlays ++
+            section "Others" other ++
+            ""
++     putStrLn ""
++     putStrLn "or: hpc @response_file_1 @response_file_2 ..."
++     putStrLn ""
++     putStrLn "The contents of a Response File must have this format:"
++     putStrLn "COMMAND ..."
++     putStrLn ""
++     putStrLn "example:"
++     putStrLn "report my_library.tix --include=ModuleA \\"
++     putStrLn "--include=ModuleB"
+   where
+     help       = ["help"]
+     reporting  = ["report","markup"]
+@@ -47,13 +63,74 @@ section msg cmds = msg ++ ":\n"
+ 
+ dispatch :: [String] -> IO ()
+ dispatch [] = do
+-             helpList
+-             exitWith ExitSuccess
++    helpList
++    exitWith ExitSuccess
+ dispatch (txt:args0) = do
+-     case lookup txt hooks' of
+-       Just plugin -> parse plugin args0
+-       _ -> parse help_plugin (txt:args0)
++    case lookup txt hooks' of
++      Just plugin -> parse plugin args0
++      _ -> case getResponseFileName txt of
++        Nothing -> parse help_plugin (txt:args0)
++        Just firstResponseFileName -> do
++          let
++            (responseFileNames', nonResponseFileNames) = partitionFileNames args0
++          -- if arguments are combination of Response Files and non-Response Files, exit with error
++          when (length nonResponseFileNames > 0) $ do
++            let
++            putStrLn $ "First argument '" <> txt <> "' is a Response File, " <>
++              "followed by non-Response File(s): '" <> intercalate "', '" nonResponseFileNames <> "'"
++            putStrLn $ "When first argument is a Response File, " <>
++              "all arguments should be Response Files."
++            exitFailure
++          let
++            responseFileNames :: NonEmpty FilePath
++            responseFileNames = firstResponseFileName :| responseFileNames'
++
++          forM_ responseFileNames $ \responseFileName -> do
++            exists <- doesPathExist responseFileName
++            when (not exists) $ do
++              putStrLn $ "Response File '" <> responseFileName <> "' does not exist"
++              exitFailure
++
++          -- read all Response Files
++          responseFileNamesAndText :: NonEmpty (FilePath, String) <-
++            forM responseFileNames $ \responseFileName ->
++              fmap (responseFileName, ) (readFile responseFileName)
++          forM_ responseFileNamesAndText $ \(responseFileName, responseFileText) ->
++            -- parse first word of Response File, which should be a command
++            case uncons $ words responseFileText of
++              Nothing -> do
++                putStrLn $ "Response File '" <> responseFileName <> "' has no command"
++                exitFailure
++              Just (responseFileCommand, args1) -> case lookup responseFileCommand hooks' of
++                -- check command for validity
++                -- It is important than a Response File cannot specify another Response File;
++                -- this is prevented
++                Nothing -> do
++                  putStrLn $ "Response File '" <> responseFileName <>
++                    "' command '" <> responseFileCommand <> "' invalid"
++                  exitFailure
++                Just plugin -> do
++                  putStrLn $ "Response File '" <> responseFileName <> "':"
++                  parse plugin args1
++
+   where
++     getResponseFileName :: String -> Maybe FilePath
++     getResponseFileName s = do
++       (firstChar, filename) <- uncons s
++       if firstChar == '@'
++         then pure filename
++         else Nothing
++
++     -- first member of tuple is list of Response File names,
++     -- second member of tuple is list of all other arguments
++     partitionFileNames :: [String] -> ([FilePath], [String])
++     partitionFileNames xs = let
++       hasFileName :: [(String, Maybe FilePath)]
++       hasFileName = fmap (\x -> (x, getResponseFileName x)) xs
++       (fileNames, nonFileNames) :: ([Maybe FilePath], [String]) =
++         bimap (fmap snd) (fmap fst) $ partition (isJust . snd) hasFileName
++       in (catMaybes fileNames, nonFileNames)
++
+      parse plugin args =
+               case getOpt Permute (options plugin []) args of
+                 (_,_,errs) | not (null errs)
+@@ -66,7 +143,7 @@ dispatch (txt:args0) = do
+                            exitFailure
+                 (o,ns,_) -> do
+                          let flags = final_flags plugin
+-                                   $ foldr (.) id o
++                                   . foldr (.) id o
+                                    $ init_flags plugin
+                          implementation plugin flags ns
+ 
+@@ -112,7 +189,7 @@ help_main _ [] = do
+ help_main _ (sub_txt:_) = do
+     case lookup sub_txt hooks' of
+       Nothing -> do
+-          putStrLn $ "no such hpc command : " ++ sub_txt
++          putStrLn $ "no such HPC command: " <> sub_txt
+           exitFailure
+       Just plugin' -> do
+           command_usage plugin'

overlays/patches/ghc/ghc-hpc-response-files.patch

@@ -35,5 +35,5 @@ in
 
     cd ..
 
-    nix --experimental-features 'nix-command flakes' flake lock --update-input ${name}
+    yes | nix --experimental-features 'nix-command flakes' flake lock --update-input ${name}
   ''

scripts/update-external.nix

@@ -15,14 +15,15 @@ let
 
   # We can easily select a different compiler when using cabal,
   # but for stack we would need a different resolver to be used..
-  cabalProj = (cabalProject' projectArgs // { inherit compiler-nix-name; });
+  cabalProj = (cabalProject' (projectArgs // { inherit compiler-nix-name; }));
   stackProj = (stackProject' projectArgs);
 
   exeExt = stdenv.hostPlatform.extensions.executable;
 
 in recurseIntoAttrs ({
   # Does not work on ghcjs because it needs zlib.
-  meta.disabled = stdenv.hostPlatform.isGhcjs;
+  # TODO projectCoverageReport is broken in master for this example.
+  meta.disabled = true || stdenv.hostPlatform.isGhcjs;
   run = stdenv.mkDerivation {
     name = "coverage-test";
 
@@ -80,16 +81,16 @@ in recurseIntoAttrs ({
 
       ${concatStringsSep "\n" (map (project: ''
         pkga_basedir="${project.hsPkgs.pkga.coverageReport}/share/hpc/vanilla"
-        dirExistsEmpty "$pkga_basedir/html/pkga-0.1.0.0"
-        dirExistsEmpty "$pkga_basedir/mix/pkga-0.1.0.0"
-        dirExistsEmpty "$pkga_basedir/tix/pkga-0.1.0.0"
+        dirExists "$pkga_basedir/html/pkga-0.1.0.0"
+        dirExistsEmpty "$pkga_basedir/mix"
+        dirExists "$pkga_basedir/tix/pkga-0.1.0.0"
   
         project_basedir="${project.projectCoverageReport}/share/hpc/vanilla"
-        dirExistsEmpty "$pkga_basedir/html/pkga-0.1.0.0"
-        dirExistsEmpty "$pkga_basedir/mix/pkga-0.1.0.0"
-        dirExistsEmpty "$pkga_basedir/tix/pkga-0.1.0.0"
-        dirExistsEmpty "$project_basedir/tix/all"
-      '') (optional (compiler-nix-name == "ghc865") stackProj))}
+        dirExists "$pkga_basedir/html/pkga-0.1.0.0"
+        dirExistsEmpty "$pkga_basedir/mix"
+        dirExists "$pkga_basedir/tix/pkga-0.1.0.0"
+        dirExists "$project_basedir/tix/all"
+      '') ([cabalProj] ++ optional (compiler-nix-name == "ghc865") stackProj))}
 
       touch $out
     '';

test/coverage-no-libs/default.nix

@@ -19,10 +19,11 @@ let
 
   # We can easily select a different compiler when using cabal,
   # but for stack we would need a different resolver to be used..
-  cabalProj = (cabalProject' projectArgs // { inherit compiler-nix-name; });
+  cabalProj = (cabalProject' (projectArgs // { inherit compiler-nix-name; }));
   stackProj = (stackProject' projectArgs);
 
   exeExt = stdenv.hostPlatform.extensions.executable;
+  crossSuffix = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) "-${stdenv.hostPlatform.config}";
 
 in recurseIntoAttrs ({
   # Does not work on ghcjs because it needs zlib.
@@ -85,11 +86,11 @@ in recurseIntoAttrs ({
       ${concatStringsSep "\n" (map (project: ''
         pkga_basedir="${project.hsPkgs.pkga.coverageReport}/share/hpc/vanilla"
         findFileExistsNonEmpty "$pkga_basedir/mix/pkga-0.1.0.0/" "PkgA.mix"
-        dirExistsEmpty "$pkga_basedir/tix/pkga-0.1.0.0"
-        dirExistsEmpty "$pkga_basedir/html/pkga-0.1.0.0"
+        dirExists "$pkga_basedir/tix/pkga-0.1.0.0"
+        dirExists "$pkga_basedir/html/pkga-0.1.0.0"
   
         pkgb_basedir="${project.hsPkgs.pkgb.coverageReport}/share/hpc/vanilla"
-        testTix="$pkgb_basedir/tix/pkgb-0.1.0.0/tests${exeExt}/tests${exeExt}.tix"
+        testTix="$pkgb_basedir/tix/pkgb-test-tests${crossSuffix}-0.1.0.0-check${crossSuffix}/tests${exeExt}.tix"
         libTix="$pkgb_basedir/tix/pkgb-0.1.0.0/pkgb-0.1.0.0.tix"
         fileExistsNonEmpty "$testTix"
         fileExistsNonEmpty "$libTix"
@@ -116,9 +117,9 @@ in recurseIntoAttrs ({
         dirExists "$project_basedir/tix/pkga-0.1.0.0"
         dirExists "$project_basedir/tix/pkgb-0.1.0.0"
         fileExistsNonEmpty "$project_basedir/tix/pkgb-0.1.0.0/pkgb-0.1.0.0.tix"
-        dirExists "$project_basedir/tix/pkgb-0.1.0.0/tests${exeExt}"
-        fileExistsNonEmpty "$project_basedir/tix/pkgb-0.1.0.0/tests${exeExt}/tests${exeExt}.tix"
-      '') (optional (compiler-nix-name == "ghc865") stackProj))}
+        dirExists "$project_basedir/tix/pkgb-test-tests${crossSuffix}-0.1.0.0-check${crossSuffix}"
+        fileExistsNonEmpty "$project_basedir/tix/pkgb-test-tests${crossSuffix}-0.1.0.0-check${crossSuffix}/tests${exeExt}.tix"
+      '') ([cabalProj] ++ optional (compiler-nix-name == "ghc865") stackProj))}
 
       touch $out
     '';