Skip to content

Add evalSystem and evalPackages project args #1546

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Jul 28, 2022
Merged

Add evalSystem and evalPackages project args #1546

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
6ce3db2
Add `evalSystem` project argument
hamishmack Jul 14, 2022
703b38e
Merge remote-tracking branch 'origin/master' into hkm/eval-system
hamishmack Jul 18, 2022
8d4ff4c
Drop `pkgs.evalPackages`, add project arg instead
hamishmack Jul 18, 2022
96e1fd3
Move hackage-quirks to /modules
hamishmack Jul 18, 2022
3a20232
Move fetchgit fix
hamishmack Jul 19, 2022
64f4d9c
Only allow single module for projectModule tools
hamishmack Jul 19, 2022
24b82e7
Better fix for gitMinimal issue
hamishmack Jul 19, 2022
93ce316
Add comments
hamishmack Jul 20, 2022
ade33e2
Make `hackage-tool` take multiple modules
hamishmack Jul 20, 2022
a6555d3
Use explicit evalPackages for cleanGit in tests
hamishmack Jul 20, 2022
0ded07f
Fix hydra job eval without builders
hamishmack Jul 21, 2022
b7b3c02
Fix for ghcjs <8.10
hamishmack Jul 21, 2022
468d425
Fix for check-hydra eval
hamishmack Jul 21, 2022
8f92091
Stop depending on builtins.currentSystem
hamishmack Jul 21, 2022
63559e2
Fix for aarch64-linux
hamishmack Jul 21, 2022
fe0895a
Fix cabal-install hackage-quirks for cross
hamishmack Jul 21, 2022
13631f3
Fix musl cross compile
hamishmack Jul 22, 2022
2cc07bd
Merge remote-tracking branch 'origin/master' into hkm/eval-system
hamishmack Jul 22, 2022
7ade088
Fix evalPackages in ci for cross compilers
hamishmack Jul 22, 2022
848c79e
Add change log entry
hamishmack Jul 22, 2022
15bb1f2
typo
hamishmack Jul 22, 2022
08bf47c
Update project option descriptions
hamishmack Jul 22, 2022
ccc01fc
Merge remote-tracking branch 'origin/master' into hkm/eval-system
hamishmack Jul 24, 2022
759adf7
Merge remote-tracking branch 'origin/master' into hkm/eval-system
hamishmack Jul 25, 2022
07a8daa
Update changelog.md
hamishmack Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions hix/project.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,20 +56,20 @@ let
nixpkgsArgs = config.haskellNix.nixpkgsArgs // {
overlays = config.haskellNix.nixpkgsArgs.overlays ++ config.overlays;
};
pkgs = import config.nixpkgs config.nixpkgsArgs;
project = config.pkgs.haskell-nix.project [
_module.args.pkgs = import config.nixpkgs config.nixpkgsArgs;
project = pkgs.haskell-nix.project [
(import ../modules/hix-project.nix)
userDefaults
projectDefaults
commandArgs'
{
({config, ...}: {
src =
if __pathExists (toString (src.origSrcSubDir or src) + "/.git")
then config.pkgs.haskell-nix.haskellLib.cleanGit {
then config.evalPackages.haskell-nix.haskellLib.cleanGit {
inherit src name;
}
else src;
}
})
];
})
];
Expand Down
22 changes: 11 additions & 11 deletions lib/cabal-project-parser.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ let
# This works because `cabal configure` does not include any of the `/nix/sore/`
# paths in the `plan.json` (so materialized plan-nix will still work as expeced).
# See tests/unit.nix for examples of input and output.
parseRepositoryBlock = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
parseRepositoryBlock = evalPackages: cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: block:
let
lines = pkgs.lib.splitString "\n" block;
# The first line will contain the repository name.
Expand All @@ -138,7 +138,7 @@ let
# The $HOME/.cabal/packages/${name} after running `cabal v2-update` to download the repository
repoContents = if inputMap ? ${attrs.url}
# If there is an input use it to make `file:` url and create a suitable `.cabal/packages/${name}` directory
then pkgs.evalPackages.runCommand name ({
then evalPackages.runCommand name ({
nativeBuildInputs = [ cabal-install ];
preferLocalBuild = true;
}) ''
Expand All @@ -155,9 +155,9 @@ let
cabal v2-update ${name}
cp -r $HOME/.cabal/packages/${name} $out
''
else pkgs.evalPackages.runCommand name ({
nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
else evalPackages.runCommand name ({
nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ];
LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
LANG = "en_US.UTF-8";
preferLocalBuild = true;
} // pkgs.lib.optionalAttrs (sha256 != null) {
Expand All @@ -175,15 +175,15 @@ let
${pkgs.lib.optionalString (attrs ? key-threshold) "key-threshold: ${attrs.key-threshold}"}
EOF

export SSL_CERT_FILE=${pkgs.evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
export SSL_CERT_FILE=${evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
cabal v2-update ${name}
cp -r $HOME/.cabal/packages/${name} $out
'';
# Output of hackage-to-nix
hackage = import (
pkgs.evalPackages.runCommand ("hackage-to-nix-" + name) {
nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
evalPackages.runCommand ("hackage-to-nix-" + name) {
nativeBuildInputs = [ cabal-install evalPackages.curl nix-tools ];
LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
LANG = "en_US.UTF-8";
preferLocalBuild = true;
} ''
Expand All @@ -196,11 +196,11 @@ let
};
};

parseRepositories = cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
parseRepositories = evalPackages: cabalProjectFileName: sha256map: inputMap: cabal-install: nix-tools: projectFile:
let
# This will leave the name of repository in the first line of each block
blocks = pkgs.lib.splitString "\nrepository " ("\n" + projectFile);
repoBlocks = builtins.map (parseRepositoryBlock cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
repoBlocks = builtins.map (parseRepositoryBlock evalPackages cabalProjectFileName sha256map inputMap cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
in {
extra-hackages = pkgs.lib.lists.map (block: block.hackage) repoBlocks;
repos = pkgs.lib.lists.foldl' (x: block: x // block.repo) {} repoBlocks;
Expand Down
37 changes: 20 additions & 17 deletions lib/call-cabal-project-to-nix.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ dotCabal, pkgs, runCommand, evalPackages, symlinkJoin, cacert, index-state-hashes, haskellLib, materialize }@defaults:
{ pkgs, runCommand, cacert, index-state-hashes, haskellLib }@defaults:
let readIfExists = src: fileName:
let origSrcDir = src.origSrcSubDir or src;
in
Expand Down Expand Up @@ -64,10 +64,13 @@ in
# would result in the "foo" subdirectory of
# any plutus-apps input being used for a
# package.
, evalPackages
, ...
}@args:

let
inherit (evalPackages.haskell-nix) materialize dotCabal;

# These defaults are hear rather than in modules/cabal-project.nix to make them
# lazy enough to avoid infinite recursion issues.
# Using null as the default also improves performance as they are not forced by the
Expand Down Expand Up @@ -219,8 +222,8 @@ let
in {
# Download the source-repository-package commit and add it to a minimal git
# repository that `cabal` will be able to access from a non fixed output derivation.
location = pkgs.evalPackages.runCommand "source-repository-package" {
nativeBuildInputs = [ pkgs.evalPackages.rsync pkgs.evalPackages.gitMinimal ];
location = evalPackages.runCommand "source-repository-package" {
nativeBuildInputs = [ evalPackages.rsync evalPackages.gitMinimal ];
} ''
mkdir $out
rsync -a --prune-empty-dirs "${fetched}/" "$out/"
Expand All @@ -241,24 +244,24 @@ let

# Parse the `repository` blocks
repoResult = pkgs.haskell-nix.haskellLib.parseRepositories
cabalProjectFileName sha256map inputMap cabal-install nix-tools sourceRepoPackageResult.otherText;
evalPackages cabalProjectFileName sha256map inputMap cabal-install nix-tools sourceRepoPackageResult.otherText;

# we need the repository content twice:
# * at eval time (below to build the fixed project file)
# Here we want to use pkgs.evalPackages.fetchgit, so one can calculate
# Here we want to use evalPackages.fetchgit, so one can calculate
# the build plan for any target without a remote builder
# * at built time (passed out)
# Here we want to use plain pkgs.fetchgit, which is what a builder
# on the target system would use, so that the derivation is unaffected
# and, say, a linux release build job can identify the derivation
# as built by a darwin builder, and fetch it from a cache
sourceReposEval = builtins.map (fetchPackageRepo pkgs.evalPackages.fetchgit) sourceRepoPackageResult.sourceRepos;
sourceReposEval = builtins.map (fetchPackageRepo evalPackages.fetchgit) sourceRepoPackageResult.sourceRepos;
sourceReposBuild = builtins.map (x: (fetchPackageRepo pkgs.fetchgit x).fetched) sourceRepoPackageResult.sourceRepos;
in {
sourceRepos = sourceReposBuild;
inherit (repoResult) repos extra-hackages;
makeFixedProjectFile = ''
cp -f ${pkgs.evalPackages.writeText "cabal.project" sourceRepoPackageResult.otherText} ./cabal.project
cp -f ${evalPackages.writeText "cabal.project" sourceRepoPackageResult.otherText} ./cabal.project
'' +
pkgs.lib.optionalString (builtins.length sourceReposEval != 0) (''
chmod +w -R ./cabal.project
Expand Down Expand Up @@ -365,12 +368,12 @@ let
'');

# Dummy `ghc` that uses the captured output
dummy-ghc = pkgs.evalPackages.writeTextFile {
dummy-ghc = evalPackages.writeTextFile {
name = "dummy-" + ghc.name;
executable = true;
destination = "/bin/${ghc.targetPrefix}ghc";
text = ''
#!${pkgs.evalPackages.runtimeShell}
#!${evalPackages.runtimeShell}
case "$*" in
--version*)
cat ${dummy-ghc-data}/ghc/version
Expand Down Expand Up @@ -408,12 +411,12 @@ let
};

# Dummy `ghc-pkg` that uses the captured output
dummy-ghc-pkg = pkgs.evalPackages.writeTextFile {
dummy-ghc-pkg = evalPackages.writeTextFile {
name = "dummy-pkg-" + ghc.name;
executable = true;
destination = "/bin/${ghc.targetPrefix}ghc-pkg";
text = ''
#!${pkgs.evalPackages.runtimeShell}
#!${evalPackages.runtimeShell}
case "$*" in
--version)
cat ${dummy-ghc-data}/ghc-pkg/version
Expand Down Expand Up @@ -449,10 +452,10 @@ let
else null;
} // pkgs.lib.optionalAttrs (checkMaterialization != null) {
inherit checkMaterialization;
}) (pkgs.evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg cabal-install pkgs.evalPackages.rsync pkgs.evalPackages.gitMinimal ];
}) (evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg cabal-install evalPackages.rsync evalPackages.gitMinimal ];
# Needed or stack-to-nix will die on unicode inputs
LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
LOCALE_ARCHIVE = pkgs.lib.optionalString (evalPackages.stdenv.buildPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
LANG = "en_US.UTF-8";
meta.platforms = pkgs.lib.platforms.all;
preferLocalBuild = false;
Expand All @@ -474,11 +477,11 @@ let
'';
in {
# These check for cabal configure failure
json = pkgs.evalPackages.runCommand (nameAndSuffix "plan-json") {} ''
json = evalPackages.runCommand (nameAndSuffix "plan-json") {} ''
${checkCabalConfigure}
cp ${plan-nix.maybeJson} $out
'';
freeze = pkgs.evalPackages.runCommand (nameAndSuffix "plan-freeze") {} ''
freeze = evalPackages.runCommand (nameAndSuffix "plan-freeze") {} ''
${checkCabalConfigure}
cp ${plan-nix.maybeFreeze} $out
'';
Expand Down Expand Up @@ -520,7 +523,7 @@ let
${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"}
${fixedProject.makeFixedProjectFile}
${pkgs.lib.optionalString (cabalProjectFreeze != null) ''
cp ${pkgs.evalPackages.writeText "cabal.project.freeze" cabalProjectFreeze} \
cp ${evalPackages.writeText "cabal.project.freeze" cabalProjectFreeze} \
cabal.project.freeze
chmod +w cabal.project.freeze
''}
Expand Down
24 changes: 13 additions & 11 deletions lib/call-stack-to-nix.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
*
* see also `call-cabal-project-to-nix`!
*/
{ runCommand, pkgs, mkCacheFile, materialize, haskellLib }:
{ name ? src.name or null # optional name for better error messages
, src
, stackYaml ? "stack.yaml"
Expand All @@ -15,21 +14,24 @@
, resolverSha256 ? null
, materialized ? null # Location of a materialized copy of the nix files
, checkMaterialization ? null # If true the nix files will be generated used to check plan-sha256 and material
, nix-tools ? pkgs.haskell-nix.internal-nix-tools # When building stack projects we use the internal nix-tools (compiled with a fixed GHC version)
, ... }:
, nix-tools
, evalPackages
, ... }@args:
let
inherit (evalPackages) runCommand;
inherit (evalPackages.haskell-nix) mkCacheFile materialize haskellLib;
inherit (haskellLib.fetchResolver {
inherit src stackYaml resolverSha256;
}) resolver fetchedResolver;

subDir' = src.origSubDir or "";
subDir = pkgs.lib.strings.removePrefix "/" subDir';
subDir = evalPackages.lib.strings.removePrefix "/" subDir';
cleanedSource = (haskellLib.cleanSourceWith {
name = if name != null then "${name}-root-cabal-files" else "source-root-cabal-files";
src = src.origSrc or src;
filter = path: type: (!(src ? filter) || src.filter path type) && (
type == "directory" ||
pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ stackYaml ".cabal" "package.yaml" ]); });
evalPackages.lib.any (i: (evalPackages.lib.hasSuffix i path)) [ stackYaml ".cabal" "package.yaml" ]); });

stackToNixArgs = builtins.concatStringsSep " " [
"--full"
Expand All @@ -43,12 +45,12 @@ let
sha256Arg = "stack-sha256";
reasonNotSafe = null;
this = "project.stack-nix" + (if name != null then " for ${name}" else "");
} // pkgs.lib.optionalAttrs (checkMaterialization != null) {
} // evalPackages.lib.optionalAttrs (checkMaterialization != null) {
inherit checkMaterialization;
}) (runCommand (if name == null then "stack-to-nix-pkgs" else name + "-stack-to-nix-pkgs") {
nativeBuildInputs = [ nix-tools pkgs.nix-prefetch-git pkgs.cacert pkgs.xorg.lndir ];
nativeBuildInputs = [ nix-tools evalPackages.nix-prefetch-git evalPackages.cacert evalPackages.xorg.lndir ];
# Needed or stack-to-nix will die on unicode inputs
LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.stdenv.hostPlatform.libc == "glibc") "${pkgs.glibcLocales}/lib/locale/locale-archive";
LOCALE_ARCHIVE = evalPackages.lib.optionalString (evalPackages.stdenv.hostPlatform.libc == "glibc") "${evalPackages.glibcLocales}/lib/locale/locale-archive";
LANG = "en_US.UTF-8";
LC_ALL = "en_US.UTF-8";
preferLocalBuild = false;
Expand All @@ -64,18 +66,18 @@ let
echo "cleaned source is empty. Did you forget to 'git add -A'?"; exit 1;
fi
lndir -silent "${cleanedSource}/." $SRC
${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"}
${evalPackages.lib.optionalString (subDir != "") "cd ${subDir}"}
${
# If a resolver was fetched use the it instead of the original stack.yaml
pkgs.lib.optionalString (fetchedResolver != null)
evalPackages.lib.optionalString (fetchedResolver != null)
# Replace the resolver path in the stack.yaml with the fetched version
''
rm ${stackYaml}
cp ${src}/${stackYaml} .
chmod +w ${stackYaml}
substituteInPlace ${stackYaml} --replace "${resolver}" "${fetchedResolver}"
''}
${pkgs.lib.optionalString (cache != null) ''
${evalPackages.lib.optionalString (cache != null) ''
cp ${mkCacheFile cache}/.stack-to-nix.cache* $out${subDir'}
''}
(cd $out${subDir'} && stack-to-nix ${stackToNixArgs})
Expand Down
27 changes: 13 additions & 14 deletions lib/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,8 @@
with haskellLib;

let
# Why `final.evalPackages.buildPackages.gitMinimal`?
# Why not just final.evalPackages.gitMinimal?
#
# A problem arises when `evalPackages` is `buildPackages`.i
# As may be the case in a flake.
# Why `final.buildPackages.buildPackages.gitMinimal`?
# Why not just final.buildPackages.gitMinimal?
#
# It turns out `git` depends on `gdb` in a round about way:
# git -> openssh -> libfido2 -> systemd -> python libxml -> Cython -> gdb
Expand All @@ -24,7 +21,7 @@ let
# * When `gdb` does not exist for `js`, so when cross
# compiling with ghcjs `final.buildPackages.git` fails
# to build at all.
inherit (pkgs.evalPackages.buildPackages) gitMinimal;
inherit (pkgs.buildPackages.buildPackages) gitMinimal;

in {
# Within the package components, these are the attribute names of
Expand Down Expand Up @@ -202,7 +199,7 @@ in {
cleanGit = import ./clean-git.nix {
inherit lib cleanSourceWith;
git = gitMinimal;
inherit (pkgs.evalPackages.buildPackages) runCommand;
inherit (pkgs.buildPackages.buildPackages) runCommand;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these changes actually okay? Won't this again mean that you can't eval without a build machine of the right system, because it will try and clean the source on a build machine?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same for the other places where this switch happens.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure about cleanGit. It should be ok on CI where .git does not exist, but it may fail for local evals. I'll do some tests.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I guess it should be fairly obvious if you try building a darwin derivation when you don't have a darwin machine or something. If it works then I'm happy! It just seems suspicious - presumably it means that the usage of evalPackages was actually unncessary in each of these places to begin with?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is a situation when not using flakes (so cleanGit would have work to do) and cross compiling where this would have been useful so I have included a note in the change log.

};

# Some times it is handy to temporarily use a relative path between git
Expand Down Expand Up @@ -260,17 +257,19 @@ in {
&& stdenv.buildPlatform == stdenv.hostPlatform
&& stdenv.hostPlatform == stdenv.targetPlatform;

# Takes a version number or attr set of arguments (for cabalProject)
# and converts it to an attr set of arguments. This allows
# Takes a version number, module or list of modules (for cabalProject)
# and converts it to an list of project modules. This allows
# the use of "1.0.0.0" or { version = "1.0.0.0"; ... }
versionOrArgsToArgs = versionOrArgs:
if lib.isAttrs versionOrArgs
then versionOrArgs
else { version = versionOrArgs; };
versionOrModToMods = versionOrMod:
if lib.isString versionOrMod
then [{ version = versionOrMod; }]
else if lib.isList versionOrMod
then versionOrMod
else [versionOrMod];

# Find the resolver in the stack.yaml file and fetch it if a sha256 value is provided
fetchResolver = import ./fetch-resolver.nix {
inherit (pkgs.evalPackages) pkgs;
inherit (pkgs.buildPackages) pkgs;
};

inherit (import ./cabal-project-parser.nix {
Expand Down
6 changes: 3 additions & 3 deletions lib/spdx/licenses.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
pkgs:
with builtins; let
spdxJson = pkgs.evalPackages.spdx-license-list-data.json or pkgs.evalPackages.spdx-license-list-data;
spdxJson = pkgs.buildPackages.spdx-license-list-data.json or pkgs.buildPackages.spdx-license-list-data;
materialized' = ../../materialized + "/spdx-${spdxJson.version}";
licensesJSON = fromJSON (replaceStrings
[ "\\u0026" "\\u0027" "\\u003d" ]
[ "&" "'" "=" ]
(readFile "${pkgs.evalPackages.haskell-nix.materialize {
(readFile "${pkgs.buildPackages.haskell-nix.materialize {
materialized = if pathExists materialized' then materialized' else null;
} (pkgs.evalPackages.runCommand "spdx-json" { inherit (spdxJson) version; } ''
} (pkgs.buildPackages.runCommand "spdx-json" { inherit (spdxJson) version; } ''
mkdir $out
cp ${spdxJson}/json/licenses.json $out
'')
Expand Down
Loading