Skip to content

Commit 8a8c57c

Browse files
committed
Use unit id for package key (input-output-hk#2239)
1 parent 1b5fd9c commit 8a8c57c

File tree

26 files changed

+454
-85
lines changed

26 files changed

+454
-85
lines changed

builder/comp-builder.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ let self =
2828
# (not just the one we are building).
2929
# Enable for tests in packages that use cabal-doctest.
3030
( haskellLib.isTest componentId &&
31-
lib.any (x: x.identifier.name or "" == "cabal-doctest") package.setup-depends
31+
lib.any (x: x.identifier.name or "" == "cabal-doctest") (package.setup-depends ++ setup.config.depends or [])
3232
)
3333
, allComponent # Used when `configureAllComponents` is set to get a suitable configuration.
3434

builder/make-config-files.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
let
66
# Sort and remove duplicates from nonReinstallablePkgs.
77
# That way changes to the order of nonReinstallablePkgs does not require rebuilds.
8-
nonReinstallablePkgs' = __attrNames (lib.genAttrs nonReinstallablePkgs (x: x));
8+
nonReinstallablePkgs' = __attrNames (lib.genAttrs (component.pre-existing or [] ++ nonReinstallablePkgs) (x: x));
99

1010
ghc = if enableDWARF then defaults.ghc.dwarf else defaults.ghc;
1111

builder/shell-for.nix

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ let
6868
selectedComponents =
6969
lib.filter isSelectedComponent (lib.attrValues transitiveDependenciesComponents);
7070

71-
allHsPkgsComponents = lib.concatMap haskellLib.getAllComponents (builtins.attrValues hsPkgs);
71+
allHsPkgsComponents = lib.concatMap haskellLib.getAllComponents
72+
(lib.filter (x: !(x.isRedirect or false)) (builtins.attrValues hsPkgs));
7273

7374
# Given a list of `depends`, removes those which are selected components
7475
removeSelectedInputs =
@@ -114,9 +115,10 @@ let
114115
# Set up a "dummy" component to use with ghcForComponent.
115116
component = {
116117
depends = packageInputs;
117-
libs = [];
118-
pkgconfig = [];
119-
frameworks = [];
118+
pre-existing = lib.concatMap (x: (haskellLib.dependToLib x).config.pre-existing or []) packageInputs;
119+
libs = lib.concatMap (x: (haskellLib.dependToLib x).config.libs or []) packageInputs;
120+
pkgconfig = lib.concatMap (x: (haskellLib.dependToLib x).config.pkgconfig or []) packageInputs;
121+
frameworks = lib.concatMap (x: (haskellLib.dependToLib x).config.frameworks or []) packageInputs;
120122
doExactConfig = false;
121123
};
122124
configFiles = makeConfigFiles {

changelog.md

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,82 @@
11
This file contains a summary of changes to Haskell.nix and `nix-tools`
22
that will impact users.
33

4+
## Sep 17, 2024
5+
6+
Cabal projects now use the more granular Unit IDs from plan.json
7+
to identify packages. This allows for different versions of a
8+
package to be used when building `built-tool-depends` and setup
9+
dependencies.
10+
11+
Overrides in the `modules` argument apply to all versions of
12+
the package. However to make this work we needed to make
13+
each `packages.somepackage` an option (instead of using an
14+
`attrsOf` the submodule type).
15+
16+
It is now an error to override a package that is not in the
17+
plan. This can be a problem if different GHC versions, target
18+
platforms, or cabal flag settings cause the package to be
19+
excluded from the plan. Adding `package-keys` can tell
20+
haskell.nix to include the option anyway:
21+
22+
```
23+
modules = [{
24+
# Tell haskell.nix that `somepackage` may exist.
25+
package-keys = ["somepackage"];
26+
# Now the following will not cause an error even
27+
# if `somepackage` is not in the plan
28+
packages.somepackage.flags.someflag = true;
29+
}];
30+
```
31+
32+
There is a helper function you can use to add `package-keys`
33+
for all of the `builtins.attrNames` of `packages`:
34+
35+
```
36+
modules = [(pkgs.haskell-nix.haskellLib.addPackageKeys {
37+
packages.somepackage.flags.someflag = true;
38+
})];
39+
```
40+
41+
Do not use the module's `pkgs` arg to look `addPackageKeys` up
42+
though or it will result an `infinite recursion` error.
43+
44+
Code that uses `options.packages` will also need to be updated.
45+
For instance the following code that uses `options.packages`
46+
to set `--Werror` for local packages:
47+
48+
```
49+
({ lib, ... }: {
50+
options.packages = lib.mkOption {
51+
type = lib.types.attrsOf (lib.types.submodule (
52+
{ config, lib, ... }:
53+
lib.mkIf config.package.isLocal
54+
{
55+
configureFlags = [ "--ghc-option=-Werror"];
56+
}
57+
));
58+
};
59+
})
60+
```
61+
62+
Now needs to do it for each of the entry in `config.package-keys`
63+
instead of using `attrsOf`:
64+
65+
```
66+
({ config, lib, ... }: {
67+
options.packages = lib.genAttrs config.package-keys (_:
68+
lib.mkOption {
69+
type = lib.types.submodule (
70+
{ config, lib, ... }:
71+
lib.mkIf config.package.isLocal
72+
{
73+
configureFlags = [ "--ghc-option=-Werror"];
74+
}
75+
);
76+
});
77+
})
78+
```
79+
480
## Jun 5, 2024
581

682
Haskell.nix now respects the `pre-existing` packages selected

lib/cover-project.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ in pkgs.runCommand "project-coverage-report"
8282
fi
8383
8484
# Copy mix, tix, and html information over from each report
85-
for f in $report/share/hpc/vanilla/mix/$identifier*; do
85+
for f in $report/share/hpc/vanilla/mix/*; do
8686
cp -Rn $f $out/share/hpc/vanilla/mix
8787
done
8888
cp -R $report/share/hpc/vanilla/tix/* $out/share/hpc/vanilla/tix/

lib/default.nix

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,13 @@ in {
9898
# Was there a reference to the package source in the `cabal.project`
9999
# This is used to make the default `packages` list for `shellFor`.
100100
isLocalPackage = p: p.isLocal or false;
101-
selectLocalPackages = lib.filterAttrs (_n: p: p != null && isLocalPackage p);
101+
isRedirectPackage = p: p.isRedirect or false;
102+
selectLocalPackages = lib.filterAttrs (_n: p: p != null && isLocalPackage p && !isRedirectPackage p);
102103

103104
# if it's a project package it has a src attribute set with an origSubDir attribute.
104105
# project packages are a subset of localPackages
105106
isProjectPackage = p: p.isProject or false;
106-
selectProjectPackages = lib.filterAttrs (_n: p: p != null && isLocalPackage p && isProjectPackage p);
107+
selectProjectPackages = lib.filterAttrs (_n: p: p != null && isLocalPackage p && isProjectPackage p && !isRedirectPackage p);
107108

108109
# Format a componentId as it should appear as a target on the
109110
# command line of the setup script.
@@ -336,7 +337,7 @@ in {
336337
# Converts from a `compoent.depends` value to a library derivation.
337338
# In the case of sublibs the `depends` value should already be the derivation.
338339
dependToLib = d:
339-
# Do simplify this to `d.components.library or d`, as that
340+
# Do not simplify this to `d.components.library or d`, as that
340341
# will not give a good error message if the `.library`
341342
# is missing (happens if the package is unplanned,
342343
# but has overrides).
@@ -609,4 +610,6 @@ in {
609610
}";
610611

611612
types = import ./types.nix { inherit lib; };
613+
614+
addPackageKeys = x: x // { package-keys = builtins.attrNames x.packages; };
612615
}

lib/load-cabal-plan.nix

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
{haskellLib, pkgs}:
2+
{callProjectResults, selectedCompiler}:
3+
let
4+
# Read the plan.json file `plan-nix` derivation
5+
plan-json = builtins.fromJSON (
6+
builtins.unsafeDiscardStringContext (
7+
builtins.readFile (callProjectResults.projectNix + "/plan.json")));
8+
# All the units in the plan indexed by unit ID.
9+
by-id = pkgs.lib.listToAttrs (map (x: { name = x.id; value = x; }) plan-json.install-plan);
10+
# Find the names of all the pre-existing packages used by a list of dependencies
11+
# (includes transitive dependencies)
12+
lookupPreExisting = depends:
13+
pkgs.lib.concatMap (d: builtins.attrNames pre-existing-depends.${d}) depends;
14+
pre-existing-depends =
15+
pkgs.lib.listToAttrs (map (p: {
16+
name = p.id;
17+
value = pkgs.lib.optionalAttrs (p.type == "pre-existing") { ${p.pkg-name} = null; } //
18+
pkgs.lib.listToAttrs (
19+
map (dname: { name = dname; value = null; }) (lookupPreExisting (p.depends or p.components.lib.depends)));
20+
}) plan-json.install-plan);
21+
# Lookup a dependency in `hsPkgs`
22+
lookupDependency = hsPkgs: d:
23+
pkgs.lib.optional (by-id.${d}.type != "pre-existing") (
24+
if by-id.${d}.component-name or "lib" == "lib"
25+
then hsPkgs.${d} or hsPkgs."${by-id.${d}.pkg-name}-${by-id.${d}.pkg-version}" or hsPkgs.${by-id.${d}.pkg-name}
26+
else hsPkgs.${d}.components.sublibs.${pkgs.lib.removePrefix "lib:" by-id.${d}.component-name});
27+
# Lookup an executable dependency in `hsPkgs.pkgsBuildBuild`
28+
lookupExeDependency = hsPkgs: d:
29+
# Try to lookup by ID, but if that fails use the name (currently a different plan is used by pkgsBuildBuild when cross compiling)
30+
(hsPkgs.pkgsBuildBuild.${d} or hsPkgs.pkgsBuildBuild.${by-id.${d}.pkg-name}).components.exes.${pkgs.lib.removePrefix "exe:" by-id.${d}.component-name};
31+
# Populate `depends`, `pre-existing` and `build-tools`
32+
lookupDependencies = hsPkgs: depends: exe-depends: {
33+
depends = pkgs.lib.concatMap (lookupDependency hsPkgs) depends;
34+
pre-existing = lookupPreExisting depends;
35+
build-tools = map (lookupExeDependency hsPkgs) exe-depends;
36+
};
37+
# Calculate the packages for a component
38+
getComponents = cabal2nixComponents: hsPkgs: p:
39+
let
40+
components = p.components or { ${p.component-name or "lib"} = { inherit (p) depends; exe-depends = p.exe-depends or []; }; };
41+
# Other than the `lib` and `setup` components, component names
42+
# have a prefix based on their type.
43+
componentsWithPrefix = collectionName: prefix:
44+
pkgs.lib.listToAttrs (pkgs.lib.concatLists (pkgs.lib.mapAttrsToList (n: c:
45+
pkgs.lib.optional (pkgs.lib.hasPrefix "${prefix}:" n) (
46+
let
47+
name = pkgs.lib.removePrefix "${prefix}:" n;
48+
value = (if cabal2nixComponents == null then {} else cabal2nixComponents.${collectionName}.${name}) // {
49+
buildable = true;
50+
} // lookupDependencies hsPkgs c.depends c.exe-depends;
51+
in { inherit name value; }
52+
)) components));
53+
in
54+
pkgs.lib.mapAttrs componentsWithPrefix haskellLib.componentPrefix
55+
// pkgs.lib.optionalAttrs (components ? lib) {
56+
library = (if cabal2nixComponents == null then {} else cabal2nixComponents.library) // {
57+
buildable = true;
58+
} // lookupDependencies hsPkgs components.lib.depends components.lib.exe-depends;
59+
} // pkgs.lib.optionalAttrs (components ? setup) {
60+
setup = {
61+
buildable = true;
62+
} // lookupDependencies hsPkgs.pkgsBuildBuild (components.setup.depends or []) (components.setup.exe-depends or []);
63+
};
64+
nixFilesDir = callProjectResults.projectNix + callProjectResults.src.origSubDir or "";
65+
in {
66+
# This replaces the `plan-nix/default.nix`
67+
pkgs = (hackage: {
68+
packages = pkgs.lib.listToAttrs (
69+
# Include entries for the `pre-existing` packages, but leave them as `null`
70+
pkgs.lib.concatMap (p:
71+
pkgs.lib.optional (p.type == "pre-existing") {
72+
name = p.id;
73+
value.revision = null;
74+
}) plan-json.install-plan
75+
# The other packages that are not part of the project itself.
76+
++ pkgs.lib.concatMap (p:
77+
pkgs.lib.optional (p.type == "configured" && (p.style == "global" || p.style == "inplace") ) {
78+
name = p.id;
79+
value.revision =
80+
{hsPkgs, ...}@args:
81+
let
82+
# Read the output of `Cabal2Nix.hs`. We need it for information not
83+
# in the `plan.json` file.
84+
cabal2nix = (
85+
if builtins.pathExists (nixFilesDir + "/cabal-files/${p.pkg-name}.nix")
86+
then import (nixFilesDir + "/cabal-files/${p.pkg-name}.nix")
87+
else if builtins.pathExists (nixFilesDir + "/.plan.nix/${p.pkg-name}.nix")
88+
then import (nixFilesDir + "/.plan.nix/${p.pkg-name}.nix")
89+
else (((hackage.${p.pkg-name}).${p.pkg-version}).revisions).default) (args // { hsPkgs = {}; });
90+
in pkgs.lib.optionalAttrs (p ? pkg-src-sha256) {
91+
sha256 = p.pkg-src-sha256;
92+
} // pkgs.lib.optionalAttrs (p.pkg-src.type or "" == "source-repo") {
93+
# Replace the source repository packages with versions created when
94+
# parsing the `cabal.project` file.
95+
src = pkgs.lib.lists.elemAt callProjectResults.sourceRepos (pkgs.lib.strings.toInt p.pkg-src.source-repo.location) + "/${p.pkg-src.source-repo.subdir}";
96+
} // pkgs.lib.optionalAttrs (cabal2nix ? package-description-override && p.pkg-version == cabal2nix.package.identifier.version) {
97+
# Use the `.cabal` file from the `Cabal2Nix` if it for the matching
98+
# version of the package (the one in the plan).
99+
inherit (cabal2nix) package-description-override;
100+
} // {
101+
flags = p.flags; # Use the flags from `plan.json`
102+
components = getComponents cabal2nix.components hsPkgs p;
103+
package = cabal2nix.package // {
104+
identifier = { name = p.pkg-name; version = p.pkg-version; id = p.id; };
105+
isProject = false;
106+
setup-depends = []; # The correct setup depends will be in `components.setup.depends`
107+
};
108+
};
109+
}) plan-json.install-plan);
110+
compiler = {
111+
inherit (selectedCompiler) version;
112+
};
113+
});
114+
# Packages in the project (those that are both configure and local)
115+
extras = (_hackage: {
116+
packages = pkgs.lib.listToAttrs (
117+
pkgs.lib.concatMap (p:
118+
pkgs.lib.optional (p.type == "configured" && p.style == "local") {
119+
name = p.id;
120+
value =
121+
{hsPkgs, ...}@args:
122+
let cabal2nix = import (nixFilesDir + "/.plan.nix/${p.pkg-name}.nix") (args // { hsPkgs = {}; });
123+
in pkgs.lib.optionalAttrs (p ? pkg-src-sha256) {
124+
sha256 = p.pkg-src-sha256;
125+
} // pkgs.lib.optionalAttrs (p.pkg-src.type or "" == "local" && cabal2nix ? cabal-generator) {
126+
inherit (cabal2nix) cabal-generator;
127+
} // pkgs.lib.optionalAttrs (p.pkg-src.type or "" == "local") {
128+
# Find the `src` location based on `p.pkg-src.path`
129+
src = if pkgs.lib.hasPrefix "/" p.pkg-src.path
130+
then p.pkg-src.path # Absolute path
131+
else haskellLib.appendSubDir {
132+
# Relative to the project path
133+
inherit (callProjectResults) src;
134+
subDir = pkgs.lib.removePrefix "./" (pkgs.lib.removePrefix "/" (pkgs.lib.removeSuffix "/." (pkgs.lib.removeSuffix "/." (
135+
if pkgs.lib.hasPrefix ".${callProjectResults.src.origSubDir or ""}/" (p.pkg-src.path + "/")
136+
then pkgs.lib.removePrefix ".${callProjectResults.src.origSubDir or ""}" p.pkg-src.path
137+
else throw "Unexpected path ${p.pkg-src.path} expected it to start with .${callProjectResults.src.origSubDir or ""}"))));
138+
includeSiblings = true; # Filtering sibling dirs of the package dir is done in the
139+
# component builder so that relative paths can be used to
140+
# reference project directories not in the package subDir.
141+
};
142+
} // {
143+
flags = p.flags; # Use the flags from `plan.json`
144+
components = getComponents cabal2nix.components hsPkgs p;
145+
package = cabal2nix.package // {
146+
identifier = { name = p.pkg-name; version = p.pkg-version; id = p.id; };
147+
isProject = true;
148+
setup-depends = []; # The correct setup depends will be in `components.setup.depends`
149+
};
150+
};
151+
}) plan-json.install-plan);
152+
});
153+
modules = [
154+
{ inherit plan-json; }
155+
(import ../modules/install-plan/non-reinstallable.nix)
156+
(import ../modules/install-plan/override-package-by-name.nix)
157+
(import ../modules/install-plan/planned.nix { inherit getComponents; })
158+
(import ../modules/install-plan/redirect.nix)
159+
];
160+
}

modules/component-driver.nix

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{ config, pkgs, lib, haskellLib, buildModules, ... }:
1+
{ config, options, pkgs, lib, haskellLib, buildModules, ... }:
22
let
33
builder = haskellLib.weakCallPackage pkgs ../builder {
44
inherit haskellLib;
@@ -94,6 +94,6 @@ in
9494
pkgsBuildBuild = buildModules.config.hsPkgs;
9595
} //
9696
lib.mapAttrs
97-
(_name: pkg: if pkg == null then null else builder.build-package config pkg)
97+
(name: pkg: if !(options.packages.${name}.isDefined or true) || pkg == null then null else builder.build-package config pkg)
9898
(config.packages // lib.genAttrs (config.nonReinstallablePkgs ++ config.bootPkgs) (_: null));
9999
}

modules/component.nix

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ in
3838
default = [ ];
3939
};
4040

41+
pre-existing = lib.mkOption {
42+
type = types.listOf types.str;
43+
default = [ ];
44+
};
45+
4146
libs = lib.mkOption {
4247
type = listOfFilteringNulls (types.either (types.nullOr types.package) (listOfFilteringNulls types.package));
4348
default = [ ];
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{pkgs, lib, config, ...}: {
2+
nonReinstallablePkgs = ["rts" "base" "ghc-prim" "integer-gmp" "integer-simple"]
3+
++ lib.optionals (builtins.compareVersions config.compiler.version "8.11" >= 0) [
4+
"ghc-bignum"]
5+
++ lib.optionals (builtins.compareVersions config.compiler.version "9.9" >= 0) [
6+
"ghc-internal"]
7+
++ lib.optionals (pkgs.stdenv.hostPlatform.isGhcjs) ([
8+
# ghci and its dependencies
9+
"ghci" "binary" "bytestring" "containers" "template-haskell" "array" "deepseq" "filepath" "ghc-boot" "ghc-boot-th" "ghc-heap" "transformers" "unix" "directory" "time" "ghc-platform" "os-string"]
10+
++ lib.optionals (builtins.compareVersions config.compiler.version "8.11" < 0) [
11+
"ghcjs-prim" "ghcjs-th"]);
12+
}
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Map overrides of the form `packages.${pkg-name}` to all the matching
2+
# packages in the plan.
3+
{pkgs, config, options, ...}: {
4+
use-package-keys = true;
5+
package-keys = map (p: p.pkg-name) config.plan-json.install-plan ++ map (p: p.id) config.plan-json.install-plan;
6+
packages = pkgs.lib.listToAttrs (map (p: {
7+
name = p.id;
8+
value = pkgs.lib.modules.mkAliasDefinitions (options.packages.${p.pkg-name});
9+
}) (pkgs.lib.filter (p: p.id != p.pkg-name) config.plan-json.install-plan));
10+
}

modules/install-plan/planned.nix

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Mark everthing in the install-plan as "planned"
2+
{getComponents}:
3+
{config, lib, ...}: {
4+
packages = lib.listToAttrs (map (p: {
5+
name = p.id;
6+
value.components = lib.mapAttrs (type: x:
7+
if type == "library" || type == "setup"
8+
then { planned = lib.mkOverride 900 true; }
9+
else
10+
lib.mapAttrs (_: _: {
11+
planned = lib.mkOverride 900 true;
12+
}) x
13+
) (getComponents null {} p);
14+
}) config.plan-json.install-plan);
15+
}

0 commit comments

Comments
 (0)