Skip to content

Add ciJobs and hydraJobs to the flake output #1761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Nov 4, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
52fbfd1
Add ciJobs and hydraJobs to the flake output
hamishmack Oct 20, 2022
5bd1869
Fix buildkite
hamishmack Oct 20, 2022
9e05b33
Fix buildkite
hamishmack Oct 20, 2022
005c6c3
Exclude unsupported ghc versions for ci
hamishmack Oct 20, 2022
5028316
ifdLevel 0
hamishmack Oct 20, 2022
4fd770b
ifdLevel 1
hamishmack Oct 20, 2022
445c559
ifdLevel 2
hamishmack Oct 21, 2022
47ee837
ifdLevel 3
hamishmack Oct 21, 2022
2283aca
Set up tullia for running ciJobs
hamishmack Oct 24, 2022
edfb8f3
Merge remote-tracking branch 'origin/master' into hkm/ci-flake
hamishmack Oct 25, 2022
9b65cc6
Test github webhook
hamishmack Oct 25, 2022
4aaa768
Add inputs arg back to release.nix
hamishmack Oct 26, 2022
c0b6ab8
Merge remote-tracking branch 'origin/master' into hkm/ci-flake
hamishmack Oct 26, 2022
1f094ff
Tidy up
hamishmack Oct 26, 2022
c917836
Default ci to evaluate on linux
hamishmack Oct 26, 2022
0ca2df0
Trace hydra eval and add windows-secp256k1 back
hamishmack Oct 26, 2022
2d9d714
Test bumping nixpkgs
hamishmack Oct 27, 2022
e7fb0d3
Fix for new nixpkgs
hamishmack Oct 27, 2022
f3ea694
ifdLevel 2
hamishmack Oct 27, 2022
284eea2
Revert "Test bumping nixpkgs"
hamishmack Oct 27, 2022
799bb58
ifdLevel 3
hamishmack Oct 27, 2022
68b5f88
Just include aggregated jobs on hydra
hamishmack Oct 28, 2022
9c1c92f
Merge remote-tracking branch 'origin/master' into hkm/ci-flake
hamishmack Nov 3, 2022
bd49d5b
ifdLevel 2
hamishmack Oct 27, 2022
988d213
ifdLevel 3
hamishmack Oct 27, 2022
f9f83df
Change order of hydra jobs
hamishmack Nov 3, 2022
e43ddeb
What if we just leave out required
hamishmack Nov 3, 2022
d4c52ac
Put non aggregated jobs again
hamishmack Nov 3, 2022
792b953
Add required back, but just for to check darwin worked
hamishmack Nov 4, 2022
567fdea
Fix for buildkite and add comment
hamishmack Nov 4, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ steps:

- label: 'Check that evaluation of hydra jobs works without using remote builders'
command:
- nix-instantiate release.nix -A required-unstable-ghc8107-x86_64-darwin-native --show-trace --builders ''
- nix-instantiate release.nix -A x86_64-darwin.required-unstable-ghc8107-native --show-trace --builders ''
agents:
system: x86_64-linux

Expand Down
4 changes: 1 addition & 3 deletions build.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
#
# It is separate from default.nix because that file is the public API
# of Haskell.nix, which shouldn't have tests, etc.
let
haskellNix = (import ./default.nix {});
in
{ nixpkgs ? haskellNix.sources.nixpkgs-unstable
, nixpkgsArgs ? haskellNix.nixpkgsArgs
, pkgs ? import nixpkgs nixpkgsArgs
Expand All @@ -13,6 +10,7 @@ in
, pkgsForHydra ? import nixpkgsForHydra (nixpkgsArgs // { inherit (pkgs) system; })
, ifdLevel ? 1000
, compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
, haskellNix ? (import ./default.nix {})
}:

let
Expand Down
2 changes: 1 addition & 1 deletion ci-lib.nix
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
# Generic nixpkgs, use *only* for lib functions that are stable across versions
pkgs ? (import ./. {}).pkgs,
pkgs,
lib ? pkgs.lib
}: rec {
inherit (import ./dimension.nix) dimension;
Expand Down
73 changes: 37 additions & 36 deletions ci.nix
Original file line number Diff line number Diff line change
@@ -1,59 +1,62 @@
# 'supportedSystems' restricts the set of systems that we will evaluate for. Useful when you're evaluating
# on a machine with e.g. no way to build the Darwin IFDs you need!
{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]
, ifdLevel ? 3
{ ifdLevel ? 3
# Whether or not we are evaluating in restricted mode. This is true in Hydra, but not in Hercules.
, restrictEval ? false
, checkMaterialization ? false
, pkgs ? (import ./. {}).pkgs }:
, compat
, system
, evalSystem ? builtins.currentSystem or "x86_64-linux"
, pkgs ? (compat { inherit system; }).pkgs }:
let
inherit (import ./ci-lib.nix { inherit pkgs; }) dimension platformFilterGeneric filterAttrsOnlyRecursive;
inherit (pkgs.haskell-nix) sources;
nixpkgsVersions = {
"R2205" = "nixpkgs-2205";
"unstable" = "nixpkgs-unstable";
};
haskellNix = import ./default.nix { inherit checkMaterialization; };
haskellNix = compat { inherit checkMaterialization system; };
nixpkgsArgs = haskellNix.nixpkgsArgs // {
# Needed for dwarf tests
config = haskellNix.nixpkgsArgs.config // {
permittedInsecurePackages = ["libdwarf-20210528" "libdwarf-20181024" "dwarfdump-20181024"];
};
};
compilerNixNames = nixpkgsName: nixpkgs: builtins.mapAttrs (compiler-nix-name: runTests: {
inherit runTests;
}) (
# GHC version to cache and whether to run the tests against them.
# This list of GHC versions should include everything for which we
# have a ./materialized/ghcXXX directory containing the materialized
# cabal-install and nix-tools plans. When removing a ghc version
# from here (so that is no longer cached) also remove ./materialized/ghcXXX.
# Update supported-ghc-versions.md to reflect any changes made here.
nixpkgs.lib.optionalAttrs (nixpkgsName == "R2205") {
ghc865 = false;
ghc8107 = false;
} // nixpkgs.lib.optionalAttrs (nixpkgsName == "unstable") {
ghc865 = false;
ghc884 = false; # Native version is used to boot 9.0.1
ghc8107 = true;
ghc902 = false;
ghc924 = true;
});
systems = nixpkgsName: nixpkgs: compiler-nix-name: nixpkgs.lib.genAttrs (
nixpkgs.lib.filter (v:
compilerNixNames = nixpkgsName: nixpkgs:
# Include only the GHC versions that are supported by haskell.nix
nixpkgs.lib.filterAttrs (compiler-nix-name: _:
# We have less x86_64-darwin build capacity so build fewer GhC versions
(v != "x86_64-darwin" || (
(system != "x86_64-darwin" || (
!__elem compiler-nix-name ["ghc8104" "ghc810420210212" "ghc8105" "ghc8106" "ghc901" "ghc921" "ghc922"]))
&&
# aarch64-darwin requires ghc 8.10.7
(v != "aarch64-darwin" || (
(system != "aarch64-darwin" || (
!__elem compiler-nix-name ["ghc865" "ghc884" "ghc8104" "ghc810420210212" "ghc8105" "ghc8106" "ghc901" "ghc921" "ghc922"]))
&&
# aarch64-linux requires ghc 8.8.4
(v != "aarch64-linux" || (
(system != "aarch64-linux" || (
!__elem compiler-nix-name ["ghc865" "ghc8104" "ghc810420210212" "ghc8105" "ghc8106" "ghc901" "ghc921" "ghc922"]
))) supportedSystems) (v: v);
crossSystems = nixpkgsName: nixpkgs: compiler-nix-name: system:
)))
(builtins.mapAttrs (compiler-nix-name: runTests: {
inherit runTests;
}) (
# GHC version to cache and whether to run the tests against them.
# This list of GHC versions should include everything for which we
# have a ./materialized/ghcXXX directory containing the materialized
# cabal-install and nix-tools plans. When removing a ghc version
# from here (so that is no longer cached) also remove ./materialized/ghcXXX.
# Update supported-ghc-versions.md to reflect any changes made here.
nixpkgs.lib.optionalAttrs (nixpkgsName == "R2205") {
ghc865 = false;
ghc8107 = false;
} // nixpkgs.lib.optionalAttrs (nixpkgsName == "unstable") {
ghc865 = false;
ghc884 = false; # Native version is used to boot 9.0.1
ghc8107 = true;
ghc902 = false;
ghc924 = true;
}));
crossSystems = nixpkgsName: nixpkgs: compiler-nix-name:
# We need to use the actual nixpkgs version we're working with here, since the values
# of 'lib.systems.examples' are not understood between all versions
let lib = nixpkgs.lib;
Expand All @@ -74,11 +77,10 @@
in
dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
evalPackages = import pinnedNixpkgsSrc nixpkgsArgs;
evalPackages = import pinnedNixpkgsSrc (nixpkgsArgs // { system = evalSystem; });
in dimension "GHC version" (compilerNixNames nixpkgsName evalPackages) (compiler-nix-name: {runTests}:
dimension "System" (systems nixpkgsName evalPackages compiler-nix-name) (systemName: system:
let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name haskellNix; };
platformFilter = platformFilterGeneric pkgs system;
in filterAttrsOnlyRecursive (_: v: platformFilter v && !(isDisabled v)) ({
# Native builds
Expand All @@ -95,10 +97,10 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
});
}
//
dimension "Cross system" (crossSystems nixpkgsName evalPackages compiler-nix-name system) (crossSystemName: crossSystem:
dimension "Cross system" (crossSystems nixpkgsName evalPackages compiler-nix-name) (crossSystemName: crossSystem:
# Cross builds
let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system crossSystem; });
build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name; };
build = import ./build.nix { inherit pkgs evalPackages ifdLevel compiler-nix-name haskellNix; };
in pkgs.recurseIntoAttrs (pkgs.lib.optionalAttrs (ifdLevel >= 1) ({
roots = pkgs.haskell-nix.roots' compiler-nix-name ifdLevel;
ghc = pkgs.buildPackages.haskell-nix.compiler."${compiler-nix-name}";
Expand All @@ -119,4 +121,3 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
))
)
)
)
Loading