Skip to content

Improve coverage reports #1548

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 19, 2022
Merged

Improve coverage reports #1548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b245f7e
Provide an empty coverage report when package has no coverage
sevanspowell Jul 15, 2022
4aa1063
Improve project coverage report index page
sevanspowell Jul 15, 2022
3820af5
Add warning explaining modules with no coverage
sevanspowell Jul 15, 2022
e01f641
Simplify interface of coverageReport
sevanspowell Jul 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 44 additions & 47 deletions lib/cover-project.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,12 @@ coverageReports:
let
toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";

# Create table rows for a project coverage index page that look something like:
#
# | Package |
# |------------------|
# | cardano-shell |
# | cardano-launcher |
coverageTableRows = coverageReport:
# Create a list element for a project coverage index page.
coverageListElement = coverageReport:
''
<tr>
<td>
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</href>
</td>
</tr>
<li>
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</a>
</li>
'';

projectIndexHtml = pkgs.writeText "index.html" ''
Expand All @@ -31,23 +24,32 @@ let
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
</head>
<body>
<table border="1" width="100%">
<tbody>
<tr>
<th>Report</th>
</tr>
${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)}
</tbody>
</table>
<div>
WARNING: Modules with no coverage are not included in any of these reports, this is just how HPC works under the hood.
</div>
<div>
<h2>Union Report</h2>
<p>The following report shows how each module is covered by any test in the project:</p>
<ul>
<li>
<a href="all/hpc_index.html">all</a>
</li>
</ul>
</div>
<div>
<h2>Individual Reports</h2>
<p>The following reports show how the tests of each package cover modules in the project:</p>
<ul>
${with lib; concatStringsSep "\n" (map coverageListElement coverageReports)}
</ul>
</div>
</body>
</html>
'';

ghc = project.pkg-set.config.ghc.package;

libs = lib.remove null (map (r: r.library) coverageReports);
libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports);

mixDirs =
map
Expand All @@ -56,6 +58,17 @@ let

srcDirs = map (l: l.srcSubDirPath) libs;

allCoverageReport = haskellLib.coverageReport {
name = "all";
checks = lib.flatten (lib.concatMap
(pkg: lib.optional (pkg ? checks) (lib.filter lib.isDerivation (lib.attrValues pkg.checks)))
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)));
mixLibraries = lib.concatMap
(pkg: lib.optional (pkg.components ? library) pkg.components.library)
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
ghc = project.pkg-set.config.ghc.package;
};

in pkgs.runCommand "project-coverage-report"
({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
LANG = "en_US.UTF-8";
Expand Down Expand Up @@ -124,30 +137,14 @@ in pkgs.runCommand "project-coverage-report"
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
'') coverageReports)}
if [ ''${#tixFiles[@]} -ne 0 ]; then
# Create tix file with test run information for all packages
tixFile="$out/share/hpc/vanilla/tix/all/all.tix"
hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile")
hpcSumCmd+=("''${tixFiles[@]}")
echo "''${hpcSumCmd[@]}"
eval "''${hpcSumCmd[@]}"
# Markup a HTML coverage report for the entire project
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
echo "report coverage-per-package $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
local markupOutDir="$out/share/hpc/vanilla/html/all"
local srcDirs=${toBashArray srcDirs}
local mixDirs=${toBashArray mixDirs}
local allMixModules=()
mkdir $markupOutDir
findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix"
# Copy out "all" coverage report
cp -R ${allCoverageReport}/share/hpc/vanilla/tix/all $out/share/hpc/vanilla/tix
cp -R ${allCoverageReport}/share/hpc/vanilla/html/all $out/share/hpc/vanilla/html
markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile"
# Markup a HTML coverage summary report for the entire project
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
fi
echo "report coverage $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
''
171 changes: 102 additions & 69 deletions lib/cover.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
# The following collects coverage information from a set of given "checks" and
# provides a coverage report showing how those "checks" cover a set of given
# "mixLibraries".
{ stdenv, lib, haskellLib, pkgs }:

# Name of the coverage report, which should be unique
# Name of the coverage report, which should be unique.
{ name
# Library to check coverage of
, library
# List of check derivations that generate coverage
, checks
# List of other libraries to include in the coverage report. The
# default value if just the derivation provided as the `library`
# argument. Use a larger list of libraries if you would like the tests
# of one local package to generate coverage for another.
, mixLibraries ? [library]
# hack for project-less projects
, ghc ? library.project.pkg-set.config.ghc.package
# List of check derivations that generate coverage.
, checks ? []
# List of libraries to include in the coverage report. If one of the above
# checks generates coverage for a particular library, coverage will only
# be included if that library is in this list.
, mixLibraries ? []
# Hack for project-less projects.
, ghc ? if mixLibraries == [] then null else (lib.head mixLibraries).project.pkg-set.config.ghc.package
}:

let
Expand All @@ -26,7 +26,7 @@ let
in pkgs.runCommand (name + "-coverage-report")
({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
passthru = {
inherit name library checks;
inherit name checks mixLibraries;
};
# HPC will fail if the Haskell file contains non-ASCII characters,
# unless our locale is set correctly. This has been fixed, but we
Expand Down Expand Up @@ -70,71 +70,105 @@ in pkgs.runCommand (name + "-coverage-report")
local -n tixFs=$2
local outFile="$3"

local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
if (( "''${#tixFs[@]}" > 0 )); then
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")

for module in "''${includedModules[@]}"; do
hpcSumCmd+=("--include=$module")
done
for module in "''${includedModules[@]}"; do
hpcSumCmd+=("--include=$module")
done

for tixFile in "''${tixFs[@]}"; do
hpcSumCmd+=("$tixFile")
done
for tixFile in "''${tixFs[@]}"; do
hpcSumCmd+=("$tixFile")
done

echo "''${hpcSumCmd[@]}"
eval "''${hpcSumCmd[@]}"
echo "''${hpcSumCmd[@]}"
eval "''${hpcSumCmd[@]}"
else
# If there are no tix files we output an empty tix file so that we can
# markup an empty HTML coverage report. This is preferable to failing to
# output a HTML report.
echo 'Tix []' > $outFile
fi
}

function findModules() {
local searchDir=$2
local -n result=$1
local -n searchDirs=$2
local pattern=$3

pushd $searchDir
mapfile -d $'\0' $1 < <(find ./ -type f \
-wholename "$pattern" -not -name "Paths*" \
-exec basename {} \; \
| sed "s/\.mix$//" \
| tr "\n" "\0")
popd
for dir in "''${searchDirs[@]}"; do
pushd $dir
local temp=()
mapfile -d $'\0' temp < <(find ./ -type f \
-wholename "$pattern" -not -name "Paths*" \
-exec basename {} \; \
| sed "s/\.mix$//" \
| tr "\n" "\0")
result+=("''${temp[@]}")
popd
done
}

local mixDirs=${toBashArray mixDirs}

mkdir -p $out/nix-support
mkdir -p $out/share/hpc/vanilla/mix/${name}
mkdir -p $out/share/hpc/vanilla/mix/
mkdir -p $out/share/hpc/vanilla/tix/${name}
mkdir -p $out/share/hpc/vanilla/html/${name}

# Copy over mix files verbatim
local srcDirs=${toBashArray srcDirs}
local mixDirs=${toBashArray mixDirs}

# Copy out mix files used for this report
for dir in "''${mixDirs[@]}"; do
if [ -d "$dir" ]; then
cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name}
cp -R "$dir" $out/share/hpc/vanilla/mix/
fi
done

local srcDirs=${toBashArray srcDirs}
local allMixModules=()
local pkgMixModules=()

# The behaviour of stack coverage reports is to provide tix files
# that include coverage information for every local package, but
# to provide HTML reports that only include coverage info for the
# current package. We emulate the same behaviour here. If the user
# includes all local packages in the mix libraries argument, they
# will get a coverage report very similar to stack.

# All mix modules
findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix"
# Only mix modules corresponding to this package
findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix"

# For each test
local mixModules=()
# Mix modules for all packages in "mixLibraries"
findModules mixModules mixDirs "*.mix"

# We need to make a distinction between library "exposed-modules" and
# "other-modules" used in test suites:
# - "exposed-modules" are addressed as "$library-$version-$hash/module"
# - "other-modules" are addressed as "module"
#
# This complicates the code required to find the mix modules. For a given mix directory:
#
# mix
# └── ntp-client-0.0.1
# └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5
# ├── Network.NTP.Client.mix
# ├── Network.NTP.Client.Packet.mix
# └── Network.NTP.Client.Query.mix
#
# Iff ntp-client uses "other-modules" in a test suite, both:
# - "mix/ntp-client-0.0.1", and
# - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5"
# need to be provided to hpc as search directories.
#
# I'd prefer to just exclude "other-modules", but I can't think of an easy
# way to do that in bash.
#
# Here we expand the search dirs and modify the mix dirs accordingly:
for dir in "''${mixDirs[@]}"; do
local otherModulesSearchDirs=()
# Simply consider any directory with a mix file as a search directory.
mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \
-wholename "*.mix" \
-exec dirname {} \; \
| uniq \
| tr "\n" "\0")
mixDirs+=("''${otherModulesSearchDirs[@]}")
done

local tixFiles=()
${lib.concatStringsSep "\n" (builtins.map (check: ''
if [ -d "${check}/share/hpc/vanilla/tix" ]; then
pushd ${check}/share/hpc/vanilla/tix

tixFile="$(find . -iwholename "*.tix" -type f -print -quit)"
local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile"
local newTixFile=$out/share/hpc/vanilla/tix/${check.name}/"$(basename $tixFile)"

mkdir -p "$(dirname $newTixFile)"
# Copy over the tix file verbatim
Expand All @@ -143,29 +177,28 @@ in pkgs.runCommand (name + "-coverage-report")
# Add the tix file to our list
tixFiles+=("$newTixFile")

# Create a coverage report for *just that test*
markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile"
# Create a coverage report for *just that check* affecting any of the
# "mixLibraries"
markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile"

popd
fi
'') checks)
}

# Sum tix files to create a tix file with all relevant tix
# information and markup a HTML report from this info.
if (( "''${#tixFiles[@]}" > 0 )); then
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
# Sum tix files to create a tix file with tix information from all tests in
# the package and markup a HTML report from this info.
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
local markupOutDir="$out/share/hpc/vanilla/html/${name}"

# Sum all of our tix file, including modules from any local package
sumTix allMixModules tixFiles "$sumTixFile"
# Sum all of our tix files
sumTix mixModules tixFiles "$sumTixFile"

# Markup a HTML report, included modules from only this package
markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile"
# Markup a HTML report
markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile"

# Provide a HTML zipfile and Hydra links
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
fi
# Provide a HTML zipfile and Hydra links
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
''
3 changes: 2 additions & 1 deletion overlays/haskell.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -591,8 +591,9 @@ final: prev: {

coverageReport = haskellLib.coverageReport (rec {
name = package.identifier.name + "-" + package.identifier.version;
library = if components ? library then components.library else null;
# Include the checks for a single package.
checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks);
# Checks from that package may provide coverage information for any library in the project.
mixLibraries = final.lib.concatMap
(pkg: final.lib.optional (pkg.components ? library) pkg.components.library)
(final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
Expand Down