Skip to content

Add coverage and results statistics report #261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 23, 2022
Merged

Add coverage and results statistics report #261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1382b39
add coverage and results statistics report
Lana243 Jun 20, 2022
6d3f0a2
Fix CI tests
Jun 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions server/proto/testgen.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ syntax = "proto3";
package testsgen;

import "util.proto";
import "google/protobuf/duration.proto";

service TestsGenService {
rpc Handshake(DummyRequest) returns(DummyResponse) {}
Expand Down Expand Up @@ -188,6 +189,7 @@ message TestResultObject {
string testname = 2;
TestStatus status = 3;
string output = 4;
google.protobuf.Duration executionTime = 5;
}

message CoverageAndResultsResponse {
Expand Down
4 changes: 2 additions & 2 deletions server/src/KleeGenerator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ KleeGenerator::buildByCDb(const CollectionUtils::MapFileTo<fs::path> &filesToBui
fs::path makefile = projectTmpPath / "GenerationCompileMakefile.mk";
FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());

auto command = MakefileUtils::makefileCommand(projectContext, makefile, "all");
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "all");
ExecUtils::ExecutionResult res = command.run();
if (res.status != 0) {
LOG_S(ERROR) << StringUtils::stringFormat("Make for \"%s\" failed.\nCommand: \"%s\"\n%s\n",
Expand Down Expand Up @@ -205,7 +205,7 @@ Result<fs::path> KleeGenerator::defaultBuild(const fs::path &hintPath,
fs::path makefile = projectTmpPath / "BCForKLEE.mk";
FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());

auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "build");
auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "build");
auto [out, status, _] = makefileCommand.run();
if (status != 0) {
LOG_S(ERROR) << "Compilation for " << sourceFilePath << " failed.\n"
Expand Down
3 changes: 3 additions & 0 deletions server/src/Paths.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ namespace Paths {
fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext) {
return projectContext.buildDir / "utbot";
}
fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext) {
return getArtifactsRootDir(projectContext) / "gtest-results.json";
}
fs::path getFlagsDir(const utbot::ProjectContext &projectContext) {
return getArtifactsRootDir(projectContext) / "flags";
}
Expand Down
2 changes: 2 additions & 0 deletions server/src/Paths.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,8 @@ namespace Paths {

fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext);

fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext);

fs::path getFlagsDir(const utbot::ProjectContext &projectContext);

fs::path getTestExecDir(const utbot::ProjectContext &projectContext);
Expand Down
4 changes: 2 additions & 2 deletions server/src/building/Linker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ Result<Linker::LinkResult> Linker::link(const CollectionUtils::MapFileTo<fs::pat
FileSystemUtils::writeToFile(linkMakefile, bitcodeLinkMakefilePrinter.ss.str());

auto command =
MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
auto [out, status, logFilePath] = command.run(testGen.serverBuildDir);
if (status != 0) {
std::string errorMessage =
Expand Down Expand Up @@ -477,7 +477,7 @@ Result<utbot::Void> Linker::linkWithStubsIfNeeded(const fs::path &linkMakefile,
return errorMessage;
}

auto command = MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, "all");
auto command = MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, "all");
auto [out, status, _] = command.run(testGen.serverBuildDir);
if (status != 0) {
std::string errorMessage =
Expand Down
6 changes: 3 additions & 3 deletions server/src/coverage/Coverage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#include "Coverage.h"

int Coverage::TestStatusMap::getNumberOfTests() {
int Coverage::TestResultMap::getNumberOfTests() {
int cnt = 0;
for (auto const &[fileName, testsStatus] : *this) {
cnt += testsStatus.size();
for (auto const &[fileName, testsResult] : *this) {
cnt += testsResult.size();
}
return cnt;
}
5 changes: 3 additions & 2 deletions server/src/coverage/Coverage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "utils/CollectionUtils.h"

#include <protobuf/testgen.grpc.pb.h>
#include <google/protobuf/util/time_util.h>

#include <unordered_map>
#include <vector>
Expand Down Expand Up @@ -37,11 +38,11 @@ namespace Coverage {
class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
};

class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
public:
int getNumberOfTests();
};

}

#endif //UNITTESTBOT_COVERAGE_H
8 changes: 5 additions & 3 deletions server/src/coverage/CoverageAndResultsGenerator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,17 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage,
}
} catch (CoverageGenerationException &e) {
showErrors();
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
return Status(StatusCode::FAILED_PRECONDITION, e.what());
} catch (ExecutionProcessException &e) {
exceptions.emplace_back(e);
showErrors();
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
return Status(StatusCode::FAILED_PRECONDITION, e.what());
} catch (CancellationException &e) {
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
return Status::CANCELLED;
}

showErrors();
return Status::OK;
}
Expand All @@ -65,7 +67,7 @@ void CoverageAndResultsGenerator::showErrors() const {
errorMessage = message;
}

coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
}

Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
Expand All @@ -83,7 +85,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
}
std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
testsgen::TEST_INTERRUPTED;
}));
if (coverageCommands.empty()) {
Expand Down
11 changes: 8 additions & 3 deletions server/src/coverage/CoverageTool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

using namespace CompilationUtils;

CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWriter(progressWriter) {
CoverageTool::CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) :
projectContext(std::move(projectContext)), progressWriter(progressWriter) {
}

std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommandsJsonPath,
Expand All @@ -29,6 +30,10 @@ std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommands
}
}

std::string CoverageTool::getTestFilter(const UnitTest &unitTest) const {
return StringUtils::stringFormat("--gtest_filter=*.%s", unitTest.testname);
std::string CoverageTool::getGTestFlags(const UnitTest &unitTest) const {
std::string gtestFilterFlag = StringUtils::stringFormat("\"--gtest_filter=*.%s\"", unitTest.testname);
std::string gtestOutputFlag = StringUtils::stringFormat("\"--gtest_output=json:%s\"",
Paths::getGTestResultsJsonPath(projectContext));
std::vector<std::string> gtestFlagsList = { gtestFilterFlag, gtestOutputFlag };
return StringUtils::joinWith(gtestFlagsList, " ");
}
5 changes: 3 additions & 2 deletions server/src/coverage/CoverageTool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ struct BuildRunCommand {
class CoverageTool {
protected:
ProgressWriter const *progressWriter;
const utbot::ProjectContext projectContext;

[[nodiscard]] std::string getTestFilter(UnitTest const &unitTest) const;
[[nodiscard]] std::string getGTestFlags(const UnitTest &unitTest) const;

public:
explicit CoverageTool(ProgressWriter const *progressWriter);
CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter);

[[nodiscard]] virtual std::vector<BuildRunCommand>
getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch, bool withCoverage) = 0;
Expand Down
8 changes: 4 additions & 4 deletions server/src/coverage/GcovCoverageTool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ using Coverage::FileCoverage;

GcovCoverageTool::GcovCoverageTool(utbot::ProjectContext projectContext,
ProgressWriter const *progressWriter)
: CoverageTool(progressWriter), projectContext(std::move(projectContext)) {
: CoverageTool(std::move(projectContext), progressWriter) {
}

std::vector<BuildRunCommand>
Expand All @@ -38,11 +38,11 @@ GcovCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
auto makefile = Paths::getMakefilePathFromSourceFilePath(
projectContext,
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath));
auto gtestFlags = getTestFilter(testToLaunch);
auto gtestFlags = getGTestFlags(testToLaunch);
auto buildCommand =
MakefileUtils::makefileCommand(projectContext, makefile, "build", gtestFlags);
MakefileUtils::MakefileCommand(projectContext, makefile, "build", gtestFlags);
auto runCommand =
MakefileUtils::makefileCommand(projectContext, makefile, "run", gtestFlags);
MakefileUtils::MakefileCommand(projectContext, makefile, "run", gtestFlags);
result.push_back({ testToLaunch, buildCommand, runCommand });
});
return result;
Expand Down
2 changes: 0 additions & 2 deletions server/src/coverage/GcovCoverageTool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ class GcovCoverageTool : public CoverageTool {
void cleanCoverage() const override;

private:
const utbot::ProjectContext projectContext;

std::vector<fs::path> getGcdaFiles() const;
};

Expand Down
12 changes: 7 additions & 5 deletions server/src/coverage/LlvmCoverageTool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#include "LlvmCoverageTool.h"

#include <utility>

#include "Coverage.h"
#include "Paths.h"
#include "TimeExecStatistics.h"
Expand All @@ -20,7 +22,7 @@ using Coverage::FileCoverage;

LlvmCoverageTool::LlvmCoverageTool(utbot::ProjectContext projectContext,
ProgressWriter const *progressWriter)
: CoverageTool(progressWriter), projectContext(projectContext) {
: CoverageTool(std::move(projectContext), progressWriter) {
}

std::vector<BuildRunCommand>
Expand All @@ -30,15 +32,15 @@ LlvmCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath);
auto makefilePath = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
auto testName = testToLaunch.testname;
auto gtestFlags = getTestFilter(testToLaunch);
auto gtestFlags = getGTestFlags(testToLaunch);
std::vector<std::string> profileEnv;
if (withCoverage) {
auto profrawFilePath = Paths::getProfrawFilePath(projectContext, testName);
profileEnv = { StringUtils::stringFormat("LLVM_PROFILE_FILE=%s", profrawFilePath) };
}
auto buildCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "build",
auto buildCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "build",
gtestFlags, profileEnv);
auto runCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "run",
auto runCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "run",
gtestFlags, profileEnv);
return BuildRunCommand{ testToLaunch, buildCommand, runCommand };
});
Expand Down Expand Up @@ -73,7 +75,7 @@ LlvmCoverageTool::getCoverageCommands(const std::vector<UnitTest> &testsToLaunch
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
fs::path makefile =
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "bin");
auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "bin");
auto res = makefileCommand.run();
if (res.status == 0) {
if (res.output.empty()) {
Expand Down
1 change: 0 additions & 1 deletion server/src/coverage/LlvmCoverageTool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ class LlvmCoverageTool : public CoverageTool {
[[nodiscard]] nlohmann::json getTotals() const override;
void cleanCoverage() const override;
private:
const utbot::ProjectContext projectContext;
void countLineCoverage(Coverage::CoverageMap& coverageMap, const std::string& filename) const;
void checkLineForPartial(Coverage::FileCoverage::SourceLine line, Coverage::FileCoverage& fileCoverage) const;
};
Expand Down
54 changes: 37 additions & 17 deletions server/src/coverage/TestRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "Paths.h"
#include "TimeExecStatistics.h"
#include "utils/FileSystemUtils.h"
#include "utils/JsonUtils.h"
#include "utils/StringUtils.h"

#include "loguru.h"
Expand Down Expand Up @@ -38,7 +39,7 @@ TestRunner::TestRunner(

std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
const fs::path &testFilePath) {
auto cmdGetAllTests = MakefileUtils::makefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
auto [out, status, _] = cmdGetAllTests.run(projectContext.buildDir, false);
if (status != 0) {
auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.buildDir, true);
Expand Down Expand Up @@ -129,21 +130,25 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
MEASURE_FUNCTION_EXECUTION_TIME
ExecUtils::throwIfCancelled();

fs::remove(Paths::getGTestResultsJsonPath(projectContext));
const auto buildRunCommands = coverageTool->getBuildRunCommands(testsToLaunch, withCoverage);
ExecUtils::doWorkWithProgress(buildRunCommands, progressWriter, "Running tests",
[this, testTimeout] (BuildRunCommand const &buildRunCommand) {
auto const &[unitTest, buildCommand, runCommand] =
buildRunCommand;
try {
auto status = runTest(runCommand, testTimeout);
testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
auto status = runTest(buildRunCommand, testTimeout);
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
ExecUtils::throwIfCancelled();
} catch (ExecutionProcessException const &e) {
testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
testsgen::TestResultObject testRes;
testRes.set_testfilepath(unitTest.testFilePath);
testRes.set_testname(unitTest.testname);
testRes.set_status(testsgen::TEST_FAILED);
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
exceptions.emplace_back(e);
}
});

LOG_S(DEBUG) << "All run commands were executed";
return Status::OK;
}
Expand All @@ -166,7 +171,7 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs
ExecUtils::throwIfCancelled();
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
if (fs::exists(makefile)) {
auto command = MakefileUtils::makefileCommand(projectContext, makefile, "build", "", {});
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "build", "", {});
LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string();
auto[out, status, logFilePath] = command.run(projectContext.buildDir, true);
if (status != 0) {
Expand All @@ -187,23 +192,38 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
return fail_count;
}

testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
auto res = command.run(projectContext.buildDir, true, true, testTimeout);
testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
GTestLogger::log(res.output);
if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) {
return testsgen::TEST_PASSED;

testsgen::TestResultObject testRes;
testRes.set_testfilepath(command.unitTest.testFilePath);
testRes.set_testname(command.unitTest.testname);
*testRes.mutable_executiontime() = google::protobuf::util::TimeUtil::NanosecondsToDuration(0);

if (BaseForkTask::wasInterrupted(res.status)) {
testRes.set_status(testsgen::TEST_INTERRUPTED);
return testRes;
}
if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) {
return testsgen::TEST_FAILED;
if (!fs::exists(Paths::getGTestResultsJsonPath(projectContext))) {
testRes.set_status(testsgen::TEST_DEATH);
return testRes;
}
if (BaseForkTask::wasInterrupted(res.status)) {
return testsgen::TEST_INTERRUPTED;
nlohmann::json gtestResultsJson = JsonUtils::getJsonFromFile(Paths::getGTestResultsJsonPath(projectContext));
if (!google::protobuf::util::TimeUtil::FromString(gtestResultsJson["time"], testRes.mutable_executiontime())) {
LOG_S(WARNING) << "Cannot parse duration of test execution";
}
if (gtestResultsJson["failures"] != 0) {
testRes.set_status(testsgen::TEST_FAILED);
} else {
testRes.set_status(testsgen::TEST_PASSED);
}
return testsgen::TEST_DEATH;
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
return testRes;
}

const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
return testStatusMap;
const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
return testResultMap;
}

bool TestRunner::hasExceptions() const {
Expand Down
Loading