Skip to content

Commit f9dbe37

Browse files
Lana243Lana243
authored
Add coverage and results statistics report (#261)
 add coverage and results statistics report
1 parent 7c600b5 commit f9dbe37

30 files changed

+291
-111
lines changed

‎server/proto/testgen.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ syntax = "proto3";
33
package testsgen;
44

55
import "util.proto";
6+
import "google/protobuf/duration.proto";
67

78
service TestsGenService {
89
rpc Handshake(DummyRequest) returns(DummyResponse) {}
@@ -188,6 +189,7 @@ message TestResultObject {
188189
string testname = 2;
189190
TestStatus status = 3;
190191
string output = 4;
192+
google.protobuf.Duration executionTime = 5;
191193
}
192194

193195
message CoverageAndResultsResponse {

‎server/src/KleeGenerator.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ KleeGenerator::buildByCDb(const CollectionUtils::MapFileTo<fs::path> &filesToBui
5757
fs::path makefile = projectTmpPath / "GenerationCompileMakefile.mk";
5858
FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());
5959

60-
auto command = MakefileUtils::makefileCommand(projectContext, makefile, "all");
60+
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "all");
6161
ExecUtils::ExecutionResult res = command.run();
6262
if (res.status != 0) {
6363
LOG_S(ERROR) << StringUtils::stringFormat("Make for \"%s\" failed.\nCommand: \"%s\"\n%s\n",
@@ -205,7 +205,7 @@ Result<fs::path> KleeGenerator::defaultBuild(const fs::path &hintPath,
205205
fs::path makefile = projectTmpPath / "BCForKLEE.mk";
206206
FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());
207207

208-
auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "build");
208+
auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "build");
209209
auto [out, status, _] = makefileCommand.run();
210210
if (status != 0) {
211211
LOG_S(ERROR) << "Compilation for " << sourceFilePath << " failed.\n"

‎server/src/Paths.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ namespace Paths {
183183
fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext) {
184184
return projectContext.buildDir / "utbot";
185185
}
186+
fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext) {
187+
return getArtifactsRootDir(projectContext) / "gtest-results.json";
188+
}
186189
fs::path getFlagsDir(const utbot::ProjectContext &projectContext) {
187190
return getArtifactsRootDir(projectContext) / "flags";
188191
}

‎server/src/Paths.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,8 @@ namespace Paths {
273273

274274
fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext);
275275

276+
fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext);
277+
276278
fs::path getFlagsDir(const utbot::ProjectContext &projectContext);
277279

278280
fs::path getTestExecDir(const utbot::ProjectContext &projectContext);

‎server/src/building/Linker.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ Result<Linker::LinkResult> Linker::link(const CollectionUtils::MapFileTo<fs::pat
364364
FileSystemUtils::writeToFile(linkMakefile, bitcodeLinkMakefilePrinter.ss.str());
365365

366366
auto command =
367-
MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
367+
MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
368368
auto [out, status, logFilePath] = command.run(testGen.serverBuildDir);
369369
if (status != 0) {
370370
std::string errorMessage =
@@ -477,7 +477,7 @@ Result<utbot::Void> Linker::linkWithStubsIfNeeded(const fs::path &linkMakefile,
477477
return errorMessage;
478478
}
479479

480-
auto command = MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, "all");
480+
auto command = MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, "all");
481481
auto [out, status, _] = command.run(testGen.serverBuildDir);
482482
if (status != 0) {
483483
std::string errorMessage =

‎server/src/coverage/Coverage.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#include "Coverage.h"
22

3-
int Coverage::TestStatusMap::getNumberOfTests() {
3+
int Coverage::TestResultMap::getNumberOfTests() {
44
int cnt = 0;
5-
for (auto const &[fileName, testsStatus] : *this) {
6-
cnt += testsStatus.size();
5+
for (auto const &[fileName, testsResult] : *this) {
6+
cnt += testsResult.size();
77
}
88
return cnt;
99
}

‎server/src/coverage/Coverage.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "utils/CollectionUtils.h"
55

66
#include <protobuf/testgen.grpc.pb.h>
7+
#include <google/protobuf/util/time_util.h>
78

89
#include <unordered_map>
910
#include <vector>
@@ -37,11 +38,11 @@ namespace Coverage {
3738
class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
3839
};
3940

40-
class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
41+
using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
42+
class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
4143
public:
4244
int getNumberOfTests();
4345
};
44-
4546
}
4647

4748
#endif //UNITTESTBOT_COVERAGE_H

‎server/src/coverage/CoverageAndResultsGenerator.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,17 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage,
3737
}
3838
} catch (CoverageGenerationException &e) {
3939
showErrors();
40+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4041
return Status(StatusCode::FAILED_PRECONDITION, e.what());
4142
} catch (ExecutionProcessException &e) {
4243
exceptions.emplace_back(e);
4344
showErrors();
45+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4446
return Status(StatusCode::FAILED_PRECONDITION, e.what());
4547
} catch (CancellationException &e) {
48+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4649
return Status::CANCELLED;
4750
}
48-
4951
showErrors();
5052
return Status::OK;
5153
}
@@ -65,7 +67,7 @@ void CoverageAndResultsGenerator::showErrors() const {
6567
errorMessage = message;
6668
}
6769

68-
coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
70+
coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
6971
}
7072

7173
Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -83,7 +85,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
8385
}
8486
std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
8587
CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
86-
return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
88+
return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
8789
testsgen::TEST_INTERRUPTED;
8890
}));
8991
if (coverageCommands.empty()) {

‎server/src/coverage/CoverageTool.cpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88

99
using namespace CompilationUtils;
1010

11-
CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWriter(progressWriter) {
11+
CoverageTool::CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) :
12+
projectContext(std::move(projectContext)), progressWriter(progressWriter) {
1213
}
1314

1415
std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommandsJsonPath,
@@ -29,6 +30,10 @@ std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommands
2930
}
3031
}
3132

32-
std::string CoverageTool::getTestFilter(const UnitTest &unitTest) const {
33-
return StringUtils::stringFormat("--gtest_filter=*.%s", unitTest.testname);
33+
std::string CoverageTool::getGTestFlags(const UnitTest &unitTest) const {
34+
std::string gtestFilterFlag = StringUtils::stringFormat("\"--gtest_filter=*.%s\"", unitTest.testname);
35+
std::string gtestOutputFlag = StringUtils::stringFormat("\"--gtest_output=json:%s\"",
36+
Paths::getGTestResultsJsonPath(projectContext));
37+
std::vector<std::string> gtestFlagsList = { gtestFilterFlag, gtestOutputFlag };
38+
return StringUtils::joinWith(gtestFlagsList, " ");
3439
}

‎server/src/coverage/CoverageTool.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@ struct BuildRunCommand {
2020
class CoverageTool {
2121
protected:
2222
ProgressWriter const *progressWriter;
23+
const utbot::ProjectContext projectContext;
2324

24-
[[nodiscard]] std::string getTestFilter(UnitTest const &unitTest) const;
25+
[[nodiscard]] std::string getGTestFlags(const UnitTest &unitTest) const;
2526

2627
public:
27-
explicit CoverageTool(ProgressWriter const *progressWriter);
28+
CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter);
2829

2930
[[nodiscard]] virtual std::vector<BuildRunCommand>
3031
getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch, bool withCoverage) = 0;

‎server/src/coverage/GcovCoverageTool.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ using Coverage::FileCoverage;
2424

2525
GcovCoverageTool::GcovCoverageTool(utbot::ProjectContext projectContext,
2626
ProgressWriter const *progressWriter)
27-
: CoverageTool(progressWriter), projectContext(std::move(projectContext)) {
27+
: CoverageTool(std::move(projectContext), progressWriter) {
2828
}
2929

3030
std::vector<BuildRunCommand>
@@ -38,11 +38,11 @@ GcovCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
3838
auto makefile = Paths::getMakefilePathFromSourceFilePath(
3939
projectContext,
4040
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath));
41-
auto gtestFlags = getTestFilter(testToLaunch);
41+
auto gtestFlags = getGTestFlags(testToLaunch);
4242
auto buildCommand =
43-
MakefileUtils::makefileCommand(projectContext, makefile, "build", gtestFlags);
43+
MakefileUtils::MakefileCommand(projectContext, makefile, "build", gtestFlags);
4444
auto runCommand =
45-
MakefileUtils::makefileCommand(projectContext, makefile, "run", gtestFlags);
45+
MakefileUtils::MakefileCommand(projectContext, makefile, "run", gtestFlags);
4646
result.push_back({ testToLaunch, buildCommand, runCommand });
4747
});
4848
return result;

‎server/src/coverage/GcovCoverageTool.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ class GcovCoverageTool : public CoverageTool {
2323
void cleanCoverage() const override;
2424

2525
private:
26-
const utbot::ProjectContext projectContext;
27-
2826
std::vector<fs::path> getGcdaFiles() const;
2927
};
3028

‎server/src/coverage/LlvmCoverageTool.cpp

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "LlvmCoverageTool.h"
22

3+
#include <utility>
4+
35
#include "Coverage.h"
46
#include "Paths.h"
57
#include "TimeExecStatistics.h"
@@ -20,7 +22,7 @@ using Coverage::FileCoverage;
2022

2123
LlvmCoverageTool::LlvmCoverageTool(utbot::ProjectContext projectContext,
2224
ProgressWriter const *progressWriter)
23-
: CoverageTool(progressWriter), projectContext(projectContext) {
25+
: CoverageTool(std::move(projectContext), progressWriter) {
2426
}
2527

2628
std::vector<BuildRunCommand>
@@ -30,15 +32,15 @@ LlvmCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
3032
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath);
3133
auto makefilePath = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
3234
auto testName = testToLaunch.testname;
33-
auto gtestFlags = getTestFilter(testToLaunch);
35+
auto gtestFlags = getGTestFlags(testToLaunch);
3436
std::vector<std::string> profileEnv;
3537
if (withCoverage) {
3638
auto profrawFilePath = Paths::getProfrawFilePath(projectContext, testName);
3739
profileEnv = { StringUtils::stringFormat("LLVM_PROFILE_FILE=%s", profrawFilePath) };
3840
}
39-
auto buildCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "build",
41+
auto buildCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "build",
4042
gtestFlags, profileEnv);
41-
auto runCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "run",
43+
auto runCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "run",
4244
gtestFlags, profileEnv);
4345
return BuildRunCommand{ testToLaunch, buildCommand, runCommand };
4446
});
@@ -73,7 +75,7 @@ LlvmCoverageTool::getCoverageCommands(const std::vector<UnitTest> &testsToLaunch
7375
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
7476
fs::path makefile =
7577
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
76-
auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "bin");
78+
auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "bin");
7779
auto res = makefileCommand.run();
7880
if (res.status == 0) {
7981
if (res.output.empty()) {

‎server/src/coverage/LlvmCoverageTool.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ class LlvmCoverageTool : public CoverageTool {
1717
[[nodiscard]] nlohmann::json getTotals() const override;
1818
void cleanCoverage() const override;
1919
private:
20-
const utbot::ProjectContext projectContext;
2120
void countLineCoverage(Coverage::CoverageMap& coverageMap, const std::string& filename) const;
2221
void checkLineForPartial(Coverage::FileCoverage::SourceLine line, Coverage::FileCoverage& fileCoverage) const;
2322
};

‎server/src/coverage/TestRunner.cpp

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "Paths.h"
55
#include "TimeExecStatistics.h"
66
#include "utils/FileSystemUtils.h"
7+
#include "utils/JsonUtils.h"
78
#include "utils/StringUtils.h"
89

910
#include "loguru.h"
@@ -38,7 +39,7 @@ TestRunner::TestRunner(
3839

3940
std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
4041
const fs::path &testFilePath) {
41-
auto cmdGetAllTests = MakefileUtils::makefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
42+
auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
4243
auto [out, status, _] = cmdGetAllTests.run(projectContext.buildDir, false);
4344
if (status != 0) {
4445
auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.buildDir, true);
@@ -129,21 +130,25 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
129130
MEASURE_FUNCTION_EXECUTION_TIME
130131
ExecUtils::throwIfCancelled();
131132

133+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
132134
const auto buildRunCommands = coverageTool->getBuildRunCommands(testsToLaunch, withCoverage);
133135
ExecUtils::doWorkWithProgress(buildRunCommands, progressWriter, "Running tests",
134136
[this, testTimeout] (BuildRunCommand const &buildRunCommand) {
135137
auto const &[unitTest, buildCommand, runCommand] =
136138
buildRunCommand;
137139
try {
138-
auto status = runTest(runCommand, testTimeout);
139-
testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
140+
auto status = runTest(buildRunCommand, testTimeout);
141+
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
140142
ExecUtils::throwIfCancelled();
141143
} catch (ExecutionProcessException const &e) {
142-
testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
144+
testsgen::TestResultObject testRes;
145+
testRes.set_testfilepath(unitTest.testFilePath);
146+
testRes.set_testname(unitTest.testname);
147+
testRes.set_status(testsgen::TEST_FAILED);
148+
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
143149
exceptions.emplace_back(e);
144150
}
145151
});
146-
147152
LOG_S(DEBUG) << "All run commands were executed";
148153
return Status::OK;
149154
}
@@ -166,7 +171,7 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs
166171
ExecUtils::throwIfCancelled();
167172
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
168173
if (fs::exists(makefile)) {
169-
auto command = MakefileUtils::makefileCommand(projectContext, makefile, "build", "", {});
174+
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "build", "", {});
170175
LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string();
171176
auto[out, status, logFilePath] = command.run(projectContext.buildDir, true);
172177
if (status != 0) {
@@ -187,23 +192,38 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
187192
return fail_count;
188193
}
189194

190-
testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
191-
auto res = command.run(projectContext.buildDir, true, true, testTimeout);
195+
testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
196+
auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
192197
GTestLogger::log(res.output);
193-
if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) {
194-
return testsgen::TEST_PASSED;
198+
199+
testsgen::TestResultObject testRes;
200+
testRes.set_testfilepath(command.unitTest.testFilePath);
201+
testRes.set_testname(command.unitTest.testname);
202+
*testRes.mutable_executiontime() = google::protobuf::util::TimeUtil::NanosecondsToDuration(0);
203+
204+
if (BaseForkTask::wasInterrupted(res.status)) {
205+
testRes.set_status(testsgen::TEST_INTERRUPTED);
206+
return testRes;
195207
}
196-
if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) {
197-
return testsgen::TEST_FAILED;
208+
if (!fs::exists(Paths::getGTestResultsJsonPath(projectContext))) {
209+
testRes.set_status(testsgen::TEST_DEATH);
210+
return testRes;
198211
}
199-
if (BaseForkTask::wasInterrupted(res.status)) {
200-
return testsgen::TEST_INTERRUPTED;
212+
nlohmann::json gtestResultsJson = JsonUtils::getJsonFromFile(Paths::getGTestResultsJsonPath(projectContext));
213+
if (!google::protobuf::util::TimeUtil::FromString(gtestResultsJson["time"], testRes.mutable_executiontime())) {
214+
LOG_S(WARNING) << "Cannot parse duration of test execution";
215+
}
216+
if (gtestResultsJson["failures"] != 0) {
217+
testRes.set_status(testsgen::TEST_FAILED);
218+
} else {
219+
testRes.set_status(testsgen::TEST_PASSED);
201220
}
202-
return testsgen::TEST_DEATH;
221+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
222+
return testRes;
203223
}
204224

205-
const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
206-
return testStatusMap;
225+
const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
226+
return testResultMap;
207227
}
208228

209229
bool TestRunner::hasExceptions() const {

0 commit comments

Comments
 (0)