Skip to content

Commit 5a2d0ec

Browse files
Lana243Lana243
committed
add coverage and results statistics report
1 parent 58cd8ce commit 5a2d0ec

30 files changed

+295
-107
lines changed

‎server/proto/testgen.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ syntax = "proto3";
33
package testsgen;
44

55
import "util.proto";
6+
import "google/protobuf/duration.proto";
67

78
service TestsGenService {
89
rpc Handshake(DummyRequest) returns(DummyResponse) {}
@@ -188,6 +189,7 @@ message TestResultObject {
188189
string testname = 2;
189190
TestStatus status = 3;
190191
string output = 4;
192+
google.protobuf.Duration executionTime = 5;
191193
}
192194

193195
message CoverageAndResultsResponse {

‎server/src/KleeGenerator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ KleeGenerator::buildByCDb(const CollectionUtils::MapFileTo<fs::path> &filesToBui
5757
fs::path makefile = projectTmpPath / "GenerationCompileMakefile.mk";
5858
FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());
5959

60-
auto command = MakefileUtils::makefileCommand(projectContext, makefile, "all");
60+
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "all");
6161
ExecUtils::ExecutionResult res = command.run();
6262
if (res.status != 0) {
6363
LOG_S(ERROR) << StringUtils::stringFormat("Make for \"%s\" failed.\nCommand: \"%s\"\n%s\n",

‎server/src/Paths.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ namespace Paths {
183183
fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext) {
184184
return projectContext.buildDir / "utbot";
185185
}
186+
fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext) {
187+
return getArtifactsRootDir(projectContext) / "gtest-results.json";
188+
}
186189
fs::path getFlagsDir(const utbot::ProjectContext &projectContext) {
187190
return getArtifactsRootDir(projectContext) / "flags";
188191
}

‎server/src/Paths.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,8 @@ namespace Paths {
273273

274274
fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext);
275275

276+
fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext);
277+
276278
fs::path getFlagsDir(const utbot::ProjectContext &projectContext);
277279

278280
fs::path getTestExecDir(const utbot::ProjectContext &projectContext);

‎server/src/building/Linker.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ Result<Linker::LinkResult> Linker::link(const CollectionUtils::MapFileTo<fs::pat
364364
FileSystemUtils::writeToFile(linkMakefile, bitcodeLinkMakefilePrinter.ss.str());
365365

366366
auto command =
367-
MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
367+
MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
368368
auto [out, status, logFilePath] = command.run(testGen.serverBuildDir);
369369
if (status != 0) {
370370
std::string errorMessage =
@@ -477,7 +477,7 @@ Result<utbot::Void> Linker::linkWithStubsIfNeeded(const fs::path &linkMakefile,
477477
return errorMessage;
478478
}
479479

480-
auto command = MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, "all");
480+
auto command = MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, "all");
481481
auto [out, status, _] = command.run(testGen.serverBuildDir);
482482
if (status != 0) {
483483
std::string errorMessage =

‎server/src/coverage/Coverage.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#include "Coverage.h"
22

3-
int Coverage::TestStatusMap::getNumberOfTests() {
3+
int Coverage::TestResultMap::getNumberOfTests() {
44
int cnt = 0;
5-
for (auto const &[fileName, testsStatus] : *this) {
6-
cnt += testsStatus.size();
5+
for (auto const &[fileName, testsResult] : *this) {
6+
cnt += testsResult.size();
77
}
88
return cnt;
99
}

‎server/src/coverage/Coverage.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "utils/CollectionUtils.h"
55

66
#include <protobuf/testgen.grpc.pb.h>
7+
#include <google/protobuf/util/time_util.h>
78

89
#include <unordered_map>
910
#include <vector>
@@ -37,11 +38,11 @@ namespace Coverage {
3738
class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
3839
};
3940

40-
class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
41+
using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
42+
class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
4143
public:
4244
int getNumberOfTests();
4345
};
44-
4546
}
4647

4748
#endif //UNITTESTBOT_COVERAGE_H

‎server/src/coverage/CoverageAndResultsGenerator.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,19 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage,
3737
}
3838
} catch (CoverageGenerationException &e) {
3939
showErrors();
40+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4041
return Status(StatusCode::FAILED_PRECONDITION, e.what());
4142
} catch (ExecutionProcessException &e) {
4243
exceptions.emplace_back(e);
4344
showErrors();
45+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4446
return Status(StatusCode::FAILED_PRECONDITION, e.what());
4547
} catch (CancellationException &e) {
48+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
4649
return Status::CANCELLED;
4750
}
48-
4951
showErrors();
52+
fs::remove(Paths::getGTestResultsJsonPath(projectContext));
5053
return Status::OK;
5154
}
5255

@@ -65,7 +68,7 @@ void CoverageAndResultsGenerator::showErrors() const {
6568
errorMessage = message;
6669
}
6770

68-
coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
71+
coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
6972
}
7073

7174
Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -83,7 +86,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
8386
}
8487
std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
8588
CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
86-
return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
89+
return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
8790
testsgen::TEST_INTERRUPTED;
8891
}));
8992
if (coverageCommands.empty()) {

‎server/src/coverage/CoverageTool.cpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88

99
using namespace CompilationUtils;
1010

11-
CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWriter(progressWriter) {
11+
CoverageTool::CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) :
12+
projectContext(std::move(projectContext)), progressWriter(progressWriter) {
1213
}
1314

1415
std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommandsJsonPath,
@@ -29,6 +30,10 @@ std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommands
2930
}
3031
}
3132

32-
std::string CoverageTool::getTestFilter(const UnitTest &unitTest) const {
33-
return StringUtils::stringFormat("--gtest_filter=*.%s", unitTest.testname);
33+
std::string CoverageTool::getGTestFlags(const UnitTest &unitTest) const {
34+
std::string gtestFilterFlag = StringUtils::stringFormat("\"--gtest_filter=*.%s\"", unitTest.testname);
35+
std::string gtestOutputFlag = StringUtils::stringFormat("\"--gtest_output=json:%s\"",
36+
Paths::getGTestResultsJsonPath(projectContext));
37+
std::vector<std::string> gtestFlagsList = { gtestFilterFlag, gtestOutputFlag };
38+
return StringUtils::joinWith(gtestFlagsList, " ");
3439
}

‎server/src/coverage/CoverageTool.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@ struct BuildRunCommand {
2020
class CoverageTool {
2121
protected:
2222
ProgressWriter const *progressWriter;
23+
const utbot::ProjectContext projectContext;
2324

24-
[[nodiscard]] std::string getTestFilter(UnitTest const &unitTest) const;
25+
[[nodiscard]] std::string getGTestFlags(const UnitTest &unitTest) const;
2526

2627
public:
27-
explicit CoverageTool(ProgressWriter const *progressWriter);
28+
CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter);
2829

2930
[[nodiscard]] virtual std::vector<BuildRunCommand>
3031
getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch, bool withCoverage) = 0;

‎server/src/coverage/GcovCoverageTool.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ using Coverage::FileCoverage;
2424

2525
GcovCoverageTool::GcovCoverageTool(utbot::ProjectContext projectContext,
2626
ProgressWriter const *progressWriter)
27-
: CoverageTool(progressWriter), projectContext(std::move(projectContext)) {
27+
: CoverageTool(std::move(projectContext), progressWriter) {
2828
}
2929

3030
std::vector<BuildRunCommand>
@@ -38,11 +38,11 @@ GcovCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
3838
auto makefile = Paths::getMakefilePathFromSourceFilePath(
3939
projectContext,
4040
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath));
41-
auto gtestFlags = getTestFilter(testToLaunch);
41+
auto gtestFlags = getGTestFlags(testToLaunch);
4242
auto buildCommand =
43-
MakefileUtils::makefileCommand(projectContext, makefile, "build", gtestFlags);
43+
MakefileUtils::MakefileCommand(projectContext, makefile, "build", gtestFlags);
4444
auto runCommand =
45-
MakefileUtils::makefileCommand(projectContext, makefile, "run", gtestFlags);
45+
MakefileUtils::MakefileCommand(projectContext, makefile, "run", gtestFlags);
4646
result.push_back({ testToLaunch, buildCommand, runCommand });
4747
});
4848
return result;

‎server/src/coverage/GcovCoverageTool.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ class GcovCoverageTool : public CoverageTool {
2323
void cleanCoverage() const override;
2424

2525
private:
26-
const utbot::ProjectContext projectContext;
27-
2826
std::vector<fs::path> getGcdaFiles() const;
2927
};
3028

‎server/src/coverage/LlvmCoverageTool.cpp

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "LlvmCoverageTool.h"
22

3+
#include <utility>
4+
35
#include "Coverage.h"
46
#include "Paths.h"
57
#include "TimeExecStatistics.h"
@@ -20,7 +22,7 @@ using Coverage::FileCoverage;
2022

2123
LlvmCoverageTool::LlvmCoverageTool(utbot::ProjectContext projectContext,
2224
ProgressWriter const *progressWriter)
23-
: CoverageTool(progressWriter), projectContext(projectContext) {
25+
: CoverageTool(std::move(projectContext), progressWriter) {
2426
}
2527

2628
std::vector<BuildRunCommand>
@@ -30,15 +32,15 @@ LlvmCoverageTool::getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch
3032
Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath);
3133
auto makefilePath = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
3234
auto testName = testToLaunch.testname;
33-
auto gtestFlags = getTestFilter(testToLaunch);
35+
auto gtestFlags = getGTestFlags(testToLaunch);
3436
std::vector<std::string> profileEnv;
3537
if (withCoverage) {
3638
auto profrawFilePath = Paths::getProfrawFilePath(projectContext, testName);
3739
profileEnv = { StringUtils::stringFormat("LLVM_PROFILE_FILE=%s", profrawFilePath) };
3840
}
39-
auto buildCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "build",
41+
auto buildCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "build",
4042
gtestFlags, profileEnv);
41-
auto runCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "run",
43+
auto runCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "run",
4244
gtestFlags, profileEnv);
4345
return BuildRunCommand{ testToLaunch, buildCommand, runCommand };
4446
});
@@ -73,7 +75,7 @@ LlvmCoverageTool::getCoverageCommands(const std::vector<UnitTest> &testsToLaunch
7375
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath);
7476
fs::path makefile =
7577
Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
76-
auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "bin");
78+
auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "bin");
7779
auto res = makefileCommand.run();
7880
if (res.status == 0) {
7981
if (res.output.empty()) {

‎server/src/coverage/LlvmCoverageTool.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ class LlvmCoverageTool : public CoverageTool {
1717
[[nodiscard]] nlohmann::json getTotals() const override;
1818
void cleanCoverage() const override;
1919
private:
20-
const utbot::ProjectContext projectContext;
2120
void countLineCoverage(Coverage::CoverageMap& coverageMap, const std::string& filename) const;
2221
void checkLineForPartial(Coverage::FileCoverage::SourceLine line, Coverage::FileCoverage& fileCoverage) const;
2322
};

‎server/src/coverage/TestRunner.cpp

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "Paths.h"
55
#include "TimeExecStatistics.h"
66
#include "utils/FileSystemUtils.h"
7+
#include "utils/JsonUtils.h"
78
#include "utils/StringUtils.h"
89

910
#include "loguru.h"
@@ -38,7 +39,7 @@ TestRunner::TestRunner(
3839

3940
std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
4041
const fs::path &testFilePath) {
41-
auto cmdGetAllTests = MakefileUtils::makefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
42+
auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"});
4243
auto [out, status, _] = cmdGetAllTests.run(projectContext.buildDir, false);
4344
if (status != 0) {
4445
auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.buildDir, true);
@@ -135,15 +136,18 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
135136
auto const &[unitTest, buildCommand, runCommand] =
136137
buildRunCommand;
137138
try {
138-
auto status = runTest(runCommand, testTimeout);
139-
testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
139+
auto status = runTest(buildRunCommand, testTimeout);
140+
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
140141
ExecUtils::throwIfCancelled();
141142
} catch (ExecutionProcessException const &e) {
142-
testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
143+
testsgen::TestResultObject testRes;
144+
testRes.set_testfilepath(unitTest.testFilePath);
145+
testRes.set_testname(unitTest.testname);
146+
testRes.set_status(testsgen::TEST_FAILED);
147+
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
143148
exceptions.emplace_back(e);
144149
}
145150
});
146-
147151
LOG_S(DEBUG) << "All run commands were executed";
148152
return Status::OK;
149153
}
@@ -166,7 +170,7 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs
166170
ExecUtils::throwIfCancelled();
167171
fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath);
168172
if (fs::exists(makefile)) {
169-
auto command = MakefileUtils::makefileCommand(projectContext, makefile, "build", "", {});
173+
auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "build", "", {});
170174
LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string();
171175
auto[out, status, logFilePath] = command.run(projectContext.buildDir, true);
172176
if (status != 0) {
@@ -187,23 +191,46 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
187191
return fail_count;
188192
}
189193

190-
testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
191-
auto res = command.run(projectContext.buildDir, true, true, testTimeout);
194+
testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
195+
auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
192196
GTestLogger::log(res.output);
197+
198+
auto executionTime = new google::protobuf::Duration;
199+
if (fs::exists(Paths::getGTestResultsJsonPath(projectContext))) {
200+
nlohmann::json gtestResultsJson = JsonUtils::getJsonFromFile(Paths::getGTestResultsJsonPath(projectContext));
201+
if (!google::protobuf::util::TimeUtil::FromString(gtestResultsJson["time"], executionTime)) {
202+
LOG_S(WARNING) << "Cannot parse duration of test execution";
203+
}
204+
} else {
205+
if (testTimeout.has_value()) {
206+
*executionTime = google::protobuf::util::TimeUtil::SecondsToDuration(testTimeout.value().count());
207+
} else {
208+
LOG_S(WARNING) << "Google test results are not generated, timeout not found";
209+
}
210+
}
211+
212+
testsgen::TestResultObject testRes;
213+
testRes.set_testfilepath(command.unitTest.testFilePath);
214+
testRes.set_testname(command.unitTest.testname);
215+
testRes.set_allocated_executiontime(executionTime);
193216
if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) {
194-
return testsgen::TEST_PASSED;
217+
testRes.set_status(testsgen::TEST_PASSED);
218+
return testRes;
195219
}
196220
if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) {
197-
return testsgen::TEST_FAILED;
221+
testRes.set_status(testsgen::TEST_FAILED);
222+
return testRes;
198223
}
199224
if (BaseForkTask::wasInterrupted(res.status)) {
200-
return testsgen::TEST_INTERRUPTED;
225+
testRes.set_status(testsgen::TEST_INTERRUPTED);
226+
return testRes;
201227
}
202-
return testsgen::TEST_DEATH;
228+
testRes.set_status(testsgen::TEST_DEATH);
229+
return testRes;
203230
}
204231

205-
const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
206-
return testStatusMap;
232+
const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
233+
return testResultMap;
207234
}
208235

209236
bool TestRunner::hasExceptions() const {

‎server/src/coverage/TestRunner.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class TestRunner {
2424

2525
std::unique_ptr<CoverageTool> coverageTool{};
2626
std::vector<UnitTest> testsToLaunch{};
27-
Coverage::TestStatusMap testStatusMap{};
27+
Coverage::TestResultMap testResultMap{};
2828

2929
std::vector<ExecutionProcessException> exceptions;
3030

@@ -48,7 +48,7 @@ class TestRunner {
4848

4949
std::vector<UnitTest> getTestsToLaunch();
5050

51-
const Coverage::TestStatusMap &getTestStatusMap() const;
51+
const Coverage::TestResultMap &getTestResultMap() const;
5252

5353
bool hasExceptions() const;
5454

@@ -68,8 +68,8 @@ class TestRunner {
6868
std::vector<UnitTest> getTestsFromMakefile(const fs::path &makefile,
6969
const fs::path &testFilePath);
7070

71-
testsgen::TestStatus runTest(const MakefileUtils::MakefileCommand &command,
72-
const std::optional<std::chrono::seconds> &testTimeout);
71+
testsgen::TestResultObject runTest(const BuildRunCommand &command,
72+
const std::optional<std::chrono::seconds> &testTimeout);
7373

7474
ServerCoverageAndResultsWriter writer{ nullptr };
7575

0 commit comments

Comments
 (0)