Skip to content

Commit bb7af90

Browse files
Lana243Lana243
committed
add statistics report for coverage and results
1 parent 6a86fef commit bb7af90

17 files changed

+197
-63
lines changed

‎server/proto/testgen.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ message TestResultObject {
188188
string testname = 2;
189189
TestStatus status = 3;
190190
string output = 4;
191+
uint32 executionTimeMs = 5;
191192
}
192193

193194
message CoverageAndResultsResponse {

‎server/src/coverage/Coverage.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44

55
#include "Coverage.h"
66

7-
int Coverage::TestStatusMap::getNumberOfTests() {
7+
int Coverage::TestResultMap::getNumberOfTests() {
88
int cnt = 0;
9-
for (auto const &[fileName, testsStatus] : *this) {
10-
cnt += testsStatus.size();
9+
for (auto const &[fileName, testsResult] : *this) {
10+
cnt += testsResult.size();
1111
}
1212
return cnt;
1313
}

‎server/src/coverage/Coverage.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,11 @@ namespace Coverage {
4141
class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
4242
};
4343

44-
class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
44+
using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
45+
class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
4546
public:
4647
int getNumberOfTests();
4748
};
48-
4949
}
5050

5151
#endif //UNITTESTBOT_COVERAGE_H

‎server/src/coverage/CoverageAndResultsGenerator.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ void CoverageAndResultsGenerator::showErrors() const {
6969
errorMessage = message;
7070
}
7171

72-
coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
72+
coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
7373
}
7474

7575
Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -87,7 +87,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
8787
}
8888
std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
8989
CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
90-
return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
90+
return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
9191
testsgen::TEST_INTERRUPTED;
9292
}));
9393
if (coverageCommands.empty()) {

‎server/src/coverage/TestRunner.cpp

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "utils/StringUtils.h"
1212

1313
#include "loguru.h"
14+
#include <regex>
1415

1516
using grpc::ServerWriter;
1617
using grpc::Status;
@@ -139,11 +140,15 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
139140
auto const &[unitTest, buildCommand, runCommand] =
140141
buildRunCommand;
141142
try {
142-
auto status = runTest(runCommand, testTimeout);
143-
testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
143+
auto status = runTest(buildRunCommand, testTimeout);
144+
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
144145
ExecUtils::throwIfCancelled();
145146
} catch (ExecutionProcessException const &e) {
146-
testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
147+
testsgen::TestResultObject testRes;
148+
testRes.set_testfilepath(unitTest.testFilePath);
149+
testRes.set_testname(unitTest.testname);
150+
testRes.set_status(testsgen::TEST_FAILED);
151+
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
147152
exceptions.emplace_back(e);
148153
}
149154
});
@@ -191,23 +196,42 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
191196
return fail_count;
192197
}
193198

194-
testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
195-
auto res = command.run(projectContext.buildDir, true, true, testTimeout);
199+
testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
200+
auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
196201
GTestLogger::log(res.output);
202+
203+
std::smatch match;
204+
std::regex expr(R"(\[==========\] 1 test from 1 test suite ran. \(\d+ ms total\))");
205+
std::regex_search(res.output, match, expr);
206+
uint32_t executionTimeMs = 0;
207+
if (!match.empty()) {
208+
std::string strMatch = res.output.substr(match.position(0), match.length(0));
209+
std::string strExecutionTime = StringUtils::splitByWhitespaces(strMatch)[8];
210+
executionTimeMs = std::stoul(strExecutionTime.substr(1));
211+
}
212+
213+
testsgen::TestResultObject testRes;
214+
testRes.set_testfilepath(command.unitTest.testFilePath);
215+
testRes.set_testname(command.unitTest.testname);
216+
testRes.set_executiontimems(executionTimeMs);
197217
if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) {
198-
return testsgen::TEST_PASSED;
218+
testRes.set_status(testsgen::TEST_PASSED);
219+
return testRes;
199220
}
200221
if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) {
201-
return testsgen::TEST_FAILED;
222+
testRes.set_status(testsgen::TEST_FAILED);
223+
return testRes;
202224
}
203225
if (BaseForkTask::wasInterrupted(res.status)) {
204-
return testsgen::TEST_INTERRUPTED;
226+
testRes.set_status(testsgen::TEST_INTERRUPTED);
227+
return testRes;
205228
}
206-
return testsgen::TEST_DEATH;
229+
testRes.set_status(testsgen::TEST_DEATH);
230+
return testRes;
207231
}
208232

209-
const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
210-
return testStatusMap;
233+
const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
234+
return testResultMap;
211235
}
212236

213237
bool TestRunner::hasExceptions() const {

‎server/src/coverage/TestRunner.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ class TestRunner {
2828

2929
std::unique_ptr<CoverageTool> coverageTool{};
3030
std::vector<UnitTest> testsToLaunch{};
31-
Coverage::TestStatusMap testStatusMap{};
31+
Coverage::TestResultMap testResultMap{};
3232

3333
std::vector<ExecutionProcessException> exceptions;
3434

@@ -52,7 +52,7 @@ class TestRunner {
5252

5353
std::vector<UnitTest> getTestsToLaunch();
5454

55-
const Coverage::TestStatusMap &getTestStatusMap() const;
55+
const Coverage::TestResultMap &getTestResultMap() const;
5656

5757
bool hasExceptions() const;
5858

@@ -72,8 +72,8 @@ class TestRunner {
7272
std::vector<UnitTest> getTestsFromMakefile(const fs::path &makefile,
7373
const fs::path &testFilePath);
7474

75-
testsgen::TestStatus runTest(const MakefileUtils::MakefileCommand &command,
76-
const std::optional<std::chrono::seconds> &testTimeout);
75+
testsgen::TestResultObject runTest(const BuildRunCommand &command,
76+
const std::optional<std::chrono::seconds> &testTimeout);
7777

7878
ServerCoverageAndResultsWriter writer{ nullptr };
7979

‎server/src/printers/CoverageAndResultsStatisticsPrinter.cpp

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#include <utils/FileSystemUtils.h>
2+
3+
#include "CoverageAndResultsStatisticsPrinter.h"
4+
#include "utils/StringUtils.h"
5+
#include "utils/CollectionUtils.h"
6+
#include "Paths.h"
7+
8+
void printer::CoverageAndResultsStatisticsPrinter::write(const utbot::ProjectContext &projectContext,
9+
const Coverage::TestResultMap &testsResultMap,
10+
const Coverage::CoverageMap &coverageMap) {
11+
for (auto const &[testPath, testsResult]: testsResultMap) {
12+
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testPath);
13+
Coverage::FileCoverage fileCoverage = CollectionUtils::getOrDefault(coverageMap,
14+
sourcePath,
15+
Coverage::FileCoverage());
16+
insert({sourcePath, FileCoverageAndResultsStatistics(testsResult, fileCoverage)});
17+
}
18+
std::vector<std::string> metricNames = {
19+
"filename", "executionTime (ms)",
20+
"totalTestsNumber", "passedTestsNumber", "failedTestsNumber", "deathTestsNumber", "interruptedTestsNumber",
21+
"totalLinesNumber", "coveredLinesNumber", "lineCoverageRatio (%)"
22+
};
23+
std::string header = StringUtils::joinWith(metricNames, ",");
24+
std::stringstream ss;
25+
ss << header << '\n';
26+
for (auto const &[sourcePath, statistics]: *this) {
27+
ss << fs::relative(sourcePath, projectContext.projectPath) << ",";
28+
ss << statistics.totalExecutionTimeMs << ',';
29+
ss << statistics.totalTestsNum << ",";
30+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_PASSED, 0u)
31+
<< ',';
32+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_FAILED, 0u)
33+
<< ',';
34+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_DEATH, 0u) << ',';
35+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_INTERRUPTED, 0u)
36+
<< ',';
37+
38+
uint32_t coveredLinesNum =
39+
statistics.coverage.fullCoverageLines.size() + statistics.coverage.partialCoverageLines.size();
40+
uint32_t totalLinesNum = coveredLinesNum + statistics.coverage.noCoverageLines.size();
41+
double lineCoverageRatio = 0;
42+
if (totalLinesNum != 0) {
43+
lineCoverageRatio = 100.0 * coveredLinesNum / totalLinesNum;
44+
}
45+
ss << totalLinesNum << ',' << coveredLinesNum << ',';
46+
ss << std::fixed << lineCoverageRatio << '\n';
47+
}
48+
fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + "-stats.csv");
49+
FileSystemUtils::writeToFile(resultsFilePath, ss.str());
50+
LOG_S(INFO) << StringUtils::stringFormat("See statistics info here: %s", resultsFilePath);
51+
}
52+
53+
printer::FileCoverageAndResultsStatistics::FileCoverageAndResultsStatistics(
54+
const Coverage::FileTestsResult &testsResult,
55+
Coverage::FileCoverage fileCoverage) {
56+
coverage = std::move(fileCoverage);
57+
totalTestsNum = 0;
58+
totalExecutionTimeMs = 0;
59+
for (const auto &[_, testResult]: testsResult) {
60+
totalTestsNum++;
61+
testsWithStatusNum[testResult.status()]++;
62+
totalExecutionTimeMs += testResult.executiontimems();
63+
}
64+
}

‎server/src/printers/CoverageAndResultsStatisticsPrinter.h

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#ifndef UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
2+
#define UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
3+
4+
#include <chrono>
5+
#include <unordered_map>
6+
#include <utility>
7+
#include <protobuf/testgen.grpc.pb.h>
8+
#include <ProjectContext.h>
9+
#include "coverage/Coverage.h"
10+
#include "loguru.h"
11+
12+
namespace printer {
13+
class FileCoverageAndResultsStatistics {
14+
public:
15+
FileCoverageAndResultsStatistics(const Coverage::FileTestsResult &testsResult, Coverage::FileCoverage coverage);
16+
17+
// Time statistics
18+
uint32_t totalExecutionTimeMs;
19+
20+
// Test runs statistics
21+
uint32_t totalTestsNum;
22+
std::unordered_map<testsgen::TestStatus, uint32_t> testsWithStatusNum;
23+
24+
// Coverage
25+
Coverage::FileCoverage coverage;
26+
};
27+
28+
class CoverageAndResultsStatisticsPrinter : CollectionUtils::MapFileTo<FileCoverageAndResultsStatistics> {
29+
public:
30+
explicit CoverageAndResultsStatisticsPrinter(fs::path resultsDirectory) : resultsDirectory(
31+
std::move(resultsDirectory)) {};
32+
void write(const utbot::ProjectContext &, const Coverage::TestResultMap &, const Coverage::CoverageMap &);
33+
34+
private:
35+
fs::path resultsDirectory;
36+
};
37+
}
38+
39+
#endif //UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H

‎server/src/streams/coverage/CLICoverageAndResultsWriter.cpp

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#include "utils/FileSystemUtils.h"
99
#include "utils/TimeUtils.h"
10+
#include <printers/CoverageAndResultsStatisticsPrinter.h>
1011

1112
#include "loguru.h"
1213

@@ -27,17 +28,18 @@ std::string statusToString(testsgen::TestStatus status) {
2728
return it == description.end() ? "UNKNOWN" : it->second;
2829
}
2930

30-
void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
31+
void CLICoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
32+
const Coverage::TestResultMap &testsResultMap,
3133
const Coverage::CoverageMap &coverageMap,
3234
const nlohmann::json &totals,
3335
std::optional<std::string> errorMessage) {
3436
std::stringstream ss;
3537

3638
ss << "Test results summary." << std::endl;
37-
for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
39+
for (const auto &[filepath, fileTestsResult] : testsResultMap) {
3840
ss << "==== Tests in " << filepath << std::endl;
39-
for (const auto &[testName, status] : fileTestsStatus) {
40-
ss << "======== " << testName << " -> " << statusToString(status) << std::endl;
41+
for (const auto &[testName, result] : fileTestsResult) {
42+
ss << "======== " << testName << " -> " << statusToString(result.status()) << std::endl;
4143
}
4244
}
4345

@@ -60,4 +62,7 @@ void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &t
6062
fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + ".log");
6163
FileSystemUtils::writeToFile(resultsFilePath, ss.str());
6264
LOG_S(INFO) << ss.str();
65+
printer::CoverageAndResultsStatisticsPrinter statsPrinter = printer::CoverageAndResultsStatisticsPrinter(
66+
resultsDirectory);
67+
statsPrinter.write(projectContext, testsResultMap, coverageMap);
6368
}

‎server/src/streams/coverage/CLICoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ class CLICoverageAndResultsWriter : public CoverageAndResultsWriter {
1212
public:
1313
explicit CLICoverageAndResultsWriter(const fs::path &resultsDirectory);
1414

15-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
15+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
16+
const Coverage::TestResultMap &testsResultMap,
1617
const Coverage::CoverageMap &coverageMap,
1718
const nlohmann::json &totals,
1819
std::optional<std::string> errorMessage) override;

‎server/src/streams/coverage/CoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ class CoverageAndResultsWriter : public utbot::ServerWriter<testsgen::CoverageAn
1919

2020
explicit CoverageAndResultsWriter(grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
2121

22-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
22+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
23+
const Coverage::TestResultMap &testsResultMap,
2324
const Coverage::CoverageMap &coverageMap,
2425
const nlohmann::json &totals,
2526
std::optional<std::string> errorMessage) = 0;

‎server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ ServerCoverageAndResultsWriter::ServerCoverageAndResultsWriter(
1414
: CoverageAndResultsWriter(writer) {
1515
}
1616

17-
void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
17+
void ServerCoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
18+
const Coverage::TestResultMap &testsResultMap,
1819
const Coverage::CoverageMap &coverageMap,
19-
const nlohmann::json &totals,
20+
const nlohmann::json &totals,
2021
std::optional<std::string> errorMessage) {
2122
if (!hasStream()) {
2223
return;
@@ -25,13 +26,10 @@ void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap
2526

2627
testsgen::CoverageAndResultsResponse response;
2728

28-
for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
29-
for (const auto &[testname, status] : fileTestsStatus) {
29+
for (const auto &[filepath, fileTestsResult] : testsResultMap) {
30+
for (const auto &[testname, result] : fileTestsResult) {
3031
auto testResultsGrpc = response.add_testrunresults();
31-
testResultsGrpc->set_testfilepath(filepath);
32-
testResultsGrpc->set_testname(testname);
33-
testResultsGrpc->set_status(status);
34-
testResultsGrpc->set_output("");
32+
*testResultsGrpc = result;
3533
}
3634
}
3735

‎server/src/streams/coverage/ServerCoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ class ServerCoverageAndResultsWriter : public CoverageAndResultsWriter {
2020
explicit ServerCoverageAndResultsWriter(
2121
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
2222

23-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
23+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
24+
const Coverage::TestResultMap &testResultMap,
2425
const Coverage::CoverageMap &coverageMap,
2526
const nlohmann::json &totals,
2627
std::optional<std::string> errorMessage) override;

0 commit comments

Comments
 (0)