Skip to content

Commit af462b0

Browse files
Lana243Lana243
committed
add statistics report for coverage and results
1 parent 58cd8ce commit af462b0

17 files changed

+197
-63
lines changed

‎server/proto/testgen.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ message TestResultObject {
188188
string testname = 2;
189189
TestStatus status = 3;
190190
string output = 4;
191+
uint32 executionTimeMs = 5;
191192
}
192193

193194
message CoverageAndResultsResponse {

‎server/src/coverage/Coverage.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#include "Coverage.h"
22

3-
int Coverage::TestStatusMap::getNumberOfTests() {
3+
int Coverage::TestResultMap::getNumberOfTests() {
44
int cnt = 0;
5-
for (auto const &[fileName, testsStatus] : *this) {
6-
cnt += testsStatus.size();
5+
for (auto const &[fileName, testsResult] : *this) {
6+
cnt += testsResult.size();
77
}
88
return cnt;
99
}

‎server/src/coverage/Coverage.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@ namespace Coverage {
3737
class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
3838
};
3939

40-
class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
40+
using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
41+
class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
4142
public:
4243
int getNumberOfTests();
4344
};
44-
4545
}
4646

4747
#endif //UNITTESTBOT_COVERAGE_H

‎server/src/coverage/CoverageAndResultsGenerator.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ void CoverageAndResultsGenerator::showErrors() const {
6565
errorMessage = message;
6666
}
6767

68-
coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
68+
coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
6969
}
7070

7171
Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -83,7 +83,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
8383
}
8484
std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
8585
CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
86-
return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
86+
return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
8787
testsgen::TEST_INTERRUPTED;
8888
}));
8989
if (coverageCommands.empty()) {

‎server/src/coverage/TestRunner.cpp

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "utils/StringUtils.h"
88

99
#include "loguru.h"
10+
#include <regex>
1011

1112
using grpc::ServerWriter;
1213
using grpc::Status;
@@ -135,11 +136,15 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
135136
auto const &[unitTest, buildCommand, runCommand] =
136137
buildRunCommand;
137138
try {
138-
auto status = runTest(runCommand, testTimeout);
139-
testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
139+
auto status = runTest(buildRunCommand, testTimeout);
140+
testResultMap[unitTest.testFilePath][unitTest.testname] = status;
140141
ExecUtils::throwIfCancelled();
141142
} catch (ExecutionProcessException const &e) {
142-
testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
143+
testsgen::TestResultObject testRes;
144+
testRes.set_testfilepath(unitTest.testFilePath);
145+
testRes.set_testname(unitTest.testname);
146+
testRes.set_status(testsgen::TEST_FAILED);
147+
testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
143148
exceptions.emplace_back(e);
144149
}
145150
});
@@ -187,23 +192,42 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
187192
return fail_count;
188193
}
189194

190-
testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
191-
auto res = command.run(projectContext.buildDir, true, true, testTimeout);
195+
testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
196+
auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
192197
GTestLogger::log(res.output);
198+
199+
std::smatch match;
200+
std::regex expr(R"(\[==========\] 1 test from 1 test suite ran. \(\d+ ms total\))");
201+
std::regex_search(res.output, match, expr);
202+
uint32_t executionTimeMs = 0;
203+
if (!match.empty()) {
204+
std::string strMatch = res.output.substr(match.position(0), match.length(0));
205+
std::string strExecutionTime = StringUtils::splitByWhitespaces(strMatch)[8];
206+
executionTimeMs = std::stoul(strExecutionTime.substr(1));
207+
}
208+
209+
testsgen::TestResultObject testRes;
210+
testRes.set_testfilepath(command.unitTest.testFilePath);
211+
testRes.set_testname(command.unitTest.testname);
212+
testRes.set_executiontimems(executionTimeMs);
193213
if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) {
194-
return testsgen::TEST_PASSED;
214+
testRes.set_status(testsgen::TEST_PASSED);
215+
return testRes;
195216
}
196217
if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) {
197-
return testsgen::TEST_FAILED;
218+
testRes.set_status(testsgen::TEST_FAILED);
219+
return testRes;
198220
}
199221
if (BaseForkTask::wasInterrupted(res.status)) {
200-
return testsgen::TEST_INTERRUPTED;
222+
testRes.set_status(testsgen::TEST_INTERRUPTED);
223+
return testRes;
201224
}
202-
return testsgen::TEST_DEATH;
225+
testRes.set_status(testsgen::TEST_DEATH);
226+
return testRes;
203227
}
204228

205-
const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
206-
return testStatusMap;
229+
const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
230+
return testResultMap;
207231
}
208232

209233
bool TestRunner::hasExceptions() const {

‎server/src/coverage/TestRunner.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class TestRunner {
2424

2525
std::unique_ptr<CoverageTool> coverageTool{};
2626
std::vector<UnitTest> testsToLaunch{};
27-
Coverage::TestStatusMap testStatusMap{};
27+
Coverage::TestResultMap testResultMap{};
2828

2929
std::vector<ExecutionProcessException> exceptions;
3030

@@ -48,7 +48,7 @@ class TestRunner {
4848

4949
std::vector<UnitTest> getTestsToLaunch();
5050

51-
const Coverage::TestStatusMap &getTestStatusMap() const;
51+
const Coverage::TestResultMap &getTestResultMap() const;
5252

5353
bool hasExceptions() const;
5454

@@ -68,8 +68,8 @@ class TestRunner {
6868
std::vector<UnitTest> getTestsFromMakefile(const fs::path &makefile,
6969
const fs::path &testFilePath);
7070

71-
testsgen::TestStatus runTest(const MakefileUtils::MakefileCommand &command,
72-
const std::optional<std::chrono::seconds> &testTimeout);
71+
testsgen::TestResultObject runTest(const BuildRunCommand &command,
72+
const std::optional<std::chrono::seconds> &testTimeout);
7373

7474
ServerCoverageAndResultsWriter writer{ nullptr };
7575

‎server/src/printers/CoverageAndResultsStatisticsPrinter.cpp

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#include <utils/FileSystemUtils.h>
2+
3+
#include "CoverageAndResultsStatisticsPrinter.h"
4+
#include "utils/StringUtils.h"
5+
#include "utils/CollectionUtils.h"
6+
#include "Paths.h"
7+
8+
void printer::CoverageAndResultsStatisticsPrinter::write(const utbot::ProjectContext &projectContext,
9+
const Coverage::TestResultMap &testsResultMap,
10+
const Coverage::CoverageMap &coverageMap) {
11+
for (auto const &[testPath, testsResult]: testsResultMap) {
12+
fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testPath);
13+
Coverage::FileCoverage fileCoverage = CollectionUtils::getOrDefault(coverageMap,
14+
sourcePath,
15+
Coverage::FileCoverage());
16+
insert({sourcePath, FileCoverageAndResultsStatistics(testsResult, fileCoverage)});
17+
}
18+
std::vector<std::string> metricNames = {
19+
"filename", "executionTime (ms)",
20+
"totalTestsNumber", "passedTestsNumber", "failedTestsNumber", "deathTestsNumber", "interruptedTestsNumber",
21+
"totalLinesNumber", "coveredLinesNumber", "lineCoverageRatio (%)"
22+
};
23+
std::string header = StringUtils::joinWith(metricNames, ",");
24+
std::stringstream ss;
25+
ss << header << '\n';
26+
for (auto const &[sourcePath, statistics]: *this) {
27+
ss << fs::relative(sourcePath, projectContext.projectPath) << ",";
28+
ss << statistics.totalExecutionTimeMs << ',';
29+
ss << statistics.totalTestsNum << ",";
30+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_PASSED, 0u)
31+
<< ',';
32+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_FAILED, 0u)
33+
<< ',';
34+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_DEATH, 0u) << ',';
35+
ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_INTERRUPTED, 0u)
36+
<< ',';
37+
38+
uint32_t coveredLinesNum =
39+
statistics.coverage.fullCoverageLines.size() + statistics.coverage.partialCoverageLines.size();
40+
uint32_t totalLinesNum = coveredLinesNum + statistics.coverage.noCoverageLines.size();
41+
double lineCoverageRatio = 0;
42+
if (totalLinesNum != 0) {
43+
lineCoverageRatio = 100.0 * coveredLinesNum / totalLinesNum;
44+
}
45+
ss << totalLinesNum << ',' << coveredLinesNum << ',';
46+
ss << std::fixed << lineCoverageRatio << '\n';
47+
}
48+
fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + "-stats.csv");
49+
FileSystemUtils::writeToFile(resultsFilePath, ss.str());
50+
LOG_S(INFO) << StringUtils::stringFormat("See statistics info here: %s", resultsFilePath);
51+
}
52+
53+
printer::FileCoverageAndResultsStatistics::FileCoverageAndResultsStatistics(
54+
const Coverage::FileTestsResult &testsResult,
55+
Coverage::FileCoverage fileCoverage) {
56+
coverage = std::move(fileCoverage);
57+
totalTestsNum = 0;
58+
totalExecutionTimeMs = 0;
59+
for (const auto &[_, testResult]: testsResult) {
60+
totalTestsNum++;
61+
testsWithStatusNum[testResult.status()]++;
62+
totalExecutionTimeMs += testResult.executiontimems();
63+
}
64+
}

‎server/src/printers/CoverageAndResultsStatisticsPrinter.h

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#ifndef UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
2+
#define UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
3+
4+
#include <chrono>
5+
#include <unordered_map>
6+
#include <utility>
7+
#include <protobuf/testgen.grpc.pb.h>
8+
#include <ProjectContext.h>
9+
#include "coverage/Coverage.h"
10+
#include "loguru.h"
11+
12+
namespace printer {
13+
class FileCoverageAndResultsStatistics {
14+
public:
15+
FileCoverageAndResultsStatistics(const Coverage::FileTestsResult &testsResult, Coverage::FileCoverage coverage);
16+
17+
// Time statistics
18+
uint32_t totalExecutionTimeMs;
19+
20+
// Test runs statistics
21+
uint32_t totalTestsNum;
22+
std::unordered_map<testsgen::TestStatus, uint32_t> testsWithStatusNum;
23+
24+
// Coverage
25+
Coverage::FileCoverage coverage;
26+
};
27+
28+
class CoverageAndResultsStatisticsPrinter : CollectionUtils::MapFileTo<FileCoverageAndResultsStatistics> {
29+
public:
30+
explicit CoverageAndResultsStatisticsPrinter(fs::path resultsDirectory) : resultsDirectory(
31+
std::move(resultsDirectory)) {};
32+
void write(const utbot::ProjectContext &, const Coverage::TestResultMap &, const Coverage::CoverageMap &);
33+
34+
private:
35+
fs::path resultsDirectory;
36+
};
37+
}
38+
39+
#endif //UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H

‎server/src/streams/coverage/CLICoverageAndResultsWriter.cpp

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "utils/FileSystemUtils.h"
44
#include "utils/TimeUtils.h"
5+
#include <printers/CoverageAndResultsStatisticsPrinter.h>
56

67
#include "loguru.h"
78

@@ -22,17 +23,18 @@ std::string statusToString(testsgen::TestStatus status) {
2223
return it == description.end() ? "UNKNOWN" : it->second;
2324
}
2425

25-
void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
26+
void CLICoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
27+
const Coverage::TestResultMap &testsResultMap,
2628
const Coverage::CoverageMap &coverageMap,
2729
const nlohmann::json &totals,
2830
std::optional<std::string> errorMessage) {
2931
std::stringstream ss;
3032

3133
ss << "Test results summary." << std::endl;
32-
for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
34+
for (const auto &[filepath, fileTestsResult] : testsResultMap) {
3335
ss << "==== Tests in " << filepath << std::endl;
34-
for (const auto &[testName, status] : fileTestsStatus) {
35-
ss << "======== " << testName << " -> " << statusToString(status) << std::endl;
36+
for (const auto &[testName, result] : fileTestsResult) {
37+
ss << "======== " << testName << " -> " << statusToString(result.status()) << std::endl;
3638
}
3739
}
3840

@@ -55,4 +57,7 @@ void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &t
5557
fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + ".log");
5658
FileSystemUtils::writeToFile(resultsFilePath, ss.str());
5759
LOG_S(INFO) << ss.str();
60+
printer::CoverageAndResultsStatisticsPrinter statsPrinter = printer::CoverageAndResultsStatisticsPrinter(
61+
resultsDirectory);
62+
statsPrinter.write(projectContext, testsResultMap, coverageMap);
5863
}

‎server/src/streams/coverage/CLICoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ class CLICoverageAndResultsWriter : public CoverageAndResultsWriter {
77
public:
88
explicit CLICoverageAndResultsWriter(const fs::path &resultsDirectory);
99

10-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
10+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
11+
const Coverage::TestResultMap &testsResultMap,
1112
const Coverage::CoverageMap &coverageMap,
1213
const nlohmann::json &totals,
1314
std::optional<std::string> errorMessage) override;

‎server/src/streams/coverage/CoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ class CoverageAndResultsWriter : public utbot::ServerWriter<testsgen::CoverageAn
1515

1616
explicit CoverageAndResultsWriter(grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
1717

18-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
18+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
19+
const Coverage::TestResultMap &testsResultMap,
1920
const Coverage::CoverageMap &coverageMap,
2021
const nlohmann::json &totals,
2122
std::optional<std::string> errorMessage) = 0;

‎server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ ServerCoverageAndResultsWriter::ServerCoverageAndResultsWriter(
99
: CoverageAndResultsWriter(writer) {
1010
}
1111

12-
void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
12+
void ServerCoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
13+
const Coverage::TestResultMap &testsResultMap,
1314
const Coverage::CoverageMap &coverageMap,
14-
const nlohmann::json &totals,
15+
const nlohmann::json &totals,
1516
std::optional<std::string> errorMessage) {
1617
if (!hasStream()) {
1718
return;
@@ -20,13 +21,10 @@ void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap
2021

2122
testsgen::CoverageAndResultsResponse response;
2223

23-
for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
24-
for (const auto &[testname, status] : fileTestsStatus) {
24+
for (const auto &[filepath, fileTestsResult] : testsResultMap) {
25+
for (const auto &[testname, result] : fileTestsResult) {
2526
auto testResultsGrpc = response.add_testrunresults();
26-
testResultsGrpc->set_testfilepath(filepath);
27-
testResultsGrpc->set_testname(testname);
28-
testResultsGrpc->set_status(status);
29-
testResultsGrpc->set_output("");
27+
*testResultsGrpc = result;
3028
}
3129
}
3230

‎server/src/streams/coverage/ServerCoverageAndResultsWriter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ class ServerCoverageAndResultsWriter : public CoverageAndResultsWriter {
1515
explicit ServerCoverageAndResultsWriter(
1616
grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
1717

18-
virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
18+
virtual void writeResponse(const utbot::ProjectContext &projectContext,
19+
const Coverage::TestResultMap &testResultMap,
1920
const Coverage::CoverageMap &coverageMap,
2021
const nlohmann::json &totals,
2122
std::optional<std::string> errorMessage) override;

0 commit comments

Comments
 (0)