Skip to content

Commit 9265a71

Browse files
palimondopalimondo
committed
Improved coverage: ReportFormatter
Coverage at 87% according to coveragy.py Also fixed spelling errors in documentation.
1 parent d178b6e commit 9265a71

File tree

2 files changed

+155
-5
lines changed

2 files changed

+155
-5
lines changed

‎benchmark/scripts/compare_perf_tests.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@
2222

2323

2424
class PerformanceTestResult(object):
25-
"""PerformanceTestResult holds results from executing individual benchmark
26-
from Swift Benchmark Suite as reported by test driver (Benchmark_O,
27-
Benchmark_Onone, Benchmark_Ounchecked or Benchmark_Driver).
25+
"""PerformanceTestResult holds results from executing an individual
26+
benchmark from the Swift Benchmark Suite as reported by the test driver
27+
(Benchmark_O, Benchmark_Onone, Benchmark_Ounchecked or Benchmark_Driver).
2828
29-
It depends on the log format emmited by the test driver in the form:
29+
It depends on the log format emitted by the test driver in the form:
3030
#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),MAX_RSS(B)
3131
32-
The last column, MAX_RSS, is emmited only for runs instrumented by the
32+
The last column, MAX_RSS, is emitted only for runs instrumented by the
3333
Benchmark_Driver to measure rough memory use during the execution of the
3434
benchmark.
3535
"""

‎benchmark/scripts/test_compare_perf_tests.py

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import unittest
2020

2121
from compare_perf_tests import PerformanceTestResult
22+
from compare_perf_tests import ReportFormatter
2223
from compare_perf_tests import ResultComparison
2324
from compare_perf_tests import TestComparator
2425
from compare_perf_tests import parse_args
@@ -226,6 +227,149 @@ def names(tests):
226227
self.assertEquals(tc.decreased, [])
227228

228229

230+
class TestReportFormatter(OldAndNewLog):
231+
def setUp(self):
232+
super(TestReportFormatter, self).setUp()
233+
self.tc = TestComparator(self.old_log, self.new_log, 0.05)
234+
self.rf = ReportFormatter(self.tc, '', '', changes_only=False)
235+
self.markdown = self.rf.markdown()
236+
self.git = self.rf.git()
237+
self.html = self.rf.html()
238+
239+
def assert_report_contains(self, texts, report):
240+
# assert not isinstance(texts, str)
241+
if isinstance(texts, str):
242+
self.assertIn(texts, report)
243+
else:
244+
for text in texts:
245+
self.assertIn(text, report)
246+
247+
def assert_markdown_contains(self, texts):
248+
self.assert_report_contains(texts, self.markdown)
249+
250+
def assert_git_contains(self, texts):
251+
self.assert_report_contains(texts, self.git)
252+
253+
def assert_html_contains(self, texts):
254+
self.assert_report_contains(texts, self.html)
255+
256+
def test_justified_columns(self):
257+
"""Table columns are all formated with same width, defined by the
258+
longest value.
259+
"""
260+
print self.markdown
261+
self.assert_markdown_contains([
262+
'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445',
263+
'Array2D | 335831 | 335831 | +0.0% | 1.00x'])
264+
self.assert_git_contains([
265+
'AnyHashableWithAClass 247027 319065 259056 10250445',
266+
'Array2D 335831 335831 +0.0% 1.00x'])
267+
268+
def test_column_headers(self):
269+
"""Report contains table headers for ResultComparisons and changed
270+
PerformanceTestResults.
271+
"""
272+
print self.git
273+
self.assert_markdown_contains([
274+
'TEST | OLD | NEW | DELTA | SPEEDUP',
275+
'--- | --- | --- | --- | --- ',
276+
'TEST | MIN | MAX | MEAN | MAX_RSS'])
277+
self.assert_git_contains([
278+
'TEST OLD NEW DELTA SPEEDUP',
279+
'TEST MIN MAX MEAN MAX_RSS'])
280+
self.assert_html_contains([
281+
"""
282+
<th align='left'>OLD</th>
283+
<th align='left'>NEW</th>
284+
<th align='left'>DELTA</th>
285+
<th align='left'>SPEEDUP</th>""",
286+
"""
287+
<th align='left'>MIN</th>
288+
<th align='left'>MAX</th>
289+
<th align='left'>MEAN</th>
290+
<th align='left'>MAX_RSS</th>"""])
291+
292+
def test_emphasize_speedup(self):
293+
"""Emphasize speedup values for regressions and improvements"""
294+
# tests in No Changes don't have emphasized speedup
295+
self.assert_markdown_contains([
296+
'BitCount | 3 | 9 | +199.9% | **0.33x**',
297+
'ByteSwap | 4 | 0 | -100.0% | **4001.00x**',
298+
'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ',
299+
'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**'
300+
])
301+
self.assert_git_contains([
302+
'BitCount 3 9 +199.9% **0.33x**',
303+
'ByteSwap 4 0 -100.0% **4001.00x**',
304+
'AngryPhonebook 10458 10458 +0.0% 1.00x',
305+
'ArrayAppend 23641 20000 -15.4% **1.18x (?)**'
306+
])
307+
print self.html
308+
self.assert_html_contains([
309+
"""
310+
<tr>
311+
<td align='left'>BitCount</td>
312+
<td align='left'>3</td>
313+
<td align='left'>9</td>
314+
<td align='left'>+199.9%</td>
315+
<td align='left'><font color='red'>0.33x</font></td>
316+
</tr>""",
317+
"""
318+
<tr>
319+
<td align='left'>ByteSwap</td>
320+
<td align='left'>4</td>
321+
<td align='left'>0</td>
322+
<td align='left'>-100.0%</td>
323+
<td align='left'><font color='green'>4001.00x</font></td>
324+
</tr>""",
325+
"""
326+
<tr>
327+
<td align='left'>AngryPhonebook</td>
328+
<td align='left'>10458</td>
329+
<td align='left'>10458</td>
330+
<td align='left'>+0.0%</td>
331+
<td align='left'><font color='black'>1.00x</font></td>
332+
</tr>"""
333+
])
334+
335+
def test_sections(self):
336+
"""Report is divided into sections with summaries."""
337+
self.assert_markdown_contains([
338+
"""<details open>
339+
<summary>Regression (1)</summary>""",
340+
"""<details >
341+
<summary>Improvement (2)</summary>""",
342+
"""<details >
343+
<summary>No Changes (2)</summary>""",
344+
"""<details open>
345+
<summary>Added (1)</summary>""",
346+
"""<details open>
347+
<summary>Removed (1)</summary>"""])
348+
self.assert_git_contains([
349+
'Regression (1): \n',
350+
'Improvement (2): \n',
351+
'No Changes (2): \n',
352+
'Added (1): \n',
353+
'Removed (1): \n'])
354+
self.assert_html_contains([
355+
"<th align='left'>Regression (1)</th>",
356+
"<th align='left'>Improvement (2)</th>",
357+
"<th align='left'>No Changes (2)</th>",
358+
"<th align='left'>Added (1)</th>",
359+
"<th align='left'>Removed (1)</th>"])
360+
361+
def test_report_only_changes(self):
362+
"""Leave out tests without significant change."""
363+
rf = ReportFormatter(self.tc, '', '', changes_only=True)
364+
markdown, git, html = rf.markdown(), rf.git(), rf.html()
365+
self.assertNotIn('No Changes', markdown)
366+
self.assertNotIn('AngryPhonebook', markdown)
367+
self.assertNotIn('No Changes', git)
368+
self.assertNotIn('AngryPhonebook', git)
369+
self.assertNotIn('No Changes', html)
370+
self.assertNotIn('AngryPhonebook', html)
371+
372+
229373
class Test_parse_args(unittest.TestCase):
230374
required = ['--old-file', 'old.log', '--new-file', 'new.log']
231375

@@ -260,6 +404,12 @@ def test_delta_threshold_argument(self):
260404
self.assertRaises(SystemExit, parse_args,
261405
self.required + ['--delta-threshold', '2,2'])
262406

407+
def test_output_argument(self):
408+
self.assertEquals(parse_args(self.required).output, None)
409+
self.assertEquals(parse_args(self.required +
410+
['--output', 'report.log']).output,
411+
'report.log')
412+
263413
def test_changes_only_argument(self):
264414
self.assertFalse(parse_args(self.required).changes_only)
265415
self.assertTrue(parse_args(self.required +

0 commit comments

Comments
 (0)