The --cov-context option for setting context per test

Author: nedbat

AUTHORS.rst

@@ -30,3 +30,4 @@ Authors
 * Семён Марьясин - https://github.com/MarSoft
 * Alexander Shadchin - https://github.com/shadchin
 * Thomas Grainger - https://graingert.co.uk
+* Ned Batchelder - https://nedbatchelder.com

CHANGELOG.rst

@@ -4,6 +4,8 @@ Changelog
 2.7.2.dev0 (unreleased)
 -----------------------
 
+* Added --cov-context option for setting the coverage.py dynamic context for
+  each test.
 * Match pytest-xdist master/worker terminology.
   Contributed in `#321 <https://github.com/pytest-dev/pytest-cov/pull/321>`_
 

docs/config.rst

@@ -63,3 +63,4 @@ The complete list of command line options is:
   --cov-append          Do not delete coverage but append to current. Default:
                         False
   --cov-branch          Enable branch coverage.
+  --cov-context         Choose the method for setting the dynamic context.

docs/contexts.rst

@@ -0,0 +1,18 @@
+========
+Contexts
+========
+
+Coverage.py 5.0 can record separate coverage data for different contexts during
+one run of a test suite.  Pytest-cov can use this feature to record coverage
+data for each test individually, with the ``--cov-context=test`` option.
+
+The context name recorded in the coverage.py database is the pytest test id,
+and the phase of execution, one of "setup", "run", or "teardown".  These two
+are separated with a pipe symbol.  You might see contexts like::
+
+    test_functions.py::test_addition|run
+    test_fancy.py::test_parametrized[1-101]|setup
+    test_oldschool.py::RegressionTests::test_error|run
+
+Note that parameterized tests include the values of the parameters in the test
+id, and each set of parameter values is recorded as a separate test.

docs/index.rst

@@ -12,6 +12,7 @@ Contents:
     debuggers
     xdist
     subprocess-support
+    contexts
     tox
     plugins
     markers-fixtures

src/pytest_cov/plugin.py

@@ -1,9 +1,9 @@
 """Coverage plugin for pytest."""
 import argparse
+import coverage
 import os
-import warnings
-
 import pytest
+import warnings
 from coverage.misc import CoverageException
 
 from . import compat
@@ -48,6 +48,14 @@ def validate_fail_under(num_str):
         return float(num_str)
 
 
+def validate_context(arg):
+    if coverage.version_info <= (5, 0):
+        raise argparse.ArgumentTypeError('Contexts are only supported with coverage.py >= 5.x')
+    if arg != "test":
+        raise argparse.ArgumentTypeError('--cov-context=test is the only supported value')
+    return arg
+
+
 class StoreReport(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         report_type, file = values
@@ -88,6 +96,9 @@ def pytest_addoption(parser):
                          'Default: False')
     group.addoption('--cov-branch', action='store_true', default=None,
                     help='Enable branch coverage.')
+    group.addoption('--cov-context', action='store', metavar='CONTEXT',
+                    type=validate_context,
+                    help='Dynamic contexts to use. "test" for now.')
 
 
 def _prepare_cov_source(cov_source):
@@ -198,6 +209,9 @@ def pytest_sessionstart(self, session):
         elif not self._started:
             self.start(engine.Central)
 
+        if self.options.cov_context == 'test':
+            session.config.pluginmanager.register(TestContextPlugin(self.cov_controller.cov), '_cov_contexts')
+
     def pytest_configure_node(self, node):
         """Delegate to our implementation.
 
@@ -308,6 +322,24 @@ def pytest_runtest_call(self, item):
             yield
 
 
+class TestContextPlugin(object):
+    def __init__(self, cov):
+        self.cov = cov
+
+    def pytest_runtest_setup(self, item):
+        self.switch_context(item, 'setup')
+
+    def pytest_runtest_teardown(self, item):
+        self.switch_context(item, 'teardown')
+
+    def pytest_runtest_call(self, item):
+        self.switch_context(item, 'run')
+
+    def switch_context(self, item, when):
+        context = "{item.nodeid}|{when}".format(item=item, when=when)
+        self.cov.switch_context(context)
+
+
 @pytest.fixture
 def no_cover():
     """A pytest fixture to disable coverage."""

tests/contextful.py

@@ -0,0 +1,105 @@
+# A test file for test_pytest_cov.py:test_contexts
+
+import unittest
+
+import pytest
+
+
+def test_01():
+    assert 1 == 1                                   # r1
+
+
+def test_02():
+    assert 2 == 2                                   # r2
+
+
+class OldStyleTests(unittest.TestCase):
+    items = []
+
+    @classmethod
+    def setUpClass(cls):
+        cls.items.append("hello")                   # s3
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.items.pop()                             # t4
+
+    def setUp(self):
+        self.number = 1                             # r3 r4
+
+    def tearDown(self):
+        self.number = None                          # r3 r4
+
+    def test_03(self):
+        assert self.number == 1                     # r3
+        assert self.items[0] == "hello"             # r3
+
+    def test_04(self):
+        assert self.number == 1                     # r4
+        assert self.items[0] == "hello"             # r4
+
+
+@pytest.fixture
+def some_data():
+    return [1, 2, 3]                                # s5 s6
+
+
+def test_05(some_data):
+    assert len(some_data) == 3                      # r5
+
+
+@pytest.fixture
+def more_data(some_data):
+    return [2*x for x in some_data]                 # s6
+
+
+def test_06(some_data, more_data):
+    assert len(some_data) == len(more_data)         # r6
+
+
+@pytest.fixture(scope='session')
+def expensive_data():
+    return list(range(10))                          # s7
+
+
+def test_07(expensive_data):
+    assert len(expensive_data) == 10                # r7
+
+
+def test_08(expensive_data):
+    assert len(expensive_data) == 10                # r8
+
+
+@pytest.fixture(params=[1, 2, 3])
+def parametrized_number(request):
+    return request.param                            # s9-1 s9-2 s9-3
+
+
+def test_09(parametrized_number):
+    assert parametrized_number > 0                  # r9-1 r9-2 r9-3
+
+
+def test_10():
+    assert 1 == 1                                   # r10
+
+
+@pytest.mark.parametrize("x, ans", [
+    (1, 101),
+    (2, 202),
+])
+def test_11(x, ans):
+    assert 100 * x + x == ans                       # r11-1 r11-2
+
+
+@pytest.mark.parametrize("x, ans", [
+    (1, 101),
+    (2, 202),
+], ids=['one', 'two'])
+def test_12(x, ans):
+    assert 100 * x + x == ans                       # r12-1 r12-2
+
+
+@pytest.mark.parametrize("x", [1, 2])
+@pytest.mark.parametrize("y", [3, 4])
+def test_13(x, y):
+    assert x + y > 0                                # r13-1 r13-2 r13-3 r13-4

tests/test_pytest_cov.py

@@ -1,6 +1,8 @@
+import collections
 import glob
 import os
 import platform
+import re
 import subprocess
 import sys
 from itertools import chain
@@ -1928,3 +1930,95 @@ def test_cov_and_no_cov(testdir):
                                script)
 
     assert result.ret == 0
+
+
+def find_labels(text, pattern):
+    all_labels = collections.defaultdict(list)
+    lines = text.splitlines()
+    for lineno, line in enumerate(lines, start=1):
+        labels = re.findall(pattern, line)
+        for label in labels:
+            all_labels[label].append(lineno)
+    return all_labels
+
+
+# The contexts and their labels in contextful.py
+EXPECTED_CONTEXTS = {
+    '': 'c0',
+    'test_contexts.py::test_01|run': 'r1',
+    'test_contexts.py::test_02|run': 'r2',
+    'test_contexts.py::OldStyleTests::test_03|setup': 's3',
+    'test_contexts.py::OldStyleTests::test_03|run': 'r3',
+    'test_contexts.py::OldStyleTests::test_04|run': 'r4',
+    'test_contexts.py::OldStyleTests::test_04|teardown': 't4',
+    'test_contexts.py::test_05|setup': 's5',
+    'test_contexts.py::test_05|run': 'r5',
+    'test_contexts.py::test_06|setup': 's6',
+    'test_contexts.py::test_06|run': 'r6',
+    'test_contexts.py::test_07|setup': 's7',
+    'test_contexts.py::test_07|run': 'r7',
+    'test_contexts.py::test_08|run': 'r8',
+    'test_contexts.py::test_09[1]|setup': 's9-1',
+    'test_contexts.py::test_09[1]|run': 'r9-1',
+    'test_contexts.py::test_09[2]|setup': 's9-2',
+    'test_contexts.py::test_09[2]|run': 'r9-2',
+    'test_contexts.py::test_09[3]|setup': 's9-3',
+    'test_contexts.py::test_09[3]|run': 'r9-3',
+    'test_contexts.py::test_10|run': 'r10',
+    'test_contexts.py::test_11[1-101]|run': 'r11-1',
+    'test_contexts.py::test_11[2-202]|run': 'r11-2',
+    'test_contexts.py::test_12[one]|run': 'r12-1',
+    'test_contexts.py::test_12[two]|run': 'r12-2',
+    'test_contexts.py::test_13[3-1]|run': 'r13-1',
+    'test_contexts.py::test_13[3-2]|run': 'r13-2',
+    'test_contexts.py::test_13[4-1]|run': 'r13-3',
+    'test_contexts.py::test_13[4-2]|run': 'r13-4',
+}
+
+
+@pytest.mark.skipif("coverage.version_info < (5, 0)")
+@xdist_params
+def test_contexts(testdir, opts):
+    with open(os.path.join(os.path.dirname(__file__), "contextful.py")) as f:
+        contextful_tests = f.read()
+    script = testdir.makepyfile(contextful_tests)
+    result = testdir.runpytest('-v',
+                               '--cov=%s' % script.dirpath(),
+                               '--cov-context=test',
+                               script,
+                               *opts.split()
+                               )
+    assert result.ret == 0
+    result.stdout.fnmatch_lines([
+        'test_contexts* 100%*',
+    ])
+
+    data = coverage.CoverageData(".coverage")
+    data.read()
+    assert data.measured_contexts() == set(EXPECTED_CONTEXTS)
+    measured = data.measured_files()
+    assert len(measured) == 1
+    test_context_path = list(measured)[0]
+    assert test_context_path.lower() == os.path.abspath("test_contexts.py").lower()
+
+    line_data = find_labels(contextful_tests, r"[crst]\d+(?:-\d+)?")
+    for context, label in EXPECTED_CONTEXTS.items():
+        if context == '':
+            continue
+        context_pattern = re.sub(r"[\[\|]", r"[\g<0>]", context)
+        actual = data.lines(test_context_path, contexts=[context_pattern])
+        assert line_data[label] == actual, "Wrong lines for context {!r}".format(context)
+
+
+@pytest.mark.skipif("coverage.version_info >= (5, 0)")
+def test_contexts_not_supported(testdir):
+    script = testdir.makepyfile("a = 1")
+    result = testdir.runpytest('-v',
+                               '--cov=%s' % script.dirpath(),
+                               '--cov-context=test',
+                               script,
+                               )
+    result.stderr.fnmatch_lines([
+        '*argument --cov-context: Contexts are only supported with coverage.py >= 5.x',
+    ])
+    assert result.ret != 0