ENH: Use flake8 to check for PEP8 violations in doctests

scripts/tests/test_validate_docstrings.py

@@ -127,6 +127,7 @@ def head1(self, n=5):
 
         Examples
         --------
+        >>> import pandas as pd
         >>> s = pd.Series(['Ant', 'Bear', 'Cow', 'Dog', 'Falcon'])
         >>> s.head()
         0   Ant
@@ -164,6 +165,8 @@ def contains(self, pat, case=True, na=np.nan):
 
         Examples
         --------
+        >>> import pandas as pd
+        >>> import numpy as np
         >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan])
         >>> s.str.contains(pat='a')
         0    False
@@ -584,6 +587,29 @@ def prefix_pandas(self):
         pass
 
 
+class BadExamples(object):
+
+    def doctest(self):
+        """
+        Return whether each value contains `pat`.
+
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> df = pd.DataFrame(np.ones((3, 3)),
+        ...                   columns=('a', 'b', 'c'))
+        >>> df.all(1)
+        0    True
+        1    True
+        2    True
+        dtype: bool
+
+        >>> df.all(bool_only=True)
+        Series([], dtype: bool)
+        """
+        pass
+
+
 class TestValidator(object):
 
     def _import_path(self, klass=None, func=None):
@@ -703,10 +729,13 @@ def test_bad_generic_functions(self, func):
         # See Also tests
         ('BadSeeAlso', 'prefix_pandas',
          ('pandas.Series.rename in `See Also` section '
-          'does not need `pandas` prefix',))
+          'does not need `pandas` prefix',)),
+        # Examples tests
+        ('BadExamples', 'doctest',
+         ('1 F821 undefined name \'np\'',))
     ])
     def test_bad_examples(self, capsys, klass, func, msgs):
-        result = validate_one(self._import_path(klass=klass, func=func))  # noqa:F821
+        result = validate_one(self._import_path(klass=klass, func=func))
         for msg in msgs:
             assert msg in ' '.join(result['errors'])
 

scripts/validate_docstrings.py

@@ -24,6 +24,11 @@
 import inspect
 import importlib
 import doctest
+import tempfile
+from contextlib import contextmanager
+
+from flake8.main.application import Application as Flake8
+
 try:
     from io import StringIO
 except ImportError:
@@ -331,6 +336,41 @@ def parameter_mismatches(self):
 
         return errs
 
+    @property
+    def pep8_violations(self):
+        with self._file_representation() as file:
+            application = Flake8()
+            application.initialize(["--doctests", "--quiet"])
+            application.run_checks([file.name])
+            application.report()
+            stats = application.guide.stats
+            return [
+                "{} {} {}".format(s.count, s.error_code, s.message)
+                for s in stats.statistics_for('')
+            ]
+
+    @contextmanager
+    def _file_representation(self):
+        """
+        Temporarily creates file with current function inside.
+        The signature and body are **not** included.
+
+        :returns file
+        """
+        create_function = 'def {name}():\n' \
+                          '    """{doc}"""\n' \
+                          '    pass\n'
+
+        name = self.name.split('.')[-1]
+        lines = self.clean_doc.split("\n")
+        indented_lines = [(' ' * 4) + line if line else ''
+                          for line in lines[1:]]
+        doc = '\n'.join([lines[0], *indented_lines])
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as file:
+            file.write(create_function.format(name=name, doc=doc))
+            file.flush()
+            yield file
+
     @property
     def correct_parameters(self):
         return not bool(self.parameter_mismatches)
@@ -446,7 +486,7 @@ def validate_one(func_name):
         if doc.summary != doc.summary.lstrip():
             errs.append('Summary contains heading whitespaces.')
         elif (doc.is_function_or_method
-                and doc.summary.split(' ')[0][-1] == 's'):
+              and doc.summary.split(' ')[0][-1] == 's'):
             errs.append('Summary must start with infinitive verb, '
                         'not third person (e.g. use "Generate" instead of '
                         '"Generates")')
@@ -490,6 +530,12 @@ def validate_one(func_name):
         for param_err in param_errs:
             errs.append('\t{}'.format(param_err))
 
+    pep8_errs = doc.pep8_violations
+    if pep8_errs:
+        errs.append('Errors in doctests')
+        for pep8_err in pep8_errs:
+            errs.append('\t{}'.format(pep8_err))
+
     if doc.is_function_or_method:
         if not doc.returns and "return" in doc.method_source:
             errs.append('No Returns section found')