Merge remote-tracking branch 'upstream/master' into doc-build-single-docstring

Author: jorisvandenbossche

.pep8speaks.yml

@@ -6,5 +6,7 @@ scanner:
 pycodestyle:
     max-line-length: 79
     ignore:  # Errors and warnings to ignore
-        - E731
-        - E402
+        - E402,  # module level import not at top of file
+        - E731,  # do not assign a lambda expression, use a def
+        - E741,  # do not use variables named 'l', 'O', or 'I'
+        - W503   # line break before binary operator

ci/lint.sh

@@ -37,12 +37,12 @@ if [ "$LINT" ]; then
     fi
     echo "Linting scripts/*.py DONE"
 
-    echo "Linting doc script"
-    flake8 doc/make.py
+    echo "Linting doc scripts"
+    flake8 doc/make.py doc/source/conf.py
     if [ $? -ne "0" ]; then
         RET=1
     fi
-    echo "Linting doc script DONE"
+    echo "Linting doc scripts DONE"
 
     echo "Linting *.pyx"
     flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403

doc/source/conf.py

@@ -2,7 +2,8 @@
 #
 # pandas documentation build configuration file, created by
 #
-# This file is execfile()d with the current directory set to its containing dir.
+# This file is execfile()d with the current directory set to its containing
+# dir.
 #
 # Note that not all possible configuration values are present in this
 # autogenerated file.
@@ -49,8 +50,9 @@
 
 # -- General configuration -----------------------------------------------
 
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.  sphinxext.
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+# sphinxext.
 
 extensions = ['sphinx.ext.autodoc',
               'sphinx.ext.autosummary',
@@ -60,7 +62,8 @@
               'numpydoc',
               'ipython_sphinxext.ipython_directive',
               'ipython_sphinxext.ipython_console_highlighting',
-              'IPython.sphinxext.ipython_console_highlighting',  # lowercase didn't work
+              # lowercase didn't work
+              'IPython.sphinxext.ipython_console_highlighting',
               'sphinx.ext.intersphinx',
               'sphinx.ext.coverage',
               'sphinx.ext.mathjax',
@@ -107,7 +110,7 @@
 import pandas
 
 # version = '%s r%s' % (pandas.__version__, svn_version())
-version = '%s' % (pandas.__version__)
+version = str(pandas.__version__)
 
 # The full version, including alpha/beta/rc tags.
 release = version
@@ -129,8 +132,8 @@
 # for source files.
 exclude_trees = []
 
-# The reST default role (used for this markup: `text`) to use for all documents.
-# default_role = None
+# The reST default role (used for this markup: `text`) to use for all
+# documents. default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 # add_function_parentheses = True
@@ -304,8 +307,8 @@
 # The font size ('10pt', '11pt' or '12pt').
 # latex_font_size = '10pt'
 
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
+# Grouping the document tree into LaTeX files. List of tuples (source start
+# file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
     ('index', 'pandas.tex',
      u('pandas: powerful Python data analysis toolkit'),
@@ -362,23 +365,23 @@
     # wherever the docs are built. The docs' target is the browser, not
     # the console, so this is fine.
     'pd.options.display.encoding="utf8"'
-    ]
+]
 
 
 # Add custom Documenter to handle attributes/methods of an AccessorProperty
 # eg pandas.Series.str and pandas.Series.dt (see GH9322)
 
 import sphinx
 from sphinx.util import rpartition
-from sphinx.ext.autodoc import Documenter, MethodDocumenter, AttributeDocumenter
+from sphinx.ext.autodoc import (
+    Documenter, MethodDocumenter, AttributeDocumenter)
 from sphinx.ext.autosummary import Autosummary
 
 
 class AccessorDocumenter(MethodDocumenter):
     """
     Specialized Documenter subclass for accessors.
     """
-
     objtype = 'accessor'
     directivetype = 'method'
 
@@ -396,7 +399,6 @@ class AccessorLevelDocumenter(Documenter):
     Specialized Documenter subclass for objects on accessor level (methods,
     attributes).
     """
-
     # This is the simple straightforward version
     # modname is None, base the last elements (eg 'hour')
     # and path the part before (eg 'Series.dt')
@@ -406,7 +408,6 @@ class AccessorLevelDocumenter(Documenter):
     #     mod_cls = mod_cls.split('.')
     #
     #     return modname, mod_cls + [base]
-
     def resolve_name(self, modname, parents, path, base):
         if modname is None:
             if path:
@@ -441,16 +442,17 @@ def resolve_name(self, modname, parents, path, base):
         return modname, parents + [base]
 
 
-class AccessorAttributeDocumenter(AccessorLevelDocumenter, AttributeDocumenter):
-
+class AccessorAttributeDocumenter(AccessorLevelDocumenter,
+                                  AttributeDocumenter):
     objtype = 'accessorattribute'
     directivetype = 'attribute'
 
-    # lower than AttributeDocumenter so this is not chosen for normal attributes
+    # lower than AttributeDocumenter so this is not chosen for normal
+    # attributes
     priority = 0.6
 
-class AccessorMethodDocumenter(AccessorLevelDocumenter, MethodDocumenter):
 
+class AccessorMethodDocumenter(AccessorLevelDocumenter, MethodDocumenter):
     objtype = 'accessormethod'
     directivetype = 'method'
 
@@ -478,7 +480,6 @@ class PandasAutosummary(Autosummary):
     This alternative autosummary class lets us override the table summary for
     Series.plot and DataFrame.plot in the API docs.
     """
-
     def _replace_pandas_items(self, display_name, sig, summary, real_name):
         # this a hack: ideally we should extract the signature from the
         # .__call__ method instead of hard coding this
@@ -531,18 +532,18 @@ def linkcode_resolve(domain, info):
         lineno = None
 
     if lineno:
-        linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1)
+        linespec = "#L{:d}-L{:d}".format(lineno, lineno + len(source) - 1)
     else:
         linespec = ""
 
     fn = os.path.relpath(fn, start=os.path.dirname(pandas.__file__))
 
     if '+' in pandas.__version__:
-        return "http://github.com/pandas-dev/pandas/blob/master/pandas/%s%s" % (
-            fn, linespec)
+        return ("http://github.com/pandas-dev/pandas/blob/master/pandas/"
+                "{}{}".format(fn, linespec))
     else:
-        return "http://github.com/pandas-dev/pandas/blob/v%s/pandas/%s%s" % (
-            pandas.__version__, fn, linespec)
+        return ("http://github.com/pandas-dev/pandas/blob/"
+                "v{}/pandas/{}{}".format(pandas.__version__, fn, linespec))
 
 
 # remove the docstring of the flags attribute (inherited from numpy ndarray)

doc/source/whatsnew/v0.23.0.txt

@@ -757,6 +757,7 @@ Datetimelike
 Timedelta
 ^^^^^^^^^
 
+- Bug in :func:`Timedelta.__mul__` where multiplying by ``NaT`` returned ``NaT`` instead of raising a ``TypeError`` (:issue:`19819`)
 - Bug in :class:`Series` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` had results cast to ``dtype='int64'`` (:issue:`17250`)
 - Bug in :class:`Series` with ``dtype='timedelta64[ns]`` where addition or subtraction of ``TimedeltaIndex`` could return a ``Series`` with an incorrect name (:issue:`19043`)
 - Bug in :func:`Timedelta.__floordiv__` and :func:`Timedelta.__rfloordiv__` dividing by many incompatible numpy objects was incorrectly allowed (:issue:`18846`)
@@ -766,6 +767,7 @@ Timedelta
 - Bug in :func:`Timedelta.__floordiv__`, :func:`Timedelta.__rfloordiv__` where operating with a ``Tick`` object would raise a ``TypeError`` instead of returning a numeric value (:issue:`19738`)
 - Bug in :func:`Period.asfreq` where periods near ``datetime(1, 1, 1)`` could be converted incorrectly (:issue:`19643`)
 - Bug in :func:`Timedelta.total_seconds()` causing precision errors i.e. ``Timedelta('30S').total_seconds()==30.000000000000004`` (:issue:`19458`)
+- Bug in :func: `Timedelta.__rmod__` where operating with a ``numpy.timedelta64`` returned a ``timedelta64`` object instead of a ``Timedelta`` (:issue:`19820`)
 - Multiplication of :class:`TimedeltaIndex` by ``TimedeltaIndex`` will now raise ``TypeError`` instead of raising ``ValueError`` in cases of length mis-match (:issue`19333`)
 -
 
@@ -899,6 +901,7 @@ Reshaping
 - Bug in :func:`DataFrame.join` which does an ``outer`` instead of a ``left`` join when being called with multiple DataFrames and some have non-unique indices (:issue:`19624`)
 - :func:`Series.rename` now accepts ``axis`` as a kwarg (:issue:`18589`)
 - Comparisons between :class:`Series` and :class:`Index` would return a ``Series`` with an incorrect name, ignoring the ``Index``'s name attribute (:issue:`19582`)
+- Bug in :func:`qcut` where datetime and timedelta data with ``NaT`` present raised a ``ValueError`` (:issue:`19768`)
 
 Other
 ^^^^^

pandas/_libs/tslibs/timedeltas.pyx

@@ -1057,7 +1057,7 @@ class Timedelta(_Timedelta):
             return other * self.to_timedelta64()
 
         elif other is NaT:
-            return NaT
+            raise TypeError('Cannot multiply Timedelta with NaT')
 
         elif not (is_integer_object(other) or is_float_object(other)):
             # only integers and floats allowed
@@ -1109,7 +1109,11 @@ class Timedelta(_Timedelta):
                 return self // other.delta
             return NotImplemented
 
-        if hasattr(other, 'dtype'):
+        elif is_timedelta64_object(other):
+            # convert to Timedelta below
+            pass
+
+        elif hasattr(other, 'dtype'):
             if other.dtype.kind == 'm':
                 # also timedelta-like
                 return _broadcast_floordiv_td64(self.value, other, _floordiv)
@@ -1144,7 +1148,11 @@ class Timedelta(_Timedelta):
                 return other.delta // self
             return NotImplemented
 
-        if hasattr(other, 'dtype'):
+        elif is_timedelta64_object(other):
+            # convert to Timedelta below
+            pass
+
+        elif hasattr(other, 'dtype'):
             if other.dtype.kind == 'm':
                 # also timedelta-like
                 return _broadcast_floordiv_td64(self.value, other, _rfloordiv)

pandas/core/reshape/tile.py

@@ -279,18 +279,22 @@ def _trim_zeros(x):
 def _coerce_to_type(x):
     """
     if the passed data is of datetime/timedelta type,
-    this method converts it to integer so that cut method can
+    this method converts it to numeric so that cut method can
     handle it
     """
     dtype = None
 
     if is_timedelta64_dtype(x):
-        x = to_timedelta(x).view(np.int64)
+        x = to_timedelta(x)
         dtype = np.timedelta64
     elif is_datetime64_dtype(x):
-        x = to_datetime(x).view(np.int64)
+        x = to_datetime(x)
         dtype = np.datetime64
 
+    if dtype is not None:
+        # GH 19768: force NaT to NaN during integer conversion
+        x = np.where(x.notna(), x.view(np.int64), np.nan)
+
     return x, dtype
 
 

pandas/tests/reshape/test_tile.py

@@ -6,7 +6,8 @@
 
 from pandas import (Series, isna, to_datetime, DatetimeIndex,
                     Timestamp, Interval, IntervalIndex, Categorical,
-                    cut, qcut, date_range)
+                    cut, qcut, date_range, NaT, TimedeltaIndex)
+from pandas.tseries.offsets import Nano, Day
 import pandas.util.testing as tm
 from pandas.api.types import CategoricalDtype as CDT
 
@@ -250,6 +251,18 @@ def test_qcut_nas(self):
         result = qcut(arr, 4)
         assert isna(result[:20]).all()
 
+    @pytest.mark.parametrize('s', [
+        Series(DatetimeIndex(['20180101', NaT, '20180103'])),
+        Series(TimedeltaIndex(['0 days', NaT, '2 days']))],
+        ids=lambda x: str(x.dtype))
+    def test_qcut_nat(self, s):
+        # GH 19768
+        intervals = IntervalIndex.from_tuples(
+            [(s[0] - Nano(), s[2] - Day()), np.nan, (s[2] - Day(), s[2])])
+        expected = Series(Categorical(intervals, ordered=True))
+        result = qcut(s, 2)
+        tm.assert_series_equal(result, expected)
+
     def test_qcut_index(self):
         result = qcut([0, 2], 2)
         intervals = [Interval(-0.001, 1), Interval(1, 2)]

pandas/tests/scalar/timedelta/test_arithmetic.py

@@ -216,6 +216,16 @@ class TestTimedeltaMultiplicationDivision(object):
     # ---------------------------------------------------------------
     # Timedelta.__mul__, __rmul__
 
+    @pytest.mark.parametrize('td_nat', [pd.NaT,
+                                        np.timedelta64('NaT', 'ns'),
+                                        np.timedelta64('NaT')])
+    @pytest.mark.parametrize('op', [operator.mul, ops.rmul])
+    def test_td_mul_nat(self, op, td_nat):
+        # GH#19819
+        td = Timedelta(10, unit='d')
+        with pytest.raises(TypeError):
+            op(td, td_nat)
+
     @pytest.mark.parametrize('op', [operator.mul, ops.rmul])
     def test_td_mul_scalar(self, op):
         # GH#19738
@@ -441,15 +451,13 @@ def test_mod_timedeltalike(self):
         result = td % NaT
         assert result is NaT
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv td64 returns td64')
     def test_mod_timedelta64_nat(self):
         # GH#19365
         td = Timedelta(hours=37)
 
         result = td % np.timedelta64('NaT', 'ns')
         assert result is NaT
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv td64 returns td64')
     def test_mod_timedelta64(self):
         # GH#19365
         td = Timedelta(hours=37)
@@ -458,7 +466,6 @@ def test_mod_timedelta64(self):
         assert isinstance(result, Timedelta)
         assert result == Timedelta(hours=1)
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv by Tick not implemented')
     def test_mod_offset(self):
         # GH#19365
         td = Timedelta(hours=37)
@@ -505,7 +512,6 @@ def test_rmod_pytimedelta(self):
         assert isinstance(result, Timedelta)
         assert result == Timedelta(minutes=1)
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv by Tick not implemented')
     def test_rmod_timedelta64(self):
         # GH#19365
         td = Timedelta(minutes=3)
@@ -564,7 +570,6 @@ def test_divmod(self):
         assert np.isnan(result[0])
         assert result[1] is pd.NaT
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv by Tick not implemented')
     def test_divmod_offset(self):
         # GH#19365
         td = Timedelta(days=2, hours=6)
@@ -588,7 +593,6 @@ def test_rdivmod_pytimedelta(self):
         assert isinstance(result[1], Timedelta)
         assert result[1] == Timedelta(hours=6)
 
-    @pytest.mark.xfail(reason='GH#19378 floordiv by Tick not implemented')
     def test_rdivmod_offset(self):
         result = divmod(pd.offsets.Hour(54), Timedelta(hours=-4))
         assert result[0] == -14

pandas/tests/scalar/timedelta/test_timedelta.py

@@ -62,11 +62,6 @@ def test_unary_ops(self):
         assert abs(-td) == td
         assert abs(-td) == Timedelta('10d')
 
-    def test_binary_ops_nat(self):
-        td = Timedelta(10, unit='d')
-        # FIXME: The next test is wrong: td * NaT should raise
-        assert (td * pd.NaT) is pd.NaT
-
 
 class TestTimedeltaComparison(object):
     def test_comparison_object_array(self):