Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-arith

Author: jbrockmendel

.github/CODE_OF_CONDUCT.md

@@ -60,4 +60,3 @@ and the [Swift Code of Conduct][swift].
 [homepage]: https://www.contributor-covenant.org
 [version]: https://www.contributor-covenant.org/version/1/3/0/
 [swift]: https://swift.org/community/#code-of-conduct
-

.pre-commit-config.yaml

@@ -47,3 +47,8 @@ repos:
     rev: v1.2.2
     hooks:
     -   id: yesqa
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: end-of-file-fixer
+        exclude: '.html$|^LICENSES/|.csv$|.txt$|.svg$|.py$'

AUTHORS.md

@@ -54,4 +54,3 @@ pandas is distributed under a 3-clause ("Simplified" or "New") BSD
 license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have
 BSD-compatible licenses, are included. Their licenses follow the pandas
 license.
-

ci/travis_process_gbq_encryption.sh

@@ -10,4 +10,3 @@ elif [[ -n ${!TRAVIS_IV_ENV} ]]; then
     export GBQ_PROJECT_ID='pandas-gbq-tests';
     echo 'Successfully decrypted gbq credentials'
 fi
-

doc/data/iris.data

@@ -148,4 +148,4 @@ SepalLength,SepalWidth,PetalLength,PetalWidth,Name
 6.3,2.5,5.0,1.9,Iris-virginica
 6.5,3.0,5.2,2.0,Iris-virginica
 6.2,3.4,5.4,2.3,Iris-virginica
-5.9,3.0,5.1,1.8,Iris-virginica
+5.9,3.0,5.1,1.8,Iris-virginica

doc/source/development/contributing.rst

@@ -837,6 +837,9 @@ to run its checks by running::
 
 without having to have done ``pre-commit install`` beforehand.
 
+Note that if you have conflicting installations of ``virtualenv``, then you may get an
+error - see `here <https://github.com/pypa/virtualenv/issues/1875>`_.
+
 Backwards compatibility
 ~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/development/developer.rst

@@ -184,4 +184,4 @@ As an example of fully-formed metadata:
     'creator': {
       'library': 'pyarrow',
       'version': '0.13.0'
-    }}
+    }}

doc/source/getting_started/intro_tutorials/index.rst

@@ -19,4 +19,3 @@ Getting started tutorials
     08_combine_dataframes
     09_timeseries
     10_text_data
-

doc/source/getting_started/overview.rst

@@ -174,4 +174,3 @@ License
 -------
 
 .. literalinclude:: ../../../LICENSE
-

doc/source/reference/general_utility_functions.rst

@@ -122,4 +122,3 @@ Bug report function
    :toctree: api/
 
    show_versions
-

doc/source/whatsnew/v0.14.0.rst

@@ -1084,4 +1084,4 @@ Bug fixes
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v0.13.1..v0.14.0
+.. contributors:: v0.13.1..v0.14.0

doc/source/whatsnew/v1.2.0.rst

@@ -336,6 +336,7 @@ Numeric
 - Bug in :class:`Series` where two :class:`Series` each have a :class:`DatetimeIndex` with different timezones having those indexes incorrectly changed when performing arithmetic operations (:issue:`33671`)
 - Bug in :meth:`pd._testing.assert_almost_equal` was incorrect for complex numeric types (:issue:`28235`)
 - Bug in :meth:`DataFrame.__rmatmul__` error handling reporting transposed shapes (:issue:`21581`)
+- Bug in :class:`IntegerArray` multiplication with ``timedelta`` and ``np.timedelta64`` objects (:issue:`36870`)
 - Bug in :class:`IntervalArray` comparisons with :class:`Series` not returning :class:`Series` (:issue:`36908`)
 
 Conversion
@@ -424,6 +425,7 @@ Reshaping
 - Bug in :func:`union_indexes` where input index names are not preserved in some cases. Affects :func:`concat` and :class:`DataFrame` constructor (:issue:`13475`)
 - Bug in func :meth:`crosstab` when using multiple columns with ``margins=True`` and ``normalize=True`` (:issue:`35144`)
 - Bug in :meth:`DataFrame.agg` with ``func={'name':<FUNC>}`` incorrectly raising ``TypeError`` when ``DataFrame.columns==['Name']`` (:issue:`36212`)
+- Bug in :meth:`Series.transform` would give incorrect results or raise when the argument ``func`` was dictionary (:issue:`35811`)
 -
 
 Sparse
@@ -446,7 +448,6 @@ Other
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`)
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`)
 - Fixed metadata propagation in the :class:`Series.dt` accessor (:issue:`28283`)
-- Bug in :meth:`Series.transform` would give incorrect results or raise when the argument ``func`` was dictionary (:issue:`35811`)
 - Bug in :meth:`Index.union` behaving differently depending on whether operand is a :class:`Index` or other list-like (:issue:`36384`)
 
 .. ---------------------------------------------------------------------------

flake8/cython-template.cfg

@@ -1,4 +1,3 @@
 [flake8]
 filename = *.pxi.in
 select = E501,E302,E203,E111,E114,E221,E303,E231,E126,F403
-

pandas/_libs/src/klib/khash_python.h

@@ -121,4 +121,4 @@ void PANDAS_INLINE kh_destroy_str_starts(kh_str_starts_t* table) {
 
 void PANDAS_INLINE kh_resize_str_starts(kh_str_starts_t* table, khint_t val) {
 	kh_resize_str(table->table, val);
-}
+}

pandas/_libs/testing.pyx

@@ -7,36 +7,13 @@ from numpy cimport import_array
 
 import_array()
 
-from pandas._libs.util cimport is_array
 from pandas._libs.lib import is_complex
+from pandas._libs.util cimport is_array, is_real_number_object
 
 from pandas.core.dtypes.common import is_dtype_equal
 from pandas.core.dtypes.missing import array_equivalent, isna
 
 
-cdef NUMERIC_TYPES = (
-    bool,
-    int,
-    float,
-    np.bool_,
-    np.int8,
-    np.int16,
-    np.int32,
-    np.int64,
-    np.uint8,
-    np.uint16,
-    np.uint32,
-    np.uint64,
-    np.float16,
-    np.float32,
-    np.float64,
-)
-
-
-cdef bint is_comparable_as_number(obj):
-    return isinstance(obj, NUMERIC_TYPES)
-
-
 cdef bint isiterable(obj):
     return hasattr(obj, '__iter__')
 
@@ -200,7 +177,7 @@ cpdef assert_almost_equal(a, b,
         # object comparison
         return True
 
-    if is_comparable_as_number(a) and is_comparable_as_number(b):
+    if is_real_number_object(a) and is_real_number_object(b):
         if array_equivalent(a, b, strict_nan=True):
             # inf comparison
             return True

pandas/_libs/tslibs/util.pxd

@@ -121,6 +121,10 @@ cdef inline bint is_bool_object(object obj) nogil:
             PyObject_TypeCheck(obj, &PyBoolArrType_Type))
 
 
+cdef inline bint is_real_number_object(object obj) nogil:
+    return is_bool_object(obj) or is_integer_object(obj) or is_float_object(obj)
+
+
 cdef inline bint is_timedelta64_object(object obj) nogil:
     """
     Cython equivalent of `isinstance(val, np.timedelta64)`

pandas/_libs/util.pxd

@@ -48,4 +48,3 @@ cdef inline void set_array_not_contiguous(ndarray ao) nogil:
     # ao->flags &= ~(NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS);
     PyArray_CLEARFLAGS(ao,
                        (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS))
-

pandas/core/aggregation.py

@@ -17,9 +17,17 @@
     Sequence,
     Tuple,
     Union,
+    cast,
 )
 
-from pandas._typing import AggFuncType, Axis, FrameOrSeries, Label
+from pandas._typing import (
+    AggFuncType,
+    AggFuncTypeBase,
+    Axis,
+    FrameOrSeries,
+    FrameOrSeriesUnion,
+    Label,
+)
 
 from pandas.core.dtypes.common import is_dict_like, is_list_like
 from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries
@@ -391,7 +399,7 @@ def validate_func_kwargs(
 
 def transform(
     obj: FrameOrSeries, func: AggFuncType, axis: Axis, *args, **kwargs
-) -> FrameOrSeries:
+) -> FrameOrSeriesUnion:
     """
     Transform a DataFrame or Series
 
@@ -424,16 +432,20 @@ def transform(
         assert not is_series
         return transform(obj.T, func, 0, *args, **kwargs).T
 
-    if isinstance(func, list):
+    if is_list_like(func) and not is_dict_like(func):
+        func = cast(List[AggFuncTypeBase], func)
+        # Convert func equivalent dict
         if is_series:
             func = {com.get_callable_name(v) or v: v for v in func}
         else:
             func = {col: func for col in obj}
 
-    if isinstance(func, dict):
+    if is_dict_like(func):
+        func = cast(Dict[Label, Union[AggFuncTypeBase, List[AggFuncTypeBase]]], func)
         return transform_dict_like(obj, func, *args, **kwargs)
 
     # func is either str or callable
+    func = cast(AggFuncTypeBase, func)
     try:
         result = transform_str_or_callable(obj, func, *args, **kwargs)
     except Exception:
@@ -451,37 +463,52 @@ def transform(
     return result
 
 
-def transform_dict_like(obj, func, *args, **kwargs):
+def transform_dict_like(
+    obj: FrameOrSeries,
+    func: Dict[Label, Union[AggFuncTypeBase, List[AggFuncTypeBase]]],
+    *args,
+    **kwargs,
+):
     """
     Compute transform in the case of a dict-like func
     """
     from pandas.core.reshape.concat import concat
 
+    if len(func) == 0:
+        raise ValueError("No transform functions were provided")
+
     if obj.ndim != 1:
+        # Check for missing columns on a frame
         cols = sorted(set(func.keys()) - set(obj.columns))
         if len(cols) > 0:
             raise SpecificationError(f"Column(s) {cols} do not exist")
 
-    if any(isinstance(v, dict) for v in func.values()):
+    # Can't use func.values(); wouldn't work for a Series
+    if any(is_dict_like(v) for _, v in func.items()):
         # GH 15931 - deprecation of renaming keys
         raise SpecificationError("nested renamer is not supported")
 
-    results = {}
+    results: Dict[Label, FrameOrSeriesUnion] = {}
     for name, how in func.items():
         colg = obj._gotitem(name, ndim=1)
         try:
             results[name] = transform(colg, how, 0, *args, **kwargs)
-        except Exception as e:
-            if str(e) == "Function did not transform":
-                raise e
+        except Exception as err:
+            if (
+                str(err) == "Function did not transform"
+                or str(err) == "No transform functions were provided"
+            ):
+                raise err
 
     # combine results
     if len(results) == 0:
         raise ValueError("Transform function failed")
     return concat(results, axis=1)
 
 
-def transform_str_or_callable(obj, func, *args, **kwargs):
+def transform_str_or_callable(
+    obj: FrameOrSeries, func: AggFuncTypeBase, *args, **kwargs
+) -> FrameOrSeriesUnion:
     """
     Compute transform in the case of a string or callable func
     """

pandas/core/arrays/_mixins.py

@@ -8,7 +8,9 @@
 from pandas.util._decorators import cache_readonly, doc
 from pandas.util._validators import validate_fillna_kwargs
 
+from pandas.core.dtypes.common import is_dtype_equal
 from pandas.core.dtypes.inference import is_array_like
+from pandas.core.dtypes.missing import array_equivalent
 
 from pandas.core import missing
 from pandas.core.algorithms import take, unique
@@ -115,6 +117,13 @@ def T(self: _T) -> _T:
 
     # ------------------------------------------------------------------------
 
+    def equals(self, other) -> bool:
+        if type(self) is not type(other):
+            return False
+        if not is_dtype_equal(self.dtype, other.dtype):
+            return False
+        return bool(array_equivalent(self._ndarray, other._ndarray))
+
     def _values_for_argsort(self):
         return self._ndarray
 

pandas/core/arrays/integer.py

@@ -1,10 +1,11 @@
+from datetime import timedelta
 import numbers
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, Union
 import warnings
 
 import numpy as np
 
-from pandas._libs import lib, missing as libmissing
+from pandas._libs import Timedelta, iNaT, lib, missing as libmissing
 from pandas._typing import ArrayLike, DtypeObj
 from pandas.compat import set_function_name
 from pandas.compat.numpy import function as nv
@@ -581,6 +582,12 @@ def _maybe_mask_result(self, result, mask, other, op_name: str):
             result[mask] = np.nan
             return result
 
+        if result.dtype == "timedelta64[ns]":
+            from pandas.core.arrays import TimedeltaArray
+
+            result[mask] = iNaT
+            return TimedeltaArray._simple_new(result)
+
         return type(self)(result, mask, copy=False)
 
     @classmethod
@@ -609,6 +616,9 @@ def integer_arithmetic_method(self, other):
                 if not (is_float_dtype(other) or is_integer_dtype(other)):
                     raise TypeError("can only perform ops with numeric values")
 
+            elif isinstance(other, (timedelta, np.timedelta64)):
+                other = Timedelta(other)
+
             else:
                 if not (is_float(other) or is_integer(other) or other is libmissing.NA):
                     raise TypeError("can only perform ops with numeric values")

pandas/core/arrays/interval.py

@@ -708,6 +708,16 @@ def astype(self, dtype, copy=True):
             msg = f"Cannot cast {type(self).__name__} to dtype {dtype}"
             raise TypeError(msg) from err
 
+    def equals(self, other) -> bool:
+        if type(self) != type(other):
+            return False
+
+        return bool(
+            self.closed == other.closed
+            and self.left.equals(other.left)
+            and self.right.equals(other.right)
+        )
+
     @classmethod
     def _concat_same_type(cls, to_concat):
         """