Merge pull request #36 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

ci/code_checks.sh

@@ -113,7 +113,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     # Imports - Check formatting using isort see setup.cfg for settings
     MSG='Check import format using isort' ; echo $MSG
-    ISORT_CMD="isort --recursive --check-only pandas asv_bench"
+    ISORT_CMD="isort --quiet --recursive --check-only pandas asv_bench"
     if [[ "$GITHUB_ACTIONS" == "true" ]]; then
         eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
     else

doc/source/whatsnew/v1.0.2.rst

@@ -15,7 +15,7 @@ including other versions of pandas.
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
--
+- Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.1.0.rst

@@ -63,7 +63,8 @@ Backwards incompatible API changes
 - :meth:`DataFrameGroupby.mean` and :meth:`SeriesGroupby.mean` (and similarly for :meth:`~DataFrameGroupby.median`, :meth:`~DataFrameGroupby.std`` and :meth:`~DataFrameGroupby.var``)
   now raise a  ``TypeError`` if a not-accepted keyword argument is passed into it.
   Previously a ``UnsupportedFunctionCall`` was raised (``AssertionError`` if ``min_count`` passed into :meth:`~DataFrameGroupby.median``) (:issue:`31485`)
-
+- :meth:`DataFrame.at` and :meth:`Series.at` will raise a ``TypeError`` instead of a ``ValueError`` if an incompatible key is passed, and ``KeyError`` if a missing key is passed, matching the behavior of ``.loc[]`` (:issue:`31722`)
+-
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/index.pyx

@@ -1,17 +1,12 @@
-from datetime import datetime, timedelta, date
 import warnings
 
-import cython
-
 import numpy as np
 cimport numpy as cnp
 from numpy cimport (ndarray, intp_t,
                     float64_t, float32_t,
                     int64_t, int32_t, int16_t, int8_t,
-                    uint64_t, uint32_t, uint16_t, uint8_t,
-                    # Note: NPY_DATETIME, NPY_TIMEDELTA are only available
-                    # for cimport in cython>=0.27.3
-                    NPY_DATETIME, NPY_TIMEDELTA)
+                    uint64_t, uint32_t, uint16_t, uint8_t
+)
 cnp.import_array()
 
 
@@ -23,7 +18,7 @@ from pandas._libs.tslibs.c_timestamp cimport _Timestamp
 from pandas._libs.hashtable cimport HashTable
 
 from pandas._libs import algos, hashtable as _hash
-from pandas._libs.tslibs import Timestamp, Timedelta, period as periodlib
+from pandas._libs.tslibs import Timedelta, period as periodlib
 from pandas._libs.missing import checknull
 
 
@@ -35,16 +30,6 @@ cdef inline bint is_definitely_invalid_key(object val):
     return False
 
 
-cpdef get_value_at(ndarray arr, object loc, object tz=None):
-    obj = util.get_value_at(arr, loc)
-
-    if arr.descr.type_num == NPY_DATETIME:
-        return Timestamp(obj, tz=tz)
-    elif arr.descr.type_num == NPY_TIMEDELTA:
-        return Timedelta(obj)
-    return obj
-
-
 # Don't populate hash tables in monotonic indexes larger than this
 _SIZE_CUTOFF = 1_000_000
 
@@ -72,21 +57,6 @@ cdef class IndexEngine:
         self._ensure_mapping_populated()
         return val in self.mapping
 
-    cpdef get_value(self, ndarray arr, object key, object tz=None):
-        """
-        Parameters
-        ----------
-        arr : 1-dimensional ndarray
-        """
-        cdef:
-            object loc
-
-        loc = self.get_loc(key)
-        if isinstance(loc, slice) or util.is_array(loc):
-            return arr[loc]
-        else:
-            return get_value_at(arr, loc, tz=tz)
-
     cpdef get_loc(self, object val):
         cdef:
             Py_ssize_t loc

pandas/_libs/util.pxd

@@ -1,7 +1,5 @@
 from pandas._libs.tslibs.util cimport *
 
-from cython cimport Py_ssize_t
-
 cimport numpy as cnp
 from numpy cimport ndarray
 
@@ -51,49 +49,3 @@ cdef inline void set_array_not_contiguous(ndarray ao) nogil:
     PyArray_CLEARFLAGS(ao,
                        (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS))
 
-
-cdef inline Py_ssize_t validate_indexer(ndarray arr, object loc) except -1:
-    """
-    Cast the given indexer `loc` to an integer.  If it is negative, i.e. a
-    python-style indexing-from-the-end indexer, translate it to a
-    from-the-front indexer.  Raise if this is not possible.
-
-    Parameters
-    ----------
-    arr : ndarray
-    loc : object
-
-    Returns
-    -------
-    idx : Py_ssize_t
-
-    Raises
-    ------
-    IndexError
-    """
-    cdef:
-        Py_ssize_t idx, size
-        int casted
-
-    if is_float_object(loc):
-        casted = int(loc)
-        if casted == loc:
-            loc = casted
-
-    idx = <Py_ssize_t>loc
-    size = cnp.PyArray_SIZE(arr)
-
-    if idx < 0 and size > 0:
-        idx += size
-    if idx >= size or size == 0 or idx < 0:
-        raise IndexError('index out of bounds')
-
-    return idx
-
-
-cdef inline object get_value_at(ndarray arr, object loc):
-    cdef:
-        Py_ssize_t i
-
-    i = validate_indexer(arr, loc)
-    return arr[i]

pandas/_testing.py

@@ -8,7 +8,7 @@
 from shutil import rmtree
 import string
 import tempfile
-from typing import Any, List, Optional, Union, cast
+from typing import Any, Callable, List, Optional, Type, Union, cast
 import warnings
 import zipfile
 
@@ -2757,3 +2757,24 @@ def convert_rows_list_to_csv_str(rows_list: List[str]):
     sep = os.linesep
     expected = sep.join(rows_list) + sep
     return expected
+
+
+def external_error_raised(
+    expected_exception: Type[Exception],
+) -> Callable[[Type[Exception], None], None]:
+    """
+    Helper function to mark pytest.raises that have an external error message.
+
+    Parameters
+    ----------
+    expected_exception : Exception
+        Expected error to raise.
+
+    Returns
+    -------
+    Callable
+        Regular `pytest.raises` function with `match` equal to `None`.
+    """
+    import pytest
+
+    return pytest.raises(expected_exception, match=None)

pandas/core/arrays/sparse/array.py

@@ -9,7 +9,7 @@
 
 import numpy as np
 
-from pandas._libs import index as libindex, lib
+from pandas._libs import lib
 import pandas._libs.sparse as splib
 from pandas._libs.sparse import BlockIndex, IntIndex, SparseIndex
 from pandas._libs.tslibs import NaT
@@ -794,7 +794,9 @@ def _get_val_at(self, loc):
         if sp_loc == -1:
             return self.fill_value
         else:
-            return libindex.get_value_at(self.sp_values, sp_loc)
+            val = self.sp_values[sp_loc]
+            val = com.maybe_box_datetimelike(val, self.sp_values.dtype)
+            return val
 
     def take(self, indices, allow_fill=False, fill_value=None):
         if is_scalar(indices):

pandas/core/common.py

@@ -72,8 +72,12 @@ def consensus_name_attr(objs):
     return name
 
 
-def maybe_box_datetimelike(value):
+def maybe_box_datetimelike(value, dtype=None):
     # turn a datetime like into a Timestamp/timedelta as needed
+    if dtype == object:
+        # If we dont have datetime64/timedelta64 dtype, we dont want to
+        #  box datetimelike scalars
+        return value
 
     if isinstance(value, (np.datetime64, datetime)):
         value = tslibs.Timestamp(value)

pandas/core/generic.py

@@ -3498,7 +3498,9 @@ def _iget_item_cache(self, item):
     def _box_item_values(self, key, values):
         raise AbstractMethodError(self)
 
-    def _slice(self: FrameOrSeries, slobj: slice, axis=0, kind=None) -> FrameOrSeries:
+    def _slice(
+        self: FrameOrSeries, slobj: slice, axis=0, kind: str = "getitem"
+    ) -> FrameOrSeries:
         """
         Construct a slice of this container.
 

pandas/core/indexes/base.py

@@ -3100,20 +3100,16 @@ def _filter_indexer_tolerance(
     # --------------------------------------------------------------------
     # Indexer Conversion Methods
 
-    def _convert_scalar_indexer(self, key, kind=None):
+    def _convert_scalar_indexer(self, key, kind: str_t):
         """
         Convert a scalar indexer.
 
         Parameters
         ----------
         key : label of the slice bound
-        kind : {'loc', 'getitem', 'iloc'} or None
+        kind : {'loc', 'getitem'}
         """
-        assert kind in ["loc", "getitem", "iloc", None]
-
-        if kind == "iloc":
-            self._validate_indexer("positional", key, "iloc")
-            return key
+        assert kind in ["loc", "getitem"]
 
         if len(self) and not isinstance(self, ABCMultiIndex):
 
@@ -4590,9 +4586,9 @@ def get_value(self, series: "Series", key):
             # If that fails, raise a KeyError if an integer
             # index, otherwise, see if key is an integer, and
             # try that
-            loc = self._engine.get_loc(key)
+            loc = self.get_loc(key)
         except KeyError:
-            if len(self) > 0 and (self.holds_integer() or self.is_boolean()):
+            if not self._should_fallback_to_positional():
                 raise
             elif is_integer(key):
                 # If the Index cannot hold integer, then this is unambiguously
@@ -4603,6 +4599,14 @@ def get_value(self, series: "Series", key):
 
         return self._get_values_for_loc(series, loc)
 
+    def _should_fallback_to_positional(self) -> bool:
+        """
+        If an integer key is not found, should we fall back to positional indexing?
+        """
+        if len(self) > 0 and (self.holds_integer() or self.is_boolean()):
+            return False
+        return True
+
     def _get_values_for_loc(self, series: "Series", loc):
         """
         Do a positional lookup on the given Series, returning either a scalar

pandas/core/indexes/category.py

@@ -439,44 +439,10 @@ def _to_safe_for_reshape(self):
         """ convert to object if we are a categorical """
         return self.astype("object")
 
-    def get_loc(self, key, method=None):
-        """
-        Get integer location, slice or boolean mask for requested label.
-
-        Parameters
-        ----------
-        key : label
-        method : {None}
-            * default: exact matches only.
-
-        Returns
-        -------
-        loc : int if unique index, slice if monotonic index, else mask
-
-        Raises
-        ------
-        KeyError : if the key is not in the index
-
-        Examples
-        --------
-        >>> unique_index = pd.CategoricalIndex(list('abc'))
-        >>> unique_index.get_loc('b')
-        1
-
-        >>> monotonic_index = pd.CategoricalIndex(list('abbc'))
-        >>> monotonic_index.get_loc('b')
-        slice(1, 3, None)
-
-        >>> non_monotonic_index = pd.CategoricalIndex(list('abcb'))
-        >>> non_monotonic_index.get_loc('b')
-        array([False,  True, False,  True], dtype=bool)
-        """
+    def _maybe_cast_indexer(self, key):
         code = self.categories.get_loc(key)
         code = self.codes.dtype.type(code)
-        try:
-            return self._engine.get_loc(code)
-        except KeyError:
-            raise KeyError(key)
+        return code
 
     def get_value(self, series: "Series", key: Any):
         """
@@ -658,10 +624,11 @@ def get_indexer_non_unique(self, target):
         return ensure_platform_int(indexer), missing
 
     @Appender(Index._convert_scalar_indexer.__doc__)
-    def _convert_scalar_indexer(self, key, kind=None):
+    def _convert_scalar_indexer(self, key, kind: str):
+        assert kind in ["loc", "getitem"]
         if kind == "loc":
             try:
-                return self.categories._convert_scalar_indexer(key, kind=kind)
+                return self.categories._convert_scalar_indexer(key, kind="loc")
             except TypeError:
                 self._invalid_indexer("label", key)
         return super()._convert_scalar_indexer(key, kind=kind)

pandas/core/indexes/datetimelike.py

@@ -385,18 +385,18 @@ def _format_attrs(self):
     # --------------------------------------------------------------------
     # Indexing Methods
 
-    def _convert_scalar_indexer(self, key, kind=None):
+    def _convert_scalar_indexer(self, key, kind: str):
         """
         We don't allow integer or float indexing on datetime-like when using
         loc.
 
         Parameters
         ----------
         key : label of the slice bound
-        kind : {'loc', 'getitem', 'iloc'} or None
+        kind : {'loc', 'getitem'}
         """
 
-        assert kind in ["loc", "getitem", "iloc", None]
+        assert kind in ["loc", "getitem"]
 
         if not is_scalar(key):
             raise TypeError(key)