Merge remote-tracking branch 'upstream/master' into ea-xs

Author: TomAugspurger

doc/source/whatsnew/v0.24.0.txt

@@ -485,6 +485,7 @@ ExtensionType Changes
 - ``ExtensionArray`` has gained the abstract methods ``.dropna()`` (:issue:`21185`)
 - ``ExtensionDtype`` has gained the ability to instantiate from string dtypes, e.g. ``decimal`` would instantiate a registered ``DecimalDtype``; furthermore
   the ``ExtensionDtype`` has gained the method ``construct_array_type`` (:issue:`21185`)
+- An ``ExtensionArray`` with a boolean dtype now works correctly as a boolean indexer. :meth:`pandas.api.types.is_bool_dtype` now properly considers them boolean (:issue:`22326`)
 - Added ``ExtensionDtype._is_numeric`` for controlling whether an extension dtype is considered numeric (:issue:`22290`).
 - The ``ExtensionArray`` constructor, ``_from_sequence`` now take the keyword arg ``copy=False`` (:issue:`21185`)
 - Bug in :meth:`Series.get` for ``Series`` using ``ExtensionArray`` and integer index (:issue:`21257`)
@@ -617,6 +618,8 @@ Categorical
 ^^^^^^^^^^^
 
 - Bug in :meth:`Categorical.from_codes` where ``NaN`` values in ``codes`` were silently converted to ``0`` (:issue:`21767`). In the future this will raise a ``ValueError``. Also changes the behavior of ``.from_codes([1.1, 2.0])``.
+- Bug when indexing with a boolean-valued ``Categorical``. Now a boolean-valued ``Categorical`` is treated as a boolean mask (:issue:`22665`)
+- Constructing a :class:`CategoricalIndex` with empty values and boolean categories was raising a ``ValueError`` after a change to dtype coercion (:issue:`22702`).
 
 Datetimelike
 ^^^^^^^^^^^^

pandas/core/arrays/categorical.py

@@ -2439,9 +2439,13 @@ def _get_codes_for_values(values, categories):
     """
     utility routine to turn values into codes given the specified categories
     """
-
     from pandas.core.algorithms import _get_data_algo, _hashtables
-    if not is_dtype_equal(values.dtype, categories.dtype):
+    if is_dtype_equal(values.dtype, categories.dtype):
+        # To prevent erroneous dtype coercion in _get_data_algo, retrieve
+        # the underlying numpy array. gh-22702
+        values = getattr(values, 'values', values)
+        categories = getattr(categories, 'values', categories)
+    else:
         values = ensure_object(values)
         categories = ensure_object(categories)
 

pandas/core/common.py

@@ -15,7 +15,9 @@
 from pandas import compat
 from pandas.compat import iteritems, PY36, OrderedDict
 from pandas.core.dtypes.generic import ABCSeries, ABCIndex, ABCIndexClass
-from pandas.core.dtypes.common import is_integer
+from pandas.core.dtypes.common import (
+    is_integer, is_bool_dtype, is_extension_array_dtype, is_array_like
+)
 from pandas.core.dtypes.inference import _iterable_not_string
 from pandas.core.dtypes.missing import isna, isnull, notnull  # noqa
 from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
@@ -100,17 +102,45 @@ def maybe_box_datetimelike(value):
 
 
 def is_bool_indexer(key):
-    if isinstance(key, (ABCSeries, np.ndarray, ABCIndex)):
+    # type: (Any) -> bool
+    """
+    Check whether `key` is a valid boolean indexer.
+
+    Parameters
+    ----------
+    key : Any
+        Only list-likes may be considered boolean indexers.
+        All other types are not considered a boolean indexer.
+        For array-like input, boolean ndarrays or ExtensionArrays
+        with ``_is_boolean`` set are considered boolean indexers.
+
+    Returns
+    -------
+    bool
+
+    Raises
+    ------
+    ValueError
+        When the array is an object-dtype ndarray or ExtensionArray
+        and contains missing values.
+    """
+    na_msg = 'cannot index with vector containing NA / NaN values'
+    if (isinstance(key, (ABCSeries, np.ndarray, ABCIndex)) or
+            (is_array_like(key) and is_extension_array_dtype(key.dtype))):
         if key.dtype == np.object_:
             key = np.asarray(values_from_object(key))
 
             if not lib.is_bool_array(key):
                 if isna(key).any():
-                    raise ValueError('cannot index with vector containing '
-                                     'NA / NaN values')
+                    raise ValueError(na_msg)
                 return False
             return True
-        elif key.dtype == np.bool_:
+        elif is_bool_dtype(key.dtype):
+            # an ndarray with bool-dtype by definition has no missing values.
+            # So we only need to check for NAs in ExtensionArrays
+            if is_extension_array_dtype(key.dtype):
+                if np.any(key.isna()):
+                    raise ValueError(na_msg)
             return True
     elif isinstance(key, list):
         try:

pandas/core/dtypes/base.py

@@ -106,6 +106,25 @@ def _is_numeric(self):
         """
         return False
 
+    @property
+    def _is_boolean(self):
+        # type: () -> bool
+        """
+        Whether this dtype should be considered boolean.
+
+        By default, ExtensionDtypes are assumed to be non-numeric.
+        Setting this to True will affect the behavior of several places,
+        e.g.
+
+        * is_bool
+        * boolean indexing
+
+        Returns
+        -------
+        bool
+        """
+        return False
+
 
 class ExtensionDtype(_DtypeOpsMixin):
     """A custom data type, to be paired with an ExtensionArray.
@@ -125,6 +144,7 @@ class ExtensionDtype(_DtypeOpsMixin):
     pandas operations
 
     * _is_numeric
+    * _is_boolean
 
     Optionally one can override construct_array_type for construction
     with the name of this dtype via the Registry. See

pandas/core/dtypes/common.py

@@ -1619,6 +1619,11 @@ def is_bool_dtype(arr_or_dtype):
     -------
     boolean : Whether or not the array or dtype is of a boolean dtype.
 
+    Notes
+    -----
+    An ExtensionArray is considered boolean when the ``_is_boolean``
+    attribute is set to True.
+
     Examples
     --------
     >>> is_bool_dtype(str)
@@ -1635,6 +1640,8 @@ def is_bool_dtype(arr_or_dtype):
     False
     >>> is_bool_dtype(np.array([True, False]))
     True
+    >>> is_bool_dtype(pd.Categorical([True, False]))
+    True
     """
 
     if arr_or_dtype is None:
@@ -1645,6 +1652,13 @@ def is_bool_dtype(arr_or_dtype):
         # this isn't even a dtype
         return False
 
+    if isinstance(arr_or_dtype, (ABCCategorical, ABCCategoricalIndex)):
+        arr_or_dtype = arr_or_dtype.dtype
+
+    if isinstance(arr_or_dtype, CategoricalDtype):
+        arr_or_dtype = arr_or_dtype.categories
+        # now we use the special definition for Index
+
     if isinstance(arr_or_dtype, ABCIndexClass):
 
         # TODO(jreback)
@@ -1653,6 +1667,9 @@ def is_bool_dtype(arr_or_dtype):
         # guess this
         return (arr_or_dtype.is_object and
                 arr_or_dtype.inferred_type == 'boolean')
+    elif is_extension_array_dtype(arr_or_dtype):
+        dtype = getattr(arr_or_dtype, 'dtype', arr_or_dtype)
+        return dtype._is_boolean
 
     return issubclass(tipo, np.bool_)
 

pandas/core/dtypes/dtypes.py

@@ -462,6 +462,12 @@ def ordered(self):
         """Whether the categories have an ordered relationship"""
         return self._ordered
 
+    @property
+    def _is_boolean(self):
+        from pandas.core.dtypes.common import is_bool_dtype
+
+        return is_bool_dtype(self.categories)
+
 
 class DatetimeTZDtypeType(type):
     """

pandas/core/frame.py

@@ -629,8 +629,8 @@ def _is_homogeneous(self):
         >>> DataFrame({"A": [1, 2], "B": [3.0, 4.0]})._is_homogeneous
         False
 
-        Items with the type but different sizes are considered different
-        types.
+        Items with the same type but different sizes are considered
+        different types.
 
         >>> DataFrame({"A": np.array([1, 2], dtype=np.int32),
         ...            "B": np.array([1, 2], dtype=np.int64)})._is_homogeneous

pandas/core/generic.py

@@ -9671,15 +9671,15 @@ def _doc_parms(cls):
       original index.
     * None : reduce all axes, return a scalar.
 
+bool_only : boolean, default None
+    Include only boolean columns. If None, will attempt to use everything,
+    then use only boolean data. Not implemented for Series.
 skipna : boolean, default True
     Exclude NA/null values. If an entire row/column is NA, the result
     will be NA.
 level : int or level name, default None
     If the axis is a MultiIndex (hierarchical), count along a
     particular level, collapsing into a %(name1)s.
-bool_only : boolean, default None
-    Include only boolean columns. If None, will attempt to use everything,
-    then use only boolean data. Not implemented for Series.
 **kwargs : any, default None
     Additional keywords have no effect but might be accepted for
     compatibility with NumPy.

pandas/core/indexes/multi.py

@@ -306,7 +306,7 @@ def _is_homogeneous(self):
         >>> MultiIndex.from_tuples([('a', 1), ('a', 2)])._is_homogeneous
         False
         """
-        return len(set(x.dtype for x in self.levels)) <= 1
+        return len({x.dtype for x in self.levels}) <= 1
 
     def _set_levels(self, levels, level=None, copy=False, validate=True,
                     verify_integrity=False):

pandas/tests/arrays/categorical/test_constructors.py

@@ -42,6 +42,12 @@ def test_constructor_empty(self):
         expected = pd.Int64Index([1, 2, 3])
         tm.assert_index_equal(c.categories, expected)
 
+    def test_constructor_empty_boolean(self):
+        # see gh-22702
+        cat = pd.Categorical([], categories=[True, False])
+        categories = sorted(cat.categories.tolist())
+        assert categories == [False, True]
+
     def test_constructor_tuples(self):
         values = np.array([(1,), (1, 2), (1,), (1, 2)], dtype=object)
         result = Categorical(values)

pandas/tests/arrays/categorical/test_indexing.py

@@ -5,7 +5,8 @@
 import numpy as np
 
 import pandas.util.testing as tm
-from pandas import Categorical, Index, CategoricalIndex, PeriodIndex
+from pandas import Categorical, Index, CategoricalIndex, PeriodIndex, Series
+import pandas.core.common as com
 from pandas.tests.arrays.categorical.common import TestCategorical
 
 
@@ -121,3 +122,27 @@ def test_get_indexer_non_unique(self, idx_values, key_values, key_class):
 
             tm.assert_numpy_array_equal(expected, result)
             tm.assert_numpy_array_equal(exp_miss, res_miss)
+
+
+@pytest.mark.parametrize("index", [True, False])
+def test_mask_with_boolean(index):
+    s = Series(range(3))
+    idx = Categorical([True, False, True])
+    if index:
+        idx = CategoricalIndex(idx)
+
+    assert com.is_bool_indexer(idx)
+    result = s[idx]
+    expected = s[idx.astype('object')]
+    tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("index", [True, False])
+def test_mask_with_boolean_raises(index):
+    s = Series(range(3))
+    idx = Categorical([True, False, None])
+    if index:
+        idx = CategoricalIndex(idx)
+
+    with tm.assert_raises_regex(ValueError, 'NA / NaN'):
+        s[idx]

pandas/tests/dtypes/test_dtypes.py

@@ -17,7 +17,7 @@
     is_dtype_equal, is_datetime64_ns_dtype,
     is_datetime64_dtype, is_interval_dtype,
     is_datetime64_any_dtype, is_string_dtype,
-    _coerce_to_dtype)
+    _coerce_to_dtype, is_bool_dtype)
 import pandas.util.testing as tm
 
 
@@ -126,6 +126,18 @@ def test_tuple_categories(self):
         result = CategoricalDtype(categories)
         assert all(result.categories == categories)
 
+    @pytest.mark.parametrize("categories, expected", [
+        ([True, False], True),
+        ([True, False, None], True),
+        ([True, False, "a", "b'"], False),
+        ([0, 1], False),
+    ])
+    def test_is_boolean(self, categories, expected):
+        cat = Categorical(categories)
+        assert cat.dtype._is_boolean is expected
+        assert is_bool_dtype(cat) is expected
+        assert is_bool_dtype(cat.dtype) is expected
+
 
 class TestDatetimeTZDtype(Base):