Merge pull request #174 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
+ci:
+    autofix_prs: false
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.0

.travis.yml

@@ -35,16 +35,9 @@ matrix:
   fast_finish: true
 
   include:
-    - arch: arm64
+    - arch: arm64-graviton2
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
-  allow_failures:
-    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
+        - JOB="3.7, arm64" PYTEST_WORKERS="auto" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 before_install:
   - echo "before_install"

asv_bench/benchmarks/groupby.py

@@ -480,7 +480,19 @@ class GroupByCythonAgg:
     param_names = ["dtype", "method"]
     params = [
         ["float64"],
-        ["sum", "prod", "min", "max", "mean", "median", "var", "first", "last"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
     ]
 
     def setup(self, dtype, method):

doc/_templates/sidebar-nav-bs.html

@@ -0,0 +1,9 @@
+<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+  <div class="bd-toc-item active">
+    {% if pagename.startswith("reference") %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=True, includehidden=True, titles_only=True) }}
+    {% else %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=False, includehidden=True, titles_only=True) }}
+    {% endif %}
+  </div>
+</nav>

doc/source/_static/style/hbetw_axNone.png

@@ -23,7 +23,7 @@ Styler properties
    :toctree: api/
 
    Styler.env
-   Styler.template
+   Styler.template_html
    Styler.loader
 
 Style application
@@ -53,6 +53,7 @@ Builtin styles
    Styler.highlight_null
    Styler.highlight_max
    Styler.highlight_min
+   Styler.highlight_between
    Styler.background_gradient
    Styler.bar
 

doc/source/_static/style/hbetw_basic.png

@@ -498,7 +498,7 @@ values across a level. For instance:
    )
    df = pd.DataFrame(np.random.randn(4, 2), index=midx)
    df
-   df2 = df.mean(level=0)
+   df2 = df.groupby(level=0).mean()
    df2
    df2.reindex(df.index, level=0)
 

doc/source/_static/style/hbetw_props.png

@@ -633,7 +633,7 @@ even if some categories are not present in the data:
         data=[[1, 2, 3], [4, 5, 6]],
         columns=pd.MultiIndex.from_arrays([["A", "B", "B"], columns]),
     )
-    df.sum(axis=1, level=1)
+    df.groupby(axis=1, level=1).sum()
 
 Groupby will also show "unused" categories:
 

doc/source/_static/style/hbetw_seq.png

@@ -320,14 +320,6 @@ number:
 
    s.groupby(level="second").sum()
 
-The aggregation functions such as ``sum`` will take the level parameter
-directly. Additionally, the resulting index will be named according to the
-chosen level:
-
-.. ipython:: python
-
-   s.sum(level="second")
-
 Grouping with multiple levels is supported.
 
 .. ipython:: python

doc/source/reference/style.rst

@@ -1710,7 +1710,7 @@
     "            Styler.loader,  # the default\n",
     "        ])\n",
     "    )\n",
-    "    template = env.get_template(\"myhtml.tpl\")"
+    "    template_html = env.get_template(\"myhtml.tpl\")"
    ]
   },
   {

doc/source/user_guide/advanced.rst

@@ -24,6 +24,7 @@ Version 1.2
 .. toctree::
    :maxdepth: 2
 
+   v1.2.5
    v1.2.4
    v1.2.3
    v1.2.2

doc/source/user_guide/categorical.rst

@@ -154,6 +154,7 @@ Other enhancements:
 - ``Series.all`` and ``Series.any`` now support the ``level`` and ``skipna`` parameters (:issue:`8302`):
 
   .. ipython:: python
+     :okwarning:
 
      s = pd.Series([False, True, False], index=[0, 0, 1])
      s.any(level=0)

doc/source/user_guide/groupby.rst

@@ -30,4 +30,4 @@ Fixed regressions
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.2.3..v1.2.4|HEAD
+.. contributors:: v1.2.3..v1.2.4

doc/source/user_guide/style.ipynb

@@ -0,0 +1,48 @@
+.. _whatsnew_125:
+
+What's new in 1.2.5 (May ??, 2021)
+----------------------------------
+
+These are the changes in pandas 1.2.5. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_125.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_125.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_125.other:
+
+Other
+~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_125.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.2.4..v1.2.5|HEAD

doc/source/whatsnew/index.rst

@@ -117,6 +117,9 @@ We provided some focused development on :class:`.Styler`, including altering met
 to accept more universal CSS language for arguments, such as ``'color:red;'`` instead of
 ``[('color', 'red')]`` (:issue:`39564`). This is also added to the built-in methods
 to allow custom CSS highlighting instead of default background coloring (:issue:`40242`).
+Enhancements to other built-in methods include extending the :meth:`.Styler.background_gradient`
+method to shade elements based on a given gradient map and not be restricted only to
+values in the DataFrame (:issue:`39930` :issue:`22727` :issue:`28901`).
 
 The :meth:`.Styler.apply` now consistently allows functions with ``ndarray`` output to
 allow more flexible development of UDFs when ``axis`` is ``None`` ``0`` or ``1`` (:issue:`39393`).
@@ -206,13 +209,17 @@ Other enhancements
 - :meth:`.Styler.background_gradient` now allows the ability to supply a specific gradient map (:issue:`22727`)
 - :meth:`.Styler.clear` now clears :attr:`Styler.hidden_index` and :attr:`Styler.hidden_columns` as well (:issue:`40484`)
 - Builtin highlighting methods in :class:`Styler` have a more consistent signature and css customisability (:issue:`40242`)
+- :meth:`.Styler.highlight_between` added to list of builtin styling methods (:issue:`39821`)
 - :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 - :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
 - Add support for parsing ``ISO 8601``-like timestamps with negative signs to :meth:`pandas.Timedelta` (:issue:`37172`)
 - Add support for unary operators in :class:`FloatingArray` (:issue:`38749`)
 - :class:`RangeIndex` can now be constructed by passing a ``range`` object directly e.g. ``pd.RangeIndex(range(3))`` (:issue:`12067`)
 - :meth:`round` being enabled for the nullable integer and floating dtypes (:issue:`38844`)
 - :meth:`pandas.read_csv` and :meth:`pandas.read_json` expose the argument ``encoding_errors`` to control how encoding errors are handled (:issue:`39450`)
+- :meth:`.GroupBy.any` and :meth:`.GroupBy.all` use Kleene logic with nullable data types (:issue:`37506`)
+- :meth:`.GroupBy.any` and :meth:`.GroupBy.all` return a ``BooleanDtype`` for columns with nullable data types (:issue:`33449`)
+-
 
 .. ---------------------------------------------------------------------------
 
@@ -563,6 +570,7 @@ Deprecations
 - Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like or dict-like and raises anything but ``TypeError``; ``func`` raising anything but a ``TypeError`` will raise in a future version (:issue:`40211`)
 - Deprecated support for ``np.ma.mrecords.MaskedRecords`` in the :class:`DataFrame` constructor, pass ``{name: data[name] for name in data.dtype.names}`` instead (:issue:`40363`)
 - Deprecated the use of ``**kwargs`` in :class:`.ExcelWriter`; use the keyword argument ``engine_kwargs`` instead (:issue:`40430`)
+- Deprecated the ``level`` keyword for :class:`DataFrame` and :class:`Series` aggregations; use groupby instead (:issue:`39983`)
 
 .. ---------------------------------------------------------------------------
 
@@ -782,6 +790,8 @@ Groupby/resample/rolling
 - Bug in :meth:`Series.asfreq` and :meth:`DataFrame.asfreq` dropping rows when the index is not sorted (:issue:`39805`)
 - Bug in aggregation functions for :class:`DataFrame` not respecting ``numeric_only`` argument when ``level`` keyword was given (:issue:`40660`)
 - Bug in :class:`core.window.RollingGroupby` where ``as_index=False`` argument in ``groupby`` was ignored (:issue:`39433`)
+- Bug in :meth:`.GroupBy.any` and :meth:`.GroupBy.all` raising ``ValueError`` when using with nullable type columns holding ``NA`` even with ``skipna=True`` (:issue:`40585`)
+
 
 Reshaping
 ^^^^^^^^^
@@ -818,24 +828,29 @@ ExtensionArray
 - Fixed a bug where some properties of subclasses of :class:`PandasExtensionDtype` where improperly cached (:issue:`40329`)
 -
 
+Styler
+^^^^^^
+
+- Bug in :class:`Styler` where ``subset`` arg in methods raised an error for some valid multiindex slices (:issue:`33562`)
+- :class:`Styler` rendered HTML output minor alterations to support w3 good code standard (:issue:`39626`)
+- Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
+- Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
+- Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
+- Bug in :class:`.Styler` where copying from Jupyter dropped top left cell and misaligned headers (:issue:`12147`)
+- Bug in :class:`.Styler.where` where ``kwargs`` were not passed to the applicable callable (:issue:`40845`)
+- Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`, :issue:`40334`)
+
+
 Other
 ^^^^^
 - Bug in :class:`Index` constructor sometimes silently ignoring a specified ``dtype`` (:issue:`38879`)
 - Bug in :func:`pandas.api.types.infer_dtype` not recognizing Series, Index or array with a period dtype (:issue:`23553`)
 - Bug in :func:`pandas.api.types.infer_dtype` raising an error for general :class:`.ExtensionArray` objects. It will now return ``"unknown-array"`` instead of raising (:issue:`37367`)
 - Bug in constructing a :class:`Series` from a list and a :class:`PandasDtype` (:issue:`39357`)
-- Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`, :issue:`40334`)
 - ``inspect.getmembers(Series)`` no longer raises an ``AbstractMethodError`` (:issue:`38782`)
 - Bug in :meth:`Series.where` with numeric dtype and ``other = None`` not casting to ``nan`` (:issue:`39761`)
 - :meth:`Index.where` behavior now mirrors :meth:`Index.putmask` behavior, i.e. ``index.where(mask, other)`` matches ``index.putmask(~mask, other)`` (:issue:`39412`)
 - Bug in :func:`pandas.testing.assert_series_equal`, :func:`pandas.testing.assert_frame_equal`, :func:`pandas.testing.assert_index_equal` and :func:`pandas.testing.assert_extension_array_equal` incorrectly raising when an attribute has an unrecognized NA type (:issue:`39461`)
-- Bug in :class:`Styler` where ``subset`` arg in methods raised an error for some valid multiindex slices (:issue:`33562`)
-- :class:`Styler` rendered HTML output minor alterations to support w3 good code standard (:issue:`39626`)
-- Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
-- Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
-- Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
-- Bug in :class:`.Styler` where copying from Jupyter dropped top left cell and misaligned headers (:issue:`12147`)
-- Bug in :class:`.Styler.where` where ``kwargs`` were not passed to the applicable callable (:issue:`40845`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
 - Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)

doc/source/whatsnew/v0.15.2.rst

@@ -48,6 +48,8 @@
 
 """
 
+from __future__ import annotations
+
 from collections import namedtuple
 from contextlib import (
     ContextDecorator,
@@ -57,12 +59,7 @@
 from typing import (
     Any,
     Callable,
-    Dict,
     Iterable,
-    List,
-    Optional,
-    Tuple,
-    Type,
     cast,
 )
 import warnings
@@ -73,16 +70,16 @@
 RegisteredOption = namedtuple("RegisteredOption", "key defval doc validator cb")
 
 # holds deprecated option metadata
-_deprecated_options: Dict[str, DeprecatedOption] = {}
+_deprecated_options: dict[str, DeprecatedOption] = {}
 
 # holds registered option metadata
-_registered_options: Dict[str, RegisteredOption] = {}
+_registered_options: dict[str, RegisteredOption] = {}
 
 # holds the current values for registered options
-_global_config: Dict[str, Any] = {}
+_global_config: dict[str, Any] = {}
 
 # keys which have a special meaning
-_reserved_keys: List[str] = ["all"]
+_reserved_keys: list[str] = ["all"]
 
 
 class OptionError(AttributeError, KeyError):
@@ -194,7 +191,7 @@ def get_default_val(pat: str):
 class DictWrapper:
     """ provide attribute-style access to a nested dict"""
 
-    def __init__(self, d: Dict[str, Any], prefix: str = ""):
+    def __init__(self, d: dict[str, Any], prefix: str = ""):
         object.__setattr__(self, "d", d)
         object.__setattr__(self, "prefix", prefix)
 
@@ -428,8 +425,8 @@ def register_option(
     key: str,
     defval: object,
     doc: str = "",
-    validator: Optional[Callable[[Any], Any]] = None,
-    cb: Optional[Callable[[str], Any]] = None,
+    validator: Callable[[Any], Any] | None = None,
+    cb: Callable[[str], Any] | None = None,
 ) -> None:
     """
     Register an option in the package-wide pandas config object
@@ -500,7 +497,7 @@ def register_option(
 
 
 def deprecate_option(
-    key: str, msg: Optional[str] = None, rkey: Optional[str] = None, removal_ver=None
+    key: str, msg: str | None = None, rkey: str | None = None, removal_ver=None
 ) -> None:
     """
     Mark option `key` as deprecated, if code attempts to access this option,
@@ -547,7 +544,7 @@ def deprecate_option(
 # functions internal to the module
 
 
-def _select_options(pat: str) -> List[str]:
+def _select_options(pat: str) -> list[str]:
     """
     returns a list of keys matching `pat`
 
@@ -565,7 +562,7 @@ def _select_options(pat: str) -> List[str]:
     return [k for k in keys if re.search(pat, k, re.I)]
 
 
-def _get_root(key: str) -> Tuple[Dict[str, Any], str]:
+def _get_root(key: str) -> tuple[dict[str, Any], str]:
     path = key.split(".")
     cursor = _global_config
     for p in path[:-1]:
@@ -674,7 +671,7 @@ def pp_options_list(keys: Iterable[str], width=80, _print: bool = False):
     from itertools import groupby
     from textwrap import wrap
 
-    def pp(name: str, ks: Iterable[str]) -> List[str]:
+    def pp(name: str, ks: Iterable[str]) -> list[str]:
         pfx = "- " + name + ".[" if name else ""
         ls = wrap(
             ", ".join(ks),
@@ -687,7 +684,7 @@ def pp(name: str, ks: Iterable[str]) -> List[str]:
             ls[-1] = ls[-1] + "]"
         return ls
 
-    ls: List[str] = []
+    ls: list[str] = []
     singles = [x for x in sorted(keys) if x.find(".") < 0]
     if singles:
         ls += pp("", singles)
@@ -760,7 +757,7 @@ def inner(key: str, *args, **kwds):
 # arg in register_option
 
 
-def is_type_factory(_type: Type[Any]) -> Callable[[Any], None]:
+def is_type_factory(_type: type[Any]) -> Callable[[Any], None]:
     """
 
     Parameters
@@ -826,7 +823,7 @@ def inner(x) -> None:
     return inner
 
 
-def is_nonnegative_int(value: Optional[int]) -> None:
+def is_nonnegative_int(value: int | None) -> None:
     """
     Verify that value is None or a positive int.