Merge branch 'main' into fix-issue-61221

Author: gsmll

.github/workflows/unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-22.04, ubuntu-24.04-arm]
-        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         pandas_future_infer_string: ["0"]
@@ -188,7 +188,7 @@ jobs:
       matrix:
         # Note: Don't use macOS latest since macos 14 appears to be arm64 only
         os: [macos-13, macos-14, windows-latest]
-        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     name: ${{ format('{0} {1}', matrix.os, matrix.env_file) }}
@@ -316,7 +316,7 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    if: false
     defaults:
       run:
         shell: bash -eou pipefail {0}

.github/workflows/wheels.yml

@@ -153,7 +153,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].2
+        uses: pypa/[email protected].3
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.4
+    rev: v0.11.8
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.0
+    rev: v20.1.3
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.7.2
+    rev: v1.8.0
     hooks:
     - id: meson-fmt
       args: ['--inplace']

Dockerfile

@@ -1,16 +1,20 @@
 FROM python:3.10.8
 WORKDIR /home/pandas
 
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get install -y build-essential bash-completion
+RUN apt-get update && \
+    apt-get --no-install-recommends -y upgrade && \
+    apt-get --no-install-recommends -y install \
+    build-essential \
+    bash-completion \
+    # hdf5 needed for pytables installation
+    libhdf5-dev \
+    # libgles2-mesa needed for pytest-qt
+    libgles2-mesa-dev && \
+    rm -rf /var/lib/apt/lists/*
 
-# hdf5 needed for pytables installation
-# libgles2-mesa needed for pytest-qt
-RUN apt-get install -y libhdf5-dev libgles2-mesa-dev
-
-RUN python -m pip install --upgrade pip
 COPY requirements-dev.txt /tmp
-RUN python -m pip install -r /tmp/requirements-dev.txt
+RUN python -m pip install --no-cache-dir --upgrade pip && \
+    python -m pip install --no-cache-dir -r /tmp/requirements-dev.txt
 RUN git config --global --add safe.directory /home/pandas
 
 ENV SHELL="/bin/bash"

ci/deps/actions-310-minimum_versions.yaml

@@ -26,7 +26,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4=4.12.3
-  - blosc=1.21.3
   - bottleneck=1.3.6
   - fastparquet=2024.2.0
   - fsspec=2024.2.0

ci/deps/actions-310.yaml

@@ -24,7 +24,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.12.3
-  - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
   - fsspec>=2024.2.0
@@ -52,7 +51,7 @@ dependencies:
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
-  - xarray>=2024.1.1, <=2024.9.0
+  - xarray>=2024.1.1
   - xlrd>=2.0.1
   - xlsxwriter>=3.2.0
   - zstandard>=0.22.0

ci/deps/actions-311-downstream_compat.yaml

@@ -25,7 +25,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.12.3
-  - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
   - fsspec>=2024.2.0
@@ -53,7 +52,7 @@ dependencies:
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
-  - xarray>=2024.1.1, <=2024.9.0
+  - xarray>=2024.1.1
   - xlrd>=2.0.1
   - xlsxwriter>=3.2.0
   - zstandard>=0.22.0
@@ -63,14 +62,12 @@ dependencies:
   - cftime
   - dask
   - ipython
-  - geopandas-base
   - seaborn
   - scikit-learn
   - statsmodels
   - coverage
   - pandas-datareader
   - pyyaml
-  - py
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0

ci/deps/actions-311.yaml

@@ -24,7 +24,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.12.3
-  - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
   - fsspec>=2024.2.0
@@ -52,7 +51,7 @@ dependencies:
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
-  - xarray>=2024.1.1, <=2024.9.0
+  - xarray>=2024.1.1
   - xlrd>=2.0.1
   - xlsxwriter>=3.2.0
   - zstandard>=0.22.0

ci/deps/actions-312.yaml

@@ -24,7 +24,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.12.3
-  - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
   - fsspec>=2024.2.0
@@ -52,7 +51,7 @@ dependencies:
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
-  - xarray>=2024.1.1, <=2024.9.0
+  - xarray>=2024.1.1
   - xlrd>=2.0.1
   - xlsxwriter>=3.2.0
   - zstandard>=0.22.0

ci/deps/actions-313.yaml

@@ -0,0 +1,63 @@
+name: pandas-dev-313
+channels:
+  - conda-forge
+dependencies:
+  - python=3.13
+
+  # build dependencies
+  - versioneer
+  - cython>=0.29.33
+  - meson=1.2.1
+  - meson-python=0.13.1
+
+  # test dependencies
+  - pytest>=7.3.2
+  - pytest-cov
+  - pytest-xdist>=3.4.0
+  - pytest-localserver>=0.8.1
+  - pytest-qt>=4.4.0
+  - boto3
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+
+  # optional dependencies
+  - beautifulsoup4>=4.12.3
+  - blosc>=1.21.3
+  - bottleneck>=1.3.6
+  - fastparquet>=2024.2.0
+  - fsspec>=2024.2.0
+  - html5lib>=1.1
+  - hypothesis>=6.84.0
+  - gcsfs>=2024.2.0
+  - jinja2>=3.1.3
+  - lxml>=4.9.2
+  - matplotlib>=3.8.3
+  - numba>=0.59.0
+  - numexpr>=2.9.0
+  - odfpy>=1.4.1
+  - qtpy>=2.3.0
+  - pyqt>=5.15.9
+  - openpyxl>=3.1.2
+  - psycopg2>=2.9.6
+  - pyarrow>=10.0.1
+  - pymysql>=1.1.0
+  - pyreadstat>=1.2.6
+  - pytables>=3.8.0
+  - python-calamine>=0.1.7
+  - pytz>=2023.4
+  - pyxlsb>=1.0.10
+  - s3fs>=2024.2.0
+  - scipy>=1.12.0
+  - sqlalchemy>=2.0.0
+  - tabulate>=0.9.0
+  - xarray>=2024.1.1
+  - xlrd>=2.0.1
+  - xlsxwriter>=3.2.0
+  - zstandard>=0.22.0
+
+  - pip:
+    - adbc-driver-postgresql>=0.10.0
+    - adbc-driver-sqlite>=0.8.0
+    - tzdata>=2022.7

doc/cheatsheet/Pandas_Cheat_Sheet.pdf

@@ -12,7 +12,7 @@ This cheat sheet, originally written by Irv Lustig, [Princeton Consultants](http
 | Pandas_Cheat_Sheet_JA  | Japanese    | <a href="https://github.com/pandas-dev/pandas/blob/main/doc/cheatsheet/Pandas_Cheat_Sheet_JA.pdf" target="_parent"><img src="https://img.shields.io/badge/Open in PDF-%23FF0000.svg?style=flat-square&logo=adobe&logoColor=white"/></a> | <a href="https://github.com/pandas-dev/pandas/blob/main/doc/cheatsheet/Pandas_Cheat_Sheet_JA.pptx" target="_parent"><img  src="https://img.shields.io/badge/Open in PPT-B7472A?style=flat-square&logo=microsoft-powerpoint&logoColor=white"/></a> |
 | Pandas_Cheat_Sheet_FA  | Persian     | <a href="https://github.com/pandas-dev/pandas/blob/main/doc/cheatsheet/Pandas_Cheat_Sheet_FA.pdf" target="_parent"><img src="https://img.shields.io/badge/Open in PDF-%23FF0000.svg?style=flat-square&logo=adobe&logoColor=white"/></a> | <a href="https://github.com/pandas-dev/pandas/blob/main/doc/cheatsheet/Pandas_Cheat_Sheet_FA.pptx" target="_parent"><img  src="https://img.shields.io/badge/Open in PPT-B7472A?style=flat-square&logo=microsoft-powerpoint&logoColor=white"/></a> |
 
-
+The English version has additional material that is not in the versions in other languages.
 
 **Alternative**
 

doc/cheatsheet/Pandas_Cheat_Sheet.pptx

@@ -249,13 +249,15 @@ ul.task-bullet > li > p:first-child {
 
 .tutorial-card .card-header {
   --bs-card-cap-color: var(--pst-color-text-base);
+  color: var(--pst-color-text-base);
   cursor: pointer;
   background-color: var(--pst-color-surface);
   border: 1px solid var(--pst-color-border)
 }
 
 .tutorial-card .card-body {
   background-color: var(--pst-color-on-background);
+  color: var(--pst-color-text-base);
 }
 
 .tutorial-card .badge {

doc/cheatsheet/README.md

@@ -305,7 +305,6 @@ Installable with ``pip install "pandas[hdf5, parquet, feather, spss, excel]"``
 Dependency                                             Minimum Version    pip extra        Notes
 ====================================================== ================== ================ ==========================================================
 `PyTables <https://github.com/PyTables/PyTables>`__    3.8.0              hdf5             HDF5-based reading / writing
-`blosc <https://github.com/Blosc/c-blosc>`__           1.21.3             hdf5             Compression for HDF5; only available on ``conda``
 `zlib <https://github.com/madler/zlib>`__                                 hdf5             Compression for HDF5
 `fastparquet <https://github.com/dask/fastparquet>`__  2024.2.0           -                Parquet reading / writing (pyarrow is default)
 `pyarrow <https://github.com/apache/arrow>`__          10.0.1             parquet, feather Parquet, ORC, and feather reading / writing

doc/source/_static/css/getting_started.css

@@ -11,7 +11,7 @@ This tutorial uses the Titanic data set, stored as CSV. The data
 consists of the following data columns:
 
 -  PassengerId: Id of every passenger.
--  Survived: Indication whether passenger survived. ``0`` for yes and ``1`` for no.
+-  Survived: Indication whether passenger survived. ``0`` for no and ``1`` for yes.
 -  Pclass: One out of the 3 ticket classes: Class ``1``, Class ``2`` and Class ``3``.
 -  Name: Name of passenger.
 -  Sex: Gender of passenger.

doc/source/getting_started/install.rst

@@ -664,6 +664,7 @@ Data type introspection
     api.types.is_datetime64_dtype
     api.types.is_datetime64_ns_dtype
     api.types.is_datetime64tz_dtype
+    api.types.is_dtype_equal
     api.types.is_extension_array_dtype
     api.types.is_float_dtype
     api.types.is_int64_dtype

doc/source/getting_started/intro_tutorials/includes/titanic.rst

@@ -26,7 +26,7 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     text, Local clipboard, :ref:`read_clipboard<io.clipboard>`, :ref:`to_clipboard<io.clipboard>`
     binary,`MS Excel <https://en.wikipedia.org/wiki/Microsoft_Excel>`__ , :ref:`read_excel<io.excel_reader>`, :ref:`to_excel<io.excel_writer>`
     binary,`OpenDocument <http://opendocumentformat.org>`__, :ref:`read_excel<io.ods>`, NA
-    binary,`HDF5 Format <https://support.hdfgroup.org/HDF5/whatishdf5.html>`__, :ref:`read_hdf<io.hdf5>`, :ref:`to_hdf<io.hdf5>`
+    binary,`HDF5 Format <https://support.hdfgroup.org/documentation/hdf5/latest/_intro_h_d_f5.html>`__, :ref:`read_hdf<io.hdf5>`, :ref:`to_hdf<io.hdf5>`
     binary,`Feather Format <https://github.com/wesm/feather>`__, :ref:`read_feather<io.feather>`, :ref:`to_feather<io.feather>`
     binary,`Parquet Format <https://parquet.apache.org/>`__, :ref:`read_parquet<io.parquet>`, :ref:`to_parquet<io.parquet>`
     binary,`ORC Format <https://orc.apache.org/>`__, :ref:`read_orc<io.orc>`, :ref:`to_orc<io.orc>`

doc/source/reference/arrays.rst

@@ -693,6 +693,7 @@ Datetimelike
 - Bug in :meth:`to_datetime` on float array with missing values throwing ``FloatingPointError`` (:issue:`58419`)
 - Bug in :meth:`to_datetime` on float32 df with year, month, day etc. columns leads to precision issues and incorrect result. (:issue:`60506`)
 - Bug in :meth:`to_datetime` reports incorrect index in case of any failure scenario. (:issue:`58298`)
+- Bug in :meth:`to_datetime` with ``format="ISO8601"`` and ``utc=True`` where naive timestamps incorrectly inherited timezone offset from previous timestamps in a series. (:issue:`61389`)
 - Bug in :meth:`to_datetime` wrongly converts when ``arg`` is a ``np.datetime64`` object with unit of ``ps``. (:issue:`60341`)
 - Bug in setting scalar values with mismatched resolution into arrays with non-nanosecond ``datetime64``, ``timedelta64`` or :class:`DatetimeTZDtype` incorrectly truncating those scalars (:issue:`56410`)
 
@@ -710,6 +711,7 @@ Numeric
 ^^^^^^^
 - Bug in :meth:`DataFrame.corr` where numerical precision errors resulted in correlations above ``1.0`` (:issue:`61120`)
 - Bug in :meth:`DataFrame.quantile` where the column type was not preserved when ``numeric_only=True`` with a list-like ``q`` produced an empty result (:issue:`59035`)
+- Bug in :meth:`Series.dot` returning ``object`` dtype for :class:`ArrowDtype` and nullable-dtype data (:issue:`61375`)
 - Bug in ``np.matmul`` with :class:`Index` inputs raising a ``TypeError`` (:issue:`57079`)
 
 Conversion
@@ -767,6 +769,7 @@ I/O
 - Bug in :meth:`DataFrame.to_dict` raises unnecessary ``UserWarning`` when columns are not unique and ``orient='tight'``. (:issue:`58281`)
 - Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`)
 - Bug in :meth:`DataFrame.to_excel` where the :class:`MultiIndex` index with a period level was not a date (:issue:`60099`)
+- Bug in :meth:`DataFrame.to_stata` when exporting a column containing both long strings (Stata strL) and :class:`pd.NA` values (:issue:`23633`)
 - Bug in :meth:`DataFrame.to_stata` when writing :class:`DataFrame` and ``byteorder=`big```. (:issue:`58969`)
 - Bug in :meth:`DataFrame.to_stata` when writing more than 32,000 value labels. (:issue:`60107`)
 - Bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
@@ -794,9 +797,11 @@ Period
 Plotting
 ^^^^^^^^
 - Bug in :meth:`.DataFrameGroupBy.boxplot` failed when there were multiple groupings (:issue:`14701`)
+- Bug in :meth:`DataFrame.plot.bar` when ``subplots`` and ``stacked=True`` are used in conjunction which causes incorrect stacking. (:issue:`61018`)
 - Bug in :meth:`DataFrame.plot.bar` with ``stacked=True`` where labels on stacked bars with zero-height segments were incorrectly positioned at the base instead of the label position of the previous segment (:issue:`59429`)
 - Bug in :meth:`DataFrame.plot.line` raising ``ValueError`` when set both color and a ``dict`` style (:issue:`59461`)
 - Bug in :meth:`DataFrame.plot` that causes a shift to the right when the frequency multiplier is greater than one. (:issue:`57587`)
+- Bug in :meth:`DataFrame.plot` where ``title`` would require extra titles when plotting more than one column per subplot. (:issue:`61019`)
 - Bug in :meth:`Series.plot` preventing a line and bar from being aligned on the same plot (:issue:`61161`)
 - Bug in :meth:`Series.plot` preventing a line and scatter plot from being aligned (:issue:`61005`)
 - Bug in :meth:`Series.plot` with ``kind="pie"`` with :class:`ArrowDtype` (:issue:`59192`)
@@ -805,10 +810,12 @@ Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 - Bug in :meth:`.DataFrameGroupBy.__len__` and :meth:`.SeriesGroupBy.__len__` would raise when the grouping contained NA values and ``dropna=False`` (:issue:`58644`)
 - Bug in :meth:`.DataFrameGroupBy.any` that returned True for groups where all Timedelta values are NaT. (:issue:`59712`)
+- Bug in :meth:`.DataFrameGroupBy.groups` and :meth:`.SeriesGroupBy.groups` would fail when the groups were :class:`Categorical` with an NA value (:issue:`61356`)
 - Bug in :meth:`.DataFrameGroupBy.groups` and :meth:`.SeriesGroupby.groups` that would not respect groupby argument ``dropna`` (:issue:`55919`)
 - Bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`)
 - Bug in :meth:`.DataFrameGroupBy.quantile` when ``interpolation="nearest"`` is inconsistent with :meth:`DataFrame.quantile` (:issue:`47942`)
 - Bug in :meth:`.Resampler.interpolate` on a :class:`DataFrame` with non-uniform sampling and/or indices not aligning with the resulting resampled index would result in wrong interpolation (:issue:`21351`)
+- Bug in :meth:`.Series.rolling` when used with a :class:`.BaseIndexer` subclass and computing min/max (:issue:`46726`)
 - Bug in :meth:`DataFrame.ewm` and :meth:`Series.ewm` when passed ``times`` and aggregation functions other than mean (:issue:`51695`)
 - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` were not keeping the index name when the index had :class:`ArrowDtype` timestamp dtype (:issue:`61222`)
 - Bug in :meth:`DataFrame.resample` changing index type to :class:`MultiIndex` when the dataframe is empty and using an upsample method (:issue:`55572`)
@@ -834,6 +841,7 @@ Reshaping
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when ``sort=False`` (:issue:`54987`, :issue:`55516`)
 - Bug in :meth:`DataFrame.merge` when merging two :class:`DataFrame` on ``intc`` or ``uintc`` types on Windows (:issue:`60091`, :issue:`58713`)
 - Bug in :meth:`DataFrame.pivot_table` incorrectly subaggregating results when called without an ``index`` argument (:issue:`58722`)
+- Bug in :meth:`DataFrame.pivot_table` incorrectly ignoring the ``values`` argument when also supplied to the ``index`` or ``columns`` parameters (:issue:`57876`, :issue:`61292`)
 - Bug in :meth:`DataFrame.stack` with the new implementation where ``ValueError`` is raised when ``level=[]`` (:issue:`60740`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when manipulating empty :class:`DataFrame` with an :class:`ExtentionDtype` (:issue:`59123`)
 - Bug in :meth:`DataFrame.unstack` where when sort is False, in frames with NA columns, unstacking causing errors or improper orders (:issue:`61221`)

doc/source/user_guide/io.rst

@@ -27,7 +27,6 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.12.3
-  - blosc
   - bottleneck>=1.3.6
   - fastparquet>=2024.2.0
   - fsspec>=2024.2.0
@@ -55,7 +54,7 @@ dependencies:
   - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
-  - xarray>=2024.1.1, <=2024.9.0
+  - xarray>=2024.1.1
   - xlrd>=2.0.1
   - xlsxwriter>=3.2.0
   - zstandard>=0.22.0
@@ -83,8 +82,6 @@ dependencies:
 
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
-  - gitdb
-  - google-auth
   - natsort  # DataFrame.sort_values doctest
   - numpydoc
   - pydata-sphinx-theme=0.16

doc/source/whatsnew/v3.0.0.rst

@@ -3,7 +3,7 @@
 __docformat__ = "restructuredtext"
 
 # Let users know if they're missing any of our hard dependencies
-_hard_dependencies = ("numpy", "dateutil")
+_hard_dependencies = ("numpy", "dateutil", "tzdata")
 
 for _dependency in _hard_dependencies:
     try: