Merge remote-tracking branch 'upstream/main' into clean/plotting/testing3

Author: mroeschke

.circleci/config.yml

@@ -1,46 +1,64 @@
 version: 2.1
 
 jobs:
-  test-arm:
+  test-linux-arm:
     machine:
       image: default
     resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-310-arm64.yaml
+      ENV_FILE: ci/deps/circle-311-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: .circleci/setup_env.sh
-      - run: >
-          PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
-          LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
-          ci/run_tests.sh
-  linux-musl:
+      - run:
+          name: Install Environment and Run Tests
+          shell: /bin/bash -exuo pipefail
+          command: |
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MAMBA_URL -O minimamba.sh
+            chmod +x minimamba.sh
+            MAMBA_DIR="$HOME/miniconda3"
+            rm -rf $MAMBA_DIR
+            ./minimamba.sh -b -p $MAMBA_DIR
+            export PATH=$MAMBA_DIR/bin:$PATH
+            conda info -a
+            conda env create -q -n pandas-dev -f $ENV_FILE
+            conda list -n pandas-dev
+            source activate pandas-dev
+            if pip show pandas 1>/dev/null; then
+                pip uninstall -y pandas
+            fi
+            python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
+            PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+            ci/run_tests.sh
+  test-linux-musl:
     docker:
       - image: quay.io/pypa/musllinux_1_1_aarch64
     resource_class: arm.large
     steps:
       # Install pkgs first to have git in the image
       # (needed for checkout)
-      - run: |
-          apk update
-          apk add git
-          apk add musl-locales
+      - run:
+          name: Install System Packages
+          command: |
+            apk update
+            apk add git
+            apk add musl-locales
       - checkout
-      - run: |
-          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
-          . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
-          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
-          python -m pip list --no-cache-dir
-      - run: |
-          . ~/virtualenvs/pandas-dev/bin/activate
-          export PANDAS_CI=1
-          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+      - run:
+          name: Install Environment and Run Tests
+          command: |
+            /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+            . ~/virtualenvs/pandas-dev/bin/activate
+            python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+            python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+            python -m pip list --no-cache-dir
+            export PANDAS_CI=1
+            python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
@@ -71,7 +89,7 @@ jobs:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.15.0
+            pip3 install cibuildwheel==2.18.1
             cibuildwheel --prerelease-pythons --output-dir wheelhouse
 
           environment:
@@ -81,7 +99,7 @@ jobs:
           name: Install Anaconda Client & Upload Wheels
           command: |
             echo "Install Mambaforge"
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
             echo "Downloading $MAMBA_URL"
             wget -q $MAMBA_URL -O minimamba.sh
             chmod +x minimamba.sh
@@ -107,14 +125,14 @@ workflows:
       not:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
-      - test-arm
+      - test-linux-arm
   test-musl:
     # Don't run trigger this one when scheduled pipeline runs
     when:
       not:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
-      - linux-musl
+      - test-linux-musl
   build-wheels:
     jobs:
       - build-aarch64:

.circleci/setup_env.sh

@@ -343,7 +343,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.arrays.NumpyExtensionArray SA01" \
         -i "pandas.arrays.SparseArray PR07,SA01" \
         -i "pandas.arrays.TimedeltaArray PR07,SA01" \
-        -i "pandas.bdate_range RT03,SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.__iter__ RT03,SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.agg RT03" \
         -i "pandas.core.groupby.DataFrameGroupBy.aggregate RT03" \
@@ -404,7 +403,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.window.rolling.Window.std PR01" \
         -i "pandas.core.window.rolling.Window.var PR01" \
         -i "pandas.date_range RT03" \
-        -i "pandas.describe_option SA01" \
         -i "pandas.errors.AbstractMethodError PR01,SA01" \
         -i "pandas.errors.AttributeConflictWarning SA01" \
         -i "pandas.errors.CSSWarning SA01" \
@@ -472,7 +470,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.merge_ordered PR07" \
         -i "pandas.period_range RT03,SA01" \
         -i "pandas.pivot PR07" \
-        -i "pandas.pivot_table PR07" \
         -i "pandas.plotting.andrews_curves RT03,SA01" \
         -i "pandas.plotting.lag_plot RT03,SA01" \
         -i "pandas.plotting.scatter_matrix PR07,SA01" \

ci/code_checks.sh

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python=3.11
 
   # build dependencies
   - versioneer[toml]

ci/deps/circle-310-arm64.yaml renamed to ci/deps/circle-311-arm64.yaml

@@ -134,8 +134,8 @@ to explore, clean, and process your data. In pandas, a data table is called a :c
             <div id="collapseTwo" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-pandas supports the integration with many file formats or data sources out of the box (csv, excel, sql, json, parquet,…). Importing data from each of these
-data sources is provided by function with the prefix ``read_*``. Similarly, the ``to_*`` methods are used to store data.
+pandas supports the integration with many file formats or data sources out of the box (csv, excel, sql, json, parquet,…). The ability to import data from each of these
+data sources is provided by functions with the prefix, ``read_*``. Similarly, the ``to_*`` methods are used to store data.
 
 .. image:: ../_static/schemas/02_io_readwrite.svg
    :align: center
@@ -181,7 +181,7 @@ data sources is provided by function with the prefix ``read_*``. Similarly, the
             <div id="collapseThree" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Selecting or filtering specific rows and/or columns? Filtering the data on a condition? Methods for slicing, selecting, and extracting the
+Selecting or filtering specific rows and/or columns? Filtering the data on a particular condition? Methods for slicing, selecting, and extracting the
 data you need are available in pandas.
 
 .. image:: ../_static/schemas/03_subset_columns_rows.svg
@@ -228,7 +228,7 @@ data you need are available in pandas.
             <div id="collapseFour" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-pandas provides plotting your data out of the box, using the power of Matplotlib. You can pick the plot type (scatter, bar, boxplot,...)
+pandas provides plotting for your data right out of the box with the power of Matplotlib. Simply pick the plot type (scatter, bar, boxplot,...)
 corresponding to your data.
 
 .. image:: ../_static/schemas/04_plot_overview.svg
@@ -275,7 +275,7 @@ corresponding to your data.
             <div id="collapseFive" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-There is no need to loop over all rows of your data table to do calculations. Data manipulations on a column work elementwise.
+There's no need to loop over all rows of your data table to do calculations. Column data manipulations work elementwise in pandas.
 Adding a column to a :class:`DataFrame` based on existing data in other columns is straightforward.
 
 .. image:: ../_static/schemas/05_newcolumn_2.svg
@@ -322,7 +322,7 @@ Adding a column to a :class:`DataFrame` based on existing data in other columns
             <div id="collapseSix" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Basic statistics (mean, median, min, max, counts...) are easily calculable. These or custom aggregations can be applied on the entire
+Basic statistics (mean, median, min, max, counts...) are easily calculable across data frames. These, or even custom aggregations, can be applied on the entire
 data set, a sliding window of the data, or grouped by categories. The latter is also known as the split-apply-combine approach.
 
 .. image:: ../_static/schemas/06_groupby.svg
@@ -369,8 +369,8 @@ data set, a sliding window of the data, or grouped by categories. The latter is
             <div id="collapseSeven" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Change the structure of your data table in multiple ways. You can :func:`~pandas.melt` your data table from wide to long/tidy form or :func:`~pandas.pivot`
-from long to wide format. With aggregations built-in, a pivot table is created with a single command.
+Change the structure of your data table in a variety of ways. You can use :func:`~pandas.melt` to reshape your data from a wide format to a long and tidy one. Use :func:`~pandas.pivot`
+ to go from long to wide format. With aggregations built-in, a pivot table can be created with a single command.
 
 .. image:: ../_static/schemas/07_melt.svg
    :align: center
@@ -416,7 +416,7 @@ from long to wide format. With aggregations built-in, a pivot table is created w
             <div id="collapseEight" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Multiple tables can be concatenated both column wise and row wise as database-like join/merge operations are provided to combine multiple tables of data.
+Multiple tables can be concatenated column wise or row wise with pandas' database-like join and merge operations.
 
 .. image:: ../_static/schemas/08_concat_row.svg
    :align: center
@@ -505,7 +505,7 @@ pandas has great support for time series and has an extensive set of tools for w
             <div id="collapseTen" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Data sets do not only contain numerical data. pandas provides a wide range of functions to clean textual data and extract useful information from it.
+Data sets often contain more than just numerical data. pandas provides a wide range of functions to clean textual data and extract useful information from it.
 
 .. raw:: html
 
@@ -551,9 +551,9 @@ the pandas-equivalent operations compared to software you already know:
         :class-card: comparison-card
         :shadow: md
 
-        The `R programming language <https://www.r-project.org/>`__ provides the
-        ``data.frame`` data structure and multiple packages, such as
-        `tidyverse <https://www.tidyverse.org>`__ use and extend ``data.frame``
+        The `R programming language <https://www.r-project.org/>`__ provides a
+        ``data.frame`` data structure as well as packages like
+        `tidyverse <https://www.tidyverse.org>`__ which use and extend ``data.frame``
         for convenient data handling functionalities similar to pandas.
 
         +++
@@ -572,8 +572,8 @@ the pandas-equivalent operations compared to software you already know:
         :class-card: comparison-card
         :shadow: md
 
-        Already familiar to ``SELECT``, ``GROUP BY``, ``JOIN``, etc.?
-        Most of these SQL manipulations do have equivalents in pandas.
+        Already familiar with ``SELECT``, ``GROUP BY``, ``JOIN``, etc.?
+        Many SQL manipulations have equivalents in pandas.
 
         +++
 
@@ -631,10 +631,10 @@ the pandas-equivalent operations compared to software you already know:
         :class-card: comparison-card
         :shadow: md
 
-        The `SAS <https://en.wikipedia.org/wiki/SAS_(software)>`__ statistical software suite
-        also provides the ``data set`` corresponding to the pandas ``DataFrame``.
-        Also SAS vectorized operations, filtering, string processing operations,
-        and more have similar functions in pandas.
+        `SAS <https://en.wikipedia.org/wiki/SAS_(software)>`__, the statistical software suite,
+        uses the ``data set`` structure, which closely corresponds pandas' ``DataFrame``.
+        Also SAS vectorized operations such as filtering or string processing operations
+        have similar functions in pandas.
 
         +++
 

doc/source/getting_started/index.rst

@@ -263,6 +263,8 @@ starting with 3.0, so it can be safely removed from your code.
 Other Deprecations
 ^^^^^^^^^^^^^^^^^^
 
+- Deprecated :func:`core.internals.api.make_block`, use public APIs instead (:issue:`56815`)
+- Deprecated :meth:`.DataFrameGroupby.corrwith` (:issue:`57158`)
 - Deprecated :meth:`Timestamp.utcfromtimestamp`, use ``Timestamp.fromtimestamp(ts, "UTC")`` instead (:issue:`56680`)
 - Deprecated :meth:`Timestamp.utcnow`, use ``Timestamp.now("UTC")`` instead (:issue:`56680`)
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.all`, :meth:`DataFrame.min`, :meth:`DataFrame.max`, :meth:`DataFrame.sum`, :meth:`DataFrame.prod`, :meth:`DataFrame.mean`, :meth:`DataFrame.median`, :meth:`DataFrame.sem`, :meth:`DataFrame.var`, :meth:`DataFrame.std`, :meth:`DataFrame.skew`, :meth:`DataFrame.kurt`, :meth:`Series.all`,  :meth:`Series.min`, :meth:`Series.max`, :meth:`Series.sum`, :meth:`Series.prod`, :meth:`Series.mean`, :meth:`Series.median`, :meth:`Series.sem`, :meth:`Series.var`, :meth:`Series.std`, :meth:`Series.skew`, and :meth:`Series.kurt`. (:issue:`57087`)
@@ -271,7 +273,6 @@ Other Deprecations
 - Deprecated behavior of :meth:`Series.dt.to_pytimedelta`, in a future version this will return a :class:`Series` containing python ``datetime.timedelta`` objects instead of an ``ndarray`` of timedelta; this matches the behavior of other :meth:`Series.dt` properties. (:issue:`57463`)
 - Deprecated parameter ``method`` in :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like` (:issue:`58667`)
 - Deprecated using ``epoch`` date format in :meth:`DataFrame.to_json` and :meth:`Series.to_json`, use ``iso`` instead. (:issue:`57063`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.prior_deprecations:
@@ -534,6 +535,7 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.quantile` when ``interpolation="nearest"`` is inconsistent with :meth:`DataFrame.quantile` (:issue:`47942`)
 - Bug in :meth:`.Resampler.interpolate` on a :class:`DataFrame` with non-uniform sampling and/or indices not aligning with the resulting resampled index would result in wrong interpolation (:issue:`21351`)
 - Bug in :meth:`DataFrame.ewm` and :meth:`Series.ewm` when passed ``times`` and aggregation functions other than mean (:issue:`51695`)
+- Bug in :meth:`DataFrameGroupBy.agg` that raises ``AttributeError`` when there is dictionary input and duplicated columns, instead of returning a DataFrame with the aggregation of all duplicate columns. (:issue:`55041`)
 - Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Bug in :meth:`DataFrameGroupBy.apply` with ``as_index=False`` that was returning :class:`MultiIndex` instead of returning :class:`Index`. (:issue:`58291`)
 - Bug in :meth:`DataFrameGroupby.transform` and :meth:`SeriesGroupby.transform` with a reducer and ``observed=False`` that coerces dtype to float when there are unobserved categories. (:issue:`55326`)

doc/source/whatsnew/v3.0.0.rst

@@ -279,6 +279,12 @@ def describe_option(pat: str = "", _print_desc: bool = True) -> str | None:
     str
         If the description(s) as a string if ``_print_desc=False``.
 
+    See Also
+    --------
+    get_option : Retrieve the value of the specified option.
+    set_option : Set the value of the specified option or options.
+    reset_option : Reset one or more options to their default value.
+
     Notes
     -----
     For all available options, please view the

pandas/_config/config.py

@@ -150,6 +150,7 @@ def pytest_collection_modifyitems(items, config) -> None:
         ("is_categorical_dtype", "is_categorical_dtype is deprecated"),
         ("is_sparse", "is_sparse is deprecated"),
         ("DataFrameGroupBy.fillna", "DataFrameGroupBy.fillna is deprecated"),
+        ("DataFrameGroupBy.corrwith", "DataFrameGroupBy.corrwith is deprecated"),
         ("NDFrame.replace", "Series.replace without 'value'"),
         ("NDFrame.clip", "Downcasting behavior in Series and DataFrame methods"),
         ("Series.idxmin", "The behavior of Series.idxmin"),