Merge branch 'main' into depr-make_block

Author: jbrockmendel

.github/workflows/wheels.yml

@@ -140,7 +140,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.3
+    rev: v0.4.7
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -40,7 +40,7 @@ repos:
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown, cython, c]
@@ -92,7 +92,7 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.4
+    rev: v18.1.5
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include

asv_bench/benchmarks/tslibs/fields.py

@@ -19,10 +19,15 @@ class TimeGetTimedeltaField:
     def setup(self, size, field):
         arr = np.random.randint(0, 10, size=size, dtype="i8")
         self.i8data = arr
+        arr = np.random.randint(-86400 * 1_000_000_000, 0, size=size, dtype="i8")
+        self.i8data_negative = arr
 
     def time_get_timedelta_field(self, size, field):
         get_timedelta_field(self.i8data, field)
 
+    def time_get_timedelta_field_negative_td(self, size, field):
+        get_timedelta_field(self.i8data_negative, field)
+
 
 class TimeGetDateField:
     params = [
@@ -72,3 +77,6 @@ def setup(self, size, side, period, freqstr, month_kw):
 
     def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
         get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)
+
+
+from ..pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -74,39 +74,31 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.DataFrame.mean RT03,SA01" \
         -i "pandas.DataFrame.median RT03,SA01" \
         -i "pandas.DataFrame.min RT03" \
-        -i "pandas.DataFrame.plot PR02,SA01" \
+        -i "pandas.DataFrame.plot PR02" \
         -i "pandas.Grouper PR02" \
-        -i "pandas.MultiIndex PR01" \
         -i "pandas.MultiIndex.append PR07,SA01" \
         -i "pandas.MultiIndex.copy PR07,RT03,SA01" \
         -i "pandas.MultiIndex.drop PR07,RT03,SA01" \
-        -i "pandas.MultiIndex.dtypes SA01" \
         -i "pandas.MultiIndex.get_level_values SA01" \
         -i "pandas.MultiIndex.get_loc PR07" \
         -i "pandas.MultiIndex.get_loc_level PR07" \
-        -i "pandas.MultiIndex.levels SA01" \
         -i "pandas.MultiIndex.levshape SA01" \
         -i "pandas.MultiIndex.names SA01" \
         -i "pandas.MultiIndex.nlevels SA01" \
         -i "pandas.MultiIndex.remove_unused_levels RT03,SA01" \
         -i "pandas.MultiIndex.reorder_levels RT03,SA01" \
-        -i "pandas.MultiIndex.set_codes SA01" \
         -i "pandas.MultiIndex.set_levels RT03,SA01" \
         -i "pandas.MultiIndex.sortlevel PR07,SA01" \
         -i "pandas.MultiIndex.to_frame RT03" \
-        -i "pandas.MultiIndex.truncate SA01" \
         -i "pandas.NA SA01" \
         -i "pandas.NaT SA01" \
         -i "pandas.NamedAgg SA01" \
-        -i "pandas.Period SA01" \
         -i "pandas.Period.asfreq SA01" \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.freqstr SA01" \
-        -i "pandas.Period.is_leap_year SA01" \
         -i "pandas.Period.month SA01" \
         -i "pandas.Period.now SA01" \
         -i "pandas.Period.ordinal GL08" \
-        -i "pandas.Period.quarter SA01" \
         -i "pandas.Period.strftime PR01,SA01" \
         -i "pandas.Period.to_timestamp SA01" \
         -i "pandas.Period.year SA01" \
@@ -172,7 +164,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.lt SA01" \
         -i "pandas.Series.ne SA01" \
         -i "pandas.Series.pad PR01,SA01" \
-        -i "pandas.Series.plot PR02,SA01" \
+        -i "pandas.Series.plot PR02" \
         -i "pandas.Series.pop RT03,SA01" \
         -i "pandas.Series.prod RT03" \
         -i "pandas.Series.product RT03" \
@@ -367,7 +359,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.DataFrameGroupBy.nth PR02" \
         -i "pandas.core.groupby.DataFrameGroupBy.nunique SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.ohlc SA01" \
-        -i "pandas.core.groupby.DataFrameGroupBy.plot PR02,SA01" \
+        -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.DataFrameGroupBy.prod SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.sem SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.sum SA01" \
@@ -385,7 +377,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.SeriesGroupBy.min SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.nth PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.ohlc SA01" \
-        -i "pandas.core.groupby.SeriesGroupBy.plot PR02,SA01" \
+        -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.prod SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.sem SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.sum SA01" \
@@ -478,31 +470,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.merge PR07" \
         -i "pandas.merge_asof PR07,RT03" \
         -i "pandas.merge_ordered PR07" \
-        -i "pandas.option_context SA01" \
         -i "pandas.period_range RT03,SA01" \
         -i "pandas.pivot PR07" \
         -i "pandas.pivot_table PR07" \
         -i "pandas.plotting.andrews_curves RT03,SA01" \
-        -i "pandas.plotting.autocorrelation_plot RT03,SA01" \
         -i "pandas.plotting.lag_plot RT03,SA01" \
-        -i "pandas.plotting.parallel_coordinates PR07,RT03,SA01" \
-        -i "pandas.plotting.plot_params SA01" \
         -i "pandas.plotting.scatter_matrix PR07,SA01" \
-        -i "pandas.plotting.table PR07,RT03,SA01" \
         -i "pandas.qcut PR07,SA01" \
-        -i "pandas.read_feather SA01" \
-        -i "pandas.read_orc SA01" \
-        -i "pandas.read_sas SA01" \
         -i "pandas.read_spss SA01" \
-        -i "pandas.reset_option SA01" \
         -i "pandas.set_eng_float_format RT03,SA01" \
-        -i "pandas.set_option SA01" \
-        -i "pandas.show_versions SA01" \
-        -i "pandas.test SA01" \
         -i "pandas.testing.assert_extension_array_equal SA01" \
-        -i "pandas.testing.assert_index_equal PR07,SA01" \
-        -i "pandas.testing.assert_series_equal PR07,SA01" \
-        -i "pandas.timedelta_range SA01" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
         -i "pandas.tseries.offsets.BQuarterBegin PR02" \
         -i "pandas.tseries.offsets.BQuarterBegin.freqstr SA01" \
@@ -787,16 +764,13 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.YearBegin.nanos GL08" \
         -i "pandas.tseries.offsets.YearBegin.normalize GL08" \
         -i "pandas.tseries.offsets.YearBegin.rule_code GL08" \
-        -i "pandas.tseries.offsets.YearEnd PR02" \
         -i "pandas.tseries.offsets.YearEnd.freqstr SA01" \
         -i "pandas.tseries.offsets.YearEnd.is_on_offset GL08" \
         -i "pandas.tseries.offsets.YearEnd.month GL08" \
         -i "pandas.tseries.offsets.YearEnd.n GL08" \
         -i "pandas.tseries.offsets.YearEnd.nanos GL08" \
         -i "pandas.tseries.offsets.YearEnd.normalize GL08" \
         -i "pandas.tseries.offsets.YearEnd.rule_code GL08" \
-        -i "pandas.unique PR07" \
-        -i "pandas.util.hash_array PR07,SA01" \
         -i "pandas.util.hash_pandas_object PR07,SA01" # There should be no backslash in the final line, please keep this comment in the last ignored function
 
     RET=$(($RET + $?)) ; echo $MSG "DONE"

doc/source/user_guide/merging.rst

@@ -974,7 +974,7 @@ with optional filling of missing data with ``fill_method``.
 :func:`merge_asof`
 ---------------------
 
-:func:`merge_asof` is similar to an ordered left-join except that mactches are on the
+:func:`merge_asof` is similar to an ordered left-join except that matches are on the
 nearest key rather than equal keys. For each row in the ``left`` :class:`DataFrame`,
 the last row in the ``right`` :class:`DataFrame` are selected where the ``on`` key is less
 than the left's key. Both :class:`DataFrame` must be sorted by the key.

doc/source/user_guide/missing_data.rst

@@ -32,7 +32,7 @@ use :class:`api.typing.NaTType`.
 :class:`NA` for :class:`StringDtype`, :class:`Int64Dtype` (and other bit widths),
 :class:`Float64Dtype` (and other bit widths), :class:`BooleanDtype` and :class:`ArrowDtype`.
 These types will maintain the original data type of the data.
-For typing applications, use :class:`api.types.NAType`.
+For typing applications, use :class:`api.typing.NAType`.
 
 .. ipython:: python
 
@@ -337,10 +337,8 @@ When taking the product, NA values or empty data will be treated as 1.
    pd.Series([], dtype="float64").prod()
 
 Cumulative methods like :meth:`~DataFrame.cumsum` and :meth:`~DataFrame.cumprod`
-ignore NA values by default preserve them in the result. This behavior can be changed
-with ``skipna``
-
-* Cumulative methods like :meth:`~DataFrame.cumsum` and :meth:`~DataFrame.cumprod` ignore NA values by default, but preserve them in the resulting arrays. To override this behaviour and include NA values, use ``skipna=False``.
+ignore NA values by default, but preserve them in the resulting array. To override
+this behaviour and include NA values in the calculation, use ``skipna=False``.
 
 
 .. ipython:: python

doc/source/user_guide/timeseries.rst

@@ -1273,6 +1273,10 @@ frequencies. We will refer to these aliases as *offset aliases*.
    are deprecated in favour of the aliases ``h``, ``bh``, ``cbh``,
    ``min``, ``s``, ``ms``, ``us``, and ``ns``.
 
+   Aliases ``Y``, ``M``, and ``Q`` are deprecated in favour of the aliases
+   ``YE``, ``ME``, ``QE``.
+
+
 .. note::
 
     When using the offset aliases above, it should be noted that functions

doc/source/whatsnew/v3.0.0.rst

@@ -44,8 +44,8 @@ Other enhancements
 - :meth:`DataFrame.fillna` and :meth:`Series.fillna` can now accept ``value=None``; for non-object dtype the corresponding NA value will be used (:issue:`57723`)
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
+- Restore support for reading Stata 104-format and enable reading 103-format dta files (:issue:`58554`)
 - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.notable_bug_fixes:
@@ -124,6 +124,69 @@ notable_bug_fix2
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. _whatsnew_300.api_breaking.datetime_resolution_inference:
+
+Datetime resolution inference
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Converting a sequence of strings, ``datetime`` objects, or ``np.datetime64`` objects to
+a ``datetime64`` dtype now performs inference on the appropriate resolution (AKA unit) for the output dtype. This affects :class:`Series`, :class:`DataFrame`, :class:`Index`, :class:`DatetimeIndex`, and :func:`to_datetime`.
+
+Previously, these would always give nanosecond resolution:
+
+.. code-block:: ipython
+
+    In [1]: dt = pd.Timestamp("2024-03-22 11:36").to_pydatetime()
+    In [2]: pd.to_datetime([dt]).dtype
+    Out[2]: dtype('<M8[ns]')
+    In [3]: pd.Index([dt]).dtype
+    Out[3]: dtype('<M8[ns]')
+    In [4]: pd.DatetimeIndex([dt]).dtype
+    Out[4]: dtype('<M8[ns]')
+    In [5]: pd.Series([dt]).dtype
+    Out[5]: dtype('<M8[ns]')
+
+This now infers the unit microsecond unit "us" from the pydatetime object, matching the scalar :class:`Timestamp` behavior.
+
+.. ipython:: python
+
+    In [1]: dt = pd.Timestamp("2024-03-22 11:36").to_pydatetime()
+    In [2]: pd.to_datetime([dt]).dtype
+    In [3]: pd.Index([dt]).dtype
+    In [4]: pd.DatetimeIndex([dt]).dtype
+    In [5]: pd.Series([dt]).dtype
+
+Similar when passed a sequence of ``np.datetime64`` objects, the resolution of the passed objects will be retained (or for lower-than-second resolution, second resolution will be used).
+
+When passing strings, the resolution will depend on the precision of the string, again matching the :class:`Timestamp` behavior. Previously:
+
+.. code-block:: ipython
+
+    In [2]: pd.to_datetime(["2024-03-22 11:43:01"]).dtype
+    Out[2]: dtype('<M8[ns]')
+    In [3]: pd.to_datetime(["2024-03-22 11:43:01.002"]).dtype
+    Out[3]: dtype('<M8[ns]')
+    In [4]: pd.to_datetime(["2024-03-22 11:43:01.002003"]).dtype
+    Out[4]: dtype('<M8[ns]')
+    In [5]: pd.to_datetime(["2024-03-22 11:43:01.002003004"]).dtype
+    Out[5]: dtype('<M8[ns]')
+
+The inferred resolution now matches that of the input strings:
+
+.. ipython:: python
+
+    In [2]: pd.to_datetime(["2024-03-22 11:43:01"]).dtype
+    In [3]: pd.to_datetime(["2024-03-22 11:43:01.002"]).dtype
+    In [4]: pd.to_datetime(["2024-03-22 11:43:01.002003"]).dtype
+    In [5]: pd.to_datetime(["2024-03-22 11:43:01.002003004"]).dtype
+
+In cases with mixed-resolution inputs, the highest resolution is used:
+
+.. code-block:: ipython
+
+    In [2]: pd.to_datetime([pd.Timestamp("2024-03-22 11:43:01"), "2024-03-22 11:43:01.002"]).dtype
+    Out[2]: dtype('<M8[ns]')
+
 .. _whatsnew_300.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -201,6 +264,7 @@ Other Deprecations
 ^^^^^^^^^^^^^^^^^^
 
 - Deprecated :func:`core.internals.api.make_block`, use public APIs instead (:issue:`56815`)
+- Deprecated :meth:`.DataFrameGroupby.corrwith` (:issue:`57158`)
 - Deprecated :meth:`Timestamp.utcfromtimestamp`, use ``Timestamp.fromtimestamp(ts, "UTC")`` instead (:issue:`56680`)
 - Deprecated :meth:`Timestamp.utcnow`, use ``Timestamp.now("UTC")`` instead (:issue:`56680`)
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.all`, :meth:`DataFrame.min`, :meth:`DataFrame.max`, :meth:`DataFrame.sum`, :meth:`DataFrame.prod`, :meth:`DataFrame.mean`, :meth:`DataFrame.median`, :meth:`DataFrame.sem`, :meth:`DataFrame.var`, :meth:`DataFrame.std`, :meth:`DataFrame.skew`, :meth:`DataFrame.kurt`, :meth:`Series.all`,  :meth:`Series.min`, :meth:`Series.max`, :meth:`Series.sum`, :meth:`Series.prod`, :meth:`Series.mean`, :meth:`Series.median`, :meth:`Series.sem`, :meth:`Series.var`, :meth:`Series.std`, :meth:`Series.skew`, and :meth:`Series.kurt`. (:issue:`57087`)
@@ -248,6 +312,7 @@ Removal of prior version deprecations/changes
 - Removed the deprecated ``delim_whitespace`` keyword in :func:`read_csv` and :func:`read_table`, use ``sep=r"\s+"`` instead (:issue:`55569`)
 - Require :meth:`SparseDtype.fill_value` to be a valid value for the :meth:`SparseDtype.subtype` (:issue:`53043`)
 - Stopped automatically casting non-datetimelike values (mainly strings) in :meth:`Series.isin` and :meth:`Index.isin` with ``datetime64``, ``timedelta64``, and :class:`PeriodDtype` dtypes (:issue:`53111`)
+- Stopped performing dtype inference in :class:`Index`, :class:`Series` and :class:`DataFrame` constructors when given a pandas object (:class:`Series`, :class:`Index`, :class:`ExtensionArray`), call ``.infer_objects`` on the input to keep the current behavior (:issue:`56012`)
 - Stopped performing dtype inference when setting a :class:`Index` into a :class:`DataFrame` (:issue:`56102`)
 - Stopped performing dtype inference with in :meth:`Index.insert` with object-dtype index; this often affects the index/columns that result when setting new entries into an empty :class:`Series` or :class:`DataFrame` (:issue:`51363`)
 - Removed the "closed" and "unit" keywords in :meth:`TimedeltaIndex.__new__` (:issue:`52628`, :issue:`55499`)
@@ -440,6 +505,7 @@ Missing
 MultiIndex
 ^^^^^^^^^^
 - :func:`DataFrame.loc` with ``axis=0``  and :class:`MultiIndex` when setting a value adds extra columns (:issue:`58116`)
+- :meth:`DataFrame.melt` would not accept multiple names in ``var_name`` when the columns were a :class:`MultiIndex` (:issue:`58033`)
 -
 
 I/O
@@ -472,7 +538,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Bug in :meth:`DataFrameGroupBy.apply` with ``as_index=False`` that was returning :class:`MultiIndex` instead of returning :class:`Index`. (:issue:`58291`)
 - Bug in :meth:`DataFrameGroupby.transform` and :meth:`SeriesGroupby.transform` with a reducer and ``observed=False`` that coerces dtype to float when there are unobserved categories. (:issue:`55326`)
-
+- Bug in :meth:`Rolling.apply` where the applied function could be called on fewer than ``min_period`` periods if ``method="table"``. (:issue:`58868`)
 
 Reshaping
 ^^^^^^^^^

pandas/_config/config.py

@@ -211,6 +211,14 @@ def set_option(*args) -> None:
     TypeError if keyword arguments are provided
     OptionError if no such option exists
 
+    See Also
+    --------
+    get_option : Retrieve the value of the specified option.
+    reset_option : Reset one or more options to their default value.
+    describe_option : Print the description for one or more registered options.
+    option_context : Context manager to temporarily set options in a ``with``
+        statement.
+
     Notes
     -----
     For all available options, please view the :ref:`User Guide <options.available>`
@@ -315,6 +323,12 @@ def reset_option(pat: str) -> None:
     None
         No return value.
 
+    See Also
+    --------
+    get_option : Retrieve the value of the specified option.
+    set_option : Set the value of the specified option or options.
+    describe_option : Print the description for one or more registered options.
+
     Notes
     -----
     For all available options, please view the
@@ -406,6 +420,13 @@ def option_context(*args) -> Generator[None, None, None]:
     None
         No return value.
 
+    See Also
+    --------
+    get_option : Retrieve the value of the specified option.
+    set_option : Set the value of the specified option.
+    reset_option : Reset one or more options to their default value.
+    describe_option : Print the description for one or more registered options.
+
     Notes
     -----
     For all available options, please view the :ref:`User Guide <options.available>`