Merge branch 'pandas-dev:main' into feature

Author: m-b-gomes

.github/workflows/wheels.yml

@@ -140,7 +140,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

asv_bench/benchmarks/tslibs/fields.py

@@ -19,10 +19,15 @@ class TimeGetTimedeltaField:
     def setup(self, size, field):
         arr = np.random.randint(0, 10, size=size, dtype="i8")
         self.i8data = arr
+        arr = np.random.randint(-86400 * 1_000_000_000, 0, size=size, dtype="i8")
+        self.i8data_negative = arr
 
     def time_get_timedelta_field(self, size, field):
         get_timedelta_field(self.i8data, field)
 
+    def time_get_timedelta_field_negative_td(self, size, field):
+        get_timedelta_field(self.i8data_negative, field)
+
 
 class TimeGetDateField:
     params = [
@@ -72,3 +77,6 @@ def setup(self, size, side, period, freqstr, month_kw):
 
     def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
         get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)
+
+
+from ..pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -74,39 +74,31 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.DataFrame.mean RT03,SA01" \
         -i "pandas.DataFrame.median RT03,SA01" \
         -i "pandas.DataFrame.min RT03" \
-        -i "pandas.DataFrame.plot PR02,SA01" \
+        -i "pandas.DataFrame.plot PR02" \
         -i "pandas.Grouper PR02" \
-        -i "pandas.MultiIndex PR01" \
         -i "pandas.MultiIndex.append PR07,SA01" \
         -i "pandas.MultiIndex.copy PR07,RT03,SA01" \
         -i "pandas.MultiIndex.drop PR07,RT03,SA01" \
-        -i "pandas.MultiIndex.dtypes SA01" \
         -i "pandas.MultiIndex.get_level_values SA01" \
         -i "pandas.MultiIndex.get_loc PR07" \
         -i "pandas.MultiIndex.get_loc_level PR07" \
-        -i "pandas.MultiIndex.levels SA01" \
         -i "pandas.MultiIndex.levshape SA01" \
         -i "pandas.MultiIndex.names SA01" \
         -i "pandas.MultiIndex.nlevels SA01" \
         -i "pandas.MultiIndex.remove_unused_levels RT03,SA01" \
         -i "pandas.MultiIndex.reorder_levels RT03,SA01" \
-        -i "pandas.MultiIndex.set_codes SA01" \
         -i "pandas.MultiIndex.set_levels RT03,SA01" \
         -i "pandas.MultiIndex.sortlevel PR07,SA01" \
         -i "pandas.MultiIndex.to_frame RT03" \
-        -i "pandas.MultiIndex.truncate SA01" \
         -i "pandas.NA SA01" \
         -i "pandas.NaT SA01" \
         -i "pandas.NamedAgg SA01" \
-        -i "pandas.Period SA01" \
         -i "pandas.Period.asfreq SA01" \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.freqstr SA01" \
-        -i "pandas.Period.is_leap_year SA01" \
         -i "pandas.Period.month SA01" \
         -i "pandas.Period.now SA01" \
         -i "pandas.Period.ordinal GL08" \
-        -i "pandas.Period.quarter SA01" \
         -i "pandas.Period.strftime PR01,SA01" \
         -i "pandas.Period.to_timestamp SA01" \
         -i "pandas.Period.year SA01" \
@@ -172,7 +164,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.lt SA01" \
         -i "pandas.Series.ne SA01" \
         -i "pandas.Series.pad PR01,SA01" \
-        -i "pandas.Series.plot PR02,SA01" \
+        -i "pandas.Series.plot PR02" \
         -i "pandas.Series.pop RT03,SA01" \
         -i "pandas.Series.prod RT03" \
         -i "pandas.Series.product RT03" \
@@ -367,7 +359,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.DataFrameGroupBy.nth PR02" \
         -i "pandas.core.groupby.DataFrameGroupBy.nunique SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.ohlc SA01" \
-        -i "pandas.core.groupby.DataFrameGroupBy.plot PR02,SA01" \
+        -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.DataFrameGroupBy.prod SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.sem SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.sum SA01" \
@@ -385,7 +377,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.SeriesGroupBy.min SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.nth PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.ohlc SA01" \
-        -i "pandas.core.groupby.SeriesGroupBy.plot PR02,SA01" \
+        -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.prod SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.sem SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.sum SA01" \
@@ -486,23 +478,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.plotting.autocorrelation_plot RT03,SA01" \
         -i "pandas.plotting.lag_plot RT03,SA01" \
         -i "pandas.plotting.parallel_coordinates PR07,RT03,SA01" \
-        -i "pandas.plotting.plot_params SA01" \
         -i "pandas.plotting.scatter_matrix PR07,SA01" \
         -i "pandas.plotting.table PR07,RT03,SA01" \
         -i "pandas.qcut PR07,SA01" \
-        -i "pandas.read_feather SA01" \
         -i "pandas.read_orc SA01" \
-        -i "pandas.read_sas SA01" \
         -i "pandas.read_spss SA01" \
         -i "pandas.reset_option SA01" \
         -i "pandas.set_eng_float_format RT03,SA01" \
-        -i "pandas.set_option SA01" \
         -i "pandas.show_versions SA01" \
-        -i "pandas.test SA01" \
         -i "pandas.testing.assert_extension_array_equal SA01" \
-        -i "pandas.testing.assert_index_equal PR07,SA01" \
         -i "pandas.testing.assert_series_equal PR07,SA01" \
-        -i "pandas.timedelta_range SA01" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
         -i "pandas.tseries.offsets.BQuarterBegin PR02" \
         -i "pandas.tseries.offsets.BQuarterBegin.freqstr SA01" \
@@ -787,7 +772,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.YearBegin.nanos GL08" \
         -i "pandas.tseries.offsets.YearBegin.normalize GL08" \
         -i "pandas.tseries.offsets.YearBegin.rule_code GL08" \
-        -i "pandas.tseries.offsets.YearEnd PR02" \
         -i "pandas.tseries.offsets.YearEnd.freqstr SA01" \
         -i "pandas.tseries.offsets.YearEnd.is_on_offset GL08" \
         -i "pandas.tseries.offsets.YearEnd.month GL08" \
@@ -796,7 +780,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.YearEnd.normalize GL08" \
         -i "pandas.tseries.offsets.YearEnd.rule_code GL08" \
         -i "pandas.unique PR07" \
-        -i "pandas.util.hash_array PR07,SA01" \
         -i "pandas.util.hash_pandas_object PR07,SA01" # There should be no backslash in the final line, please keep this comment in the last ignored function
 
     RET=$(($RET + $?)) ; echo $MSG "DONE"

doc/source/user_guide/merging.rst

@@ -974,7 +974,7 @@ with optional filling of missing data with ``fill_method``.
 :func:`merge_asof`
 ---------------------
 
-:func:`merge_asof` is similar to an ordered left-join except that mactches are on the
+:func:`merge_asof` is similar to an ordered left-join except that matches are on the
 nearest key rather than equal keys. For each row in the ``left`` :class:`DataFrame`,
 the last row in the ``right`` :class:`DataFrame` are selected where the ``on`` key is less
 than the left's key. Both :class:`DataFrame` must be sorted by the key.

doc/source/user_guide/missing_data.rst

@@ -32,7 +32,7 @@ use :class:`api.typing.NaTType`.
 :class:`NA` for :class:`StringDtype`, :class:`Int64Dtype` (and other bit widths),
 :class:`Float64Dtype` (and other bit widths), :class:`BooleanDtype` and :class:`ArrowDtype`.
 These types will maintain the original data type of the data.
-For typing applications, use :class:`api.types.NAType`.
+For typing applications, use :class:`api.typing.NAType`.
 
 .. ipython:: python
 

doc/source/user_guide/timeseries.rst

@@ -1273,6 +1273,10 @@ frequencies. We will refer to these aliases as *offset aliases*.
    are deprecated in favour of the aliases ``h``, ``bh``, ``cbh``,
    ``min``, ``s``, ``ms``, ``us``, and ``ns``.
 
+   Aliases ``Y``, ``M``, and ``Q`` are deprecated in favour of the aliases
+   ``YE``, ``ME``, ``QE``.
+
+
 .. note::
 
     When using the offset aliases above, it should be noted that functions

doc/source/whatsnew/v3.0.0.rst

@@ -124,6 +124,69 @@ notable_bug_fix2
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. _whatsnew_300.api_breaking.datetime_resolution_inference:
+
+Datetime resolution inference
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Converting a sequence of strings, ``datetime`` objects, or ``np.datetime64`` objects to
+a ``datetime64`` dtype now performs inference on the appropriate resolution (AKA unit) for the output dtype. This affects :class:`Series`, :class:`DataFrame`, :class:`Index`, :class:`DatetimeIndex`, and :func:`to_datetime`.
+
+Previously, these would always give nanosecond resolution:
+
+.. code-block:: ipython
+
+    In [1]: dt = pd.Timestamp("2024-03-22 11:36").to_pydatetime()
+    In [2]: pd.to_datetime([dt]).dtype
+    Out[2]: dtype('<M8[ns]')
+    In [3]: pd.Index([dt]).dtype
+    Out[3]: dtype('<M8[ns]')
+    In [4]: pd.DatetimeIndex([dt]).dtype
+    Out[4]: dtype('<M8[ns]')
+    In [5]: pd.Series([dt]).dtype
+    Out[5]: dtype('<M8[ns]')
+
+This now infers the unit microsecond unit "us" from the pydatetime object, matching the scalar :class:`Timestamp` behavior.
+
+.. ipython:: python
+
+    In [1]: dt = pd.Timestamp("2024-03-22 11:36").to_pydatetime()
+    In [2]: pd.to_datetime([dt]).dtype
+    In [3]: pd.Index([dt]).dtype
+    In [4]: pd.DatetimeIndex([dt]).dtype
+    In [5]: pd.Series([dt]).dtype
+
+Similar when passed a sequence of ``np.datetime64`` objects, the resolution of the passed objects will be retained (or for lower-than-second resolution, second resolution will be used).
+
+When passing strings, the resolution will depend on the precision of the string, again matching the :class:`Timestamp` behavior. Previously:
+
+.. code-block:: ipython
+
+    In [2]: pd.to_datetime(["2024-03-22 11:43:01"]).dtype
+    Out[2]: dtype('<M8[ns]')
+    In [3]: pd.to_datetime(["2024-03-22 11:43:01.002"]).dtype
+    Out[3]: dtype('<M8[ns]')
+    In [4]: pd.to_datetime(["2024-03-22 11:43:01.002003"]).dtype
+    Out[4]: dtype('<M8[ns]')
+    In [5]: pd.to_datetime(["2024-03-22 11:43:01.002003004"]).dtype
+    Out[5]: dtype('<M8[ns]')
+
+The inferred resolution now matches that of the input strings:
+
+.. ipython:: python
+
+    In [2]: pd.to_datetime(["2024-03-22 11:43:01"]).dtype
+    In [3]: pd.to_datetime(["2024-03-22 11:43:01.002"]).dtype
+    In [4]: pd.to_datetime(["2024-03-22 11:43:01.002003"]).dtype
+    In [5]: pd.to_datetime(["2024-03-22 11:43:01.002003004"]).dtype
+
+In cases with mixed-resolution inputs, the highest resolution is used:
+
+.. code-block:: ipython
+
+    In [2]: pd.to_datetime([pd.Timestamp("2024-03-22 11:43:01"), "2024-03-22 11:43:01.002"]).dtype
+    Out[2]: dtype('<M8[ns]')
+
 .. _whatsnew_300.api_breaking.deps:
 
 Increased minimum versions for dependencies

pandas/_config/config.py

@@ -211,6 +211,14 @@ def set_option(*args) -> None:
     TypeError if keyword arguments are provided
     OptionError if no such option exists
 
+    See Also
+    --------
+    get_option : Retrieve the value of the specified option.
+    reset_option : Reset one or more options to their default value.
+    describe_option : Print the description for one or more registered options.
+    option_context : Context manager to temporarily set options in a ``with``
+        statement.
+
     Notes
     -----
     For all available options, please view the :ref:`User Guide <options.available>`

pandas/_libs/include/pandas/vendored/klib/khash_python.h

@@ -156,20 +156,20 @@ KHASH_MAP_INIT_COMPLEX128(complex128, size_t)
 
 // NaN-floats should be in the same equivalency class, see GH 22119
 static inline int floatobject_cmp(PyFloatObject *a, PyFloatObject *b) {
-  return (Py_IS_NAN(PyFloat_AS_DOUBLE(a)) && Py_IS_NAN(PyFloat_AS_DOUBLE(b))) ||
+  return (isnan(PyFloat_AS_DOUBLE(a)) && isnan(PyFloat_AS_DOUBLE(b))) ||
          (PyFloat_AS_DOUBLE(a) == PyFloat_AS_DOUBLE(b));
 }
 
 // NaNs should be in the same equivalency class, see GH 41836
 // PyObject_RichCompareBool for complexobjects has a different behavior
 // needs to be replaced
 static inline int complexobject_cmp(PyComplexObject *a, PyComplexObject *b) {
-  return (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) &&
-          Py_IS_NAN(a->cval.imag) && Py_IS_NAN(b->cval.imag)) ||
-         (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) &&
+  return (isnan(a->cval.real) && isnan(b->cval.real) && isnan(a->cval.imag) &&
+          isnan(b->cval.imag)) ||
+         (isnan(a->cval.real) && isnan(b->cval.real) &&
           a->cval.imag == b->cval.imag) ||
-         (a->cval.real == b->cval.real && Py_IS_NAN(a->cval.imag) &&
-          Py_IS_NAN(b->cval.imag)) ||
+         (a->cval.real == b->cval.real && isnan(a->cval.imag) &&
+          isnan(b->cval.imag)) ||
          (a->cval.real == b->cval.real && a->cval.imag == b->cval.imag);
 }
 
@@ -223,7 +223,7 @@ static inline int pyobject_cmp(PyObject *a, PyObject *b) {
 
 static inline Py_hash_t _Pandas_HashDouble(double val) {
   // Since Python3.10, nan is no longer has hash 0
-  if (Py_IS_NAN(val)) {
+  if (isnan(val)) {
     return 0;
   }
 #if PY_VERSION_HEX < 0x030A0000

pandas/_libs/lib.pyx

@@ -96,16 +96,12 @@ from pandas._libs.missing cimport (
     is_null_datetime64,
     is_null_timedelta64,
 )
-from pandas._libs.tslibs.conversion cimport (
-    _TSObject,
-    convert_to_tsobject,
-)
+from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
     checknull_with_nat,
 )
-from pandas._libs.tslibs.np_datetime cimport NPY_FR_ns
 from pandas._libs.tslibs.offsets cimport is_offset_object
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
@@ -2497,7 +2493,6 @@ def maybe_convert_objects(ndarray[object] objects,
         ndarray[uint8_t] mask
         Seen seen = Seen()
         object val
-        _TSObject tsobj
         float64_t fnan = NaN
 
     if dtype_if_all_nat is not None:
@@ -2604,8 +2599,7 @@ def maybe_convert_objects(ndarray[object] objects,
                 else:
                     seen.datetime_ = True
                     try:
-                        tsobj = convert_to_tsobject(val, None, None, 0, 0)
-                        tsobj.ensure_reso(NPY_FR_ns)
+                        convert_to_tsobject(val, None, None, 0, 0)
                     except OutOfBoundsDatetime:
                         # e.g. test_out_of_s_bounds_datetime64
                         seen.object_ = True

pandas/_libs/tslib.pyx

@@ -63,7 +63,10 @@ from pandas._libs.tslibs.conversion cimport (
     get_datetime64_nanos,
     parse_pydatetime,
 )
-from pandas._libs.tslibs.dtypes cimport npy_unit_to_abbrev
+from pandas._libs.tslibs.dtypes cimport (
+    get_supported_reso,
+    npy_unit_to_abbrev,
+)
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_nat_strings as nat_strings,
@@ -260,7 +263,7 @@ cpdef array_to_datetime(
     bint dayfirst=False,
     bint yearfirst=False,
     bint utc=False,
-    NPY_DATETIMEUNIT creso=NPY_FR_ns,
+    NPY_DATETIMEUNIT creso=NPY_DATETIMEUNIT.NPY_FR_GENERIC,
     str unit_for_numerics=None,
 ):
     """
@@ -288,8 +291,8 @@ cpdef array_to_datetime(
         yearfirst parsing behavior when encountering datetime strings
     utc : bool, default False
         indicator whether the dates should be UTC
-    creso : NPY_DATETIMEUNIT, default NPY_FR_ns
-        Set to NPY_FR_GENERIC to infer a resolution.
+    creso : NPY_DATETIMEUNIT, default NPY_FR_GENERIC
+        If NPY_FR_GENERIC, conduct inference.
     unit_for_numerics : str, default "ns"
 
     Returns
@@ -389,7 +392,7 @@ cpdef array_to_datetime(
                     # GH#32264 np.str_ object
                     val = str(val)
 
-                if parse_today_now(val, &iresult[i], utc, creso):
+                if parse_today_now(val, &iresult[i], utc, creso, infer_reso=infer_reso):
                     # We can't _quite_ dispatch this to convert_str_to_tsobject
                     #  bc there isn't a nice way to pass "utc"
                     item_reso = NPY_DATETIMEUNIT.NPY_FR_us
@@ -533,7 +536,9 @@ def array_to_datetime_with_tz(
         if state.creso_ever_changed:
             # We encountered mismatched resolutions, need to re-parse with
             #  the correct one.
-            return array_to_datetime_with_tz(values, tz=tz, creso=creso)
+            return array_to_datetime_with_tz(
+                values, tz=tz, dayfirst=dayfirst, yearfirst=yearfirst, creso=creso
+            )
         elif creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
             # i.e. we never encountered anything non-NaT, default to "s". This
             # ensures that insert and concat-like operations with NaT

pandas/_libs/tslibs/offsets.pyx

@@ -2579,7 +2579,7 @@ cdef class YearEnd(YearOffset):
 
     YearEnd goes to the next date which is the end of the year.
 
-    Parameters
+    Attributes
     ----------
     n : int, default 1
         The number of years represented.