Merge remote-tracking branch 'upstream/main' into fix-merge-suffixes-61402

Author: Farsidetfs

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.11.12
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -74,7 +74,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.1
+    rev: v3.20.0
     hooks:
     -   id: pyupgrade
         args: [--py310-plus]
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.3
+    rev: v20.1.5
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.0
+    rev: v1.8.1
     hooks:
     - id: meson-fmt
       args: ['--inplace']

asv_bench/asv.conf.json

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "pip+build": [],
-        "Cython": ["3.0"],
+        "Cython": [],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

ci/deps/actions-310-minimum_versions.yaml

@@ -8,7 +8,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-310.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-311-downstream_compat.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-311-numpydev.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - meson=1.2.1
   - meson-python=0.13.1
-  - cython>=0.29.33
+  - cython<4.0.0a0
 
   # test dependencies
   - pytest>=7.3.2

ci/deps/actions-311-pyarrownightly.yaml

@@ -7,7 +7,7 @@ dependencies:
   # build dependencies
   - versioneer
   - meson=1.2.1
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson-python=0.13.1
 
   # test dependencies

ci/deps/actions-311.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-312.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-313-freethreading.yaml

@@ -7,6 +7,7 @@ dependencies:
   # build dependencies
   - setuptools
   - versioneer
+  - cython<4.0.0a0
   - meson=1.8.0
   - meson-python=0.18.0
 
@@ -21,10 +22,8 @@ dependencies:
   # optional dependencies
   - hypothesis>=6.84.0
 
-  # Move Cython to build dependencies, once they release a version that supports freethreading
   - pip:
     # No free-threaded coveragepy (with the C-extension) on conda-forge yet
     - pytest-cov
     - "tzdata>=2022.7"
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-    - "cython"

ci/deps/actions-313.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-pypy-39.yaml

@@ -9,7 +9,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

doc/cheatsheet/Pandas_Cheat_Sheet.pdf

@@ -251,7 +251,7 @@ This option allows you to configure where meson stores your built C extensions,
 Sometimes, it might be useful to compile pandas with debugging symbols, when debugging C extensions.
 Appending ``-Csetup-args="-Ddebug=true"`` will do the trick.
 
-With pip, it is possible to chain together multiple config settings (for example specifying both a build directory
+With pip, it is possible to chain together multiple config settings. For example, specifying both a build directory
 and building with debug symbols would look like
 ``-Cbuilddir="your builddir here" -Csetup-args="-Dbuildtype=debug"``.
 

doc/cheatsheet/Pandas_Cheat_Sheet.pptx

@@ -325,7 +325,7 @@ The ``.loc`` attribute is the primary access method. The following are valid inp
 
 * A single label, e.g. ``5`` or ``'a'`` (Note that ``5`` is interpreted as a *label* of the index. This use is **not** an integer position along the index.).
 * A list or array of labels ``['a', 'b', 'c']``.
-* A slice object with labels ``'a':'f'`` (Note that contrary to usual Python
+* A slice object with labels ``'a':'f'``. Note that contrary to usual Python
   slices, **both** the start and the stop are included, when present in the
   index! See :ref:`Slicing with labels <indexing.slicing_with_labels>`.
 * A boolean array.
@@ -1461,16 +1461,33 @@ Looking up values by index/column labels
 
 Sometimes you want to extract a set of values given a sequence of row labels
 and column labels, this can be achieved by ``pandas.factorize``  and NumPy indexing.
-For instance:
 
-.. ipython:: python
+For heterogeneous column types, we subset columns to avoid unnecessary NumPy conversions:
+
+.. code-block:: python
+
+   def pd_lookup_het(df, row_labels, col_labels):
+      rows = df.index.get_indexer(row_labels)
+      cols = df.columns.get_indexer(col_labels)
+      sub = df.take(np.unique(cols), axis=1)
+      sub = sub.take(np.unique(rows), axis=0)
+      rows = sub.index.get_indexer(row_labels)
+      values = sub.melt()["value"]
+      cols = sub.columns.get_indexer(col_labels)
+      flat_index = rows + cols * len(sub)
+      result = values[flat_index]
+      return result
+
+For homogeneous column types, it is fastest to skip column subsetting and go directly to NumPy:
+
+.. code-block:: python
 
-    df = pd.DataFrame({'col': ["A", "A", "B", "B"],
-                       'A': [80, 23, np.nan, 22],
-                       'B': [80, 55, 76, 67]})
-    df
-    idx, cols = pd.factorize(df['col'])
-    df.reindex(cols, axis=1).to_numpy()[np.arange(len(df)), idx]
+   def pd_lookup_hom(df, row_labels, col_labels):
+       rows = df.index.get_indexer(row_labels)
+       df = df.loc[:, sorted(set(col_labels))]
+       cols = df.columns.get_indexer(col_labels)
+       result = df.to_numpy()[rows, cols]
+       return result
 
 Formerly this could be achieved with the dedicated ``DataFrame.lookup`` method
 which was deprecated in version 1.2.0 and removed in version 2.0.0.

doc/source/development/contributing_environment.rst

@@ -1415,7 +1415,7 @@ of multi-columns indices.
 
 .. note::
    If an ``index_col`` is not specified (e.g. you don't have an index, or wrote it
-   with ``df.to_csv(..., index=False)``, then any ``names`` on the columns index will
+   with ``df.to_csv(..., index=False)``), then any ``names`` on the columns index will
    be *lost*.
 
 .. ipython:: python

doc/source/user_guide/indexing.rst

@@ -40,8 +40,8 @@ and in the Python interpreter.
 
 .. ipython:: python
 
-   'dense : {:0.2f} bytes'.format(df.memory_usage().sum() / 1e3)
-   'sparse: {:0.2f} bytes'.format(sdf.memory_usage().sum() / 1e3)
+   f'dense: {df.memory_usage().sum()} bytes'
+   f'sparse: {sdf.memory_usage().sum()} bytes'
 
 Functionally, their behavior should be nearly
 identical to their dense counterparts.

doc/source/user_guide/io.rst

@@ -2458,7 +2458,7 @@ you can use the ``tz_convert`` method.
 
     For ``pytz`` time zones, it is incorrect to pass a time zone object directly into
     the ``datetime.datetime`` constructor
-    (e.g., ``datetime.datetime(2011, 1, 1, tzinfo=pytz.timezone('US/Eastern'))``.
+    (e.g., ``datetime.datetime(2011, 1, 1, tzinfo=pytz.timezone('US/Eastern'))``).
     Instead, the datetime needs to be localized using the ``localize`` method
     on the ``pytz`` time zone object.