Merge branch 'master' into groupby-mean-datetimelike

Author: AlexeyGy

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
 - [ ] closes #xxxx
 - [ ] tests added / passed
-- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing.html#code-standards) for how to run them
+- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit) for how to run them
 - [ ] whatsnew entry

.github/workflows/ci.yml

@@ -168,6 +168,7 @@ jobs:
         PANDAS_DATA_MANAGER: array
         PATTERN: ${{ matrix.pattern }}
         PYTEST_WORKERS: "auto"
+        PYTEST_TARGET: pandas
       run: |
         source activate pandas-dev
         ci/run_tests.sh

.github/workflows/posix.yml

@@ -44,6 +44,7 @@ jobs:
       LC_ALL: ${{ matrix.settings[4] }}
       PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
       TEST_ARGS: ${{ matrix.settings[6] }}
+      PYTEST_TARGET:  pandas
     concurrency:
       group: ${{ github.ref }}-${{ matrix.settings[0] }}
       cancel-in-progress: ${{github.event_name == 'pull_request'}}

.github/workflows/python-dev.yml

@@ -17,6 +17,7 @@ env:
   PANDAS_CI: 1
   PATTERN: "not slow and not network and not clipboard"
   COVERAGE: true
+  PYTEST_TARGET:  pandas
 
 jobs:
   build:

asv_bench/benchmarks/sparse.py

@@ -91,6 +91,20 @@ def time_sparse_series_to_coo_single_level(self, sort_labels):
         self.ss_two_lvl.sparse.to_coo(sort_labels=sort_labels)
 
 
+class ToCooFrame:
+    def setup(self):
+        N = 10000
+        k = 10
+        arr = np.full((N, k), np.nan)
+        arr[0, 0] = 3.0
+        arr[12, 7] = -1.0
+        arr[0, 9] = 11.2
+        self.df = pd.DataFrame(arr, dtype=pd.SparseDtype("float"))
+
+    def time_to_coo(self):
+        self.df.sparse.to_coo()
+
+
 class Arithmetic:
 
     params = ([0.1, 0.01], [0, np.nan])

azure-pipelines.yml

@@ -17,6 +17,7 @@ pr:
 
 variables:
   PYTEST_WORKERS: auto
+  PYTEST_TARGET:  pandas
 
 jobs:
 # Mac and Linux use the same template

ci/azure/windows.yml

@@ -8,17 +8,33 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py38_np18:
+      py38_np18_1:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
         PATTERN: "not slow and not network"
         PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[a-i]*"
 
-      py39:
+      py38_np18_2:
+        ENV_FILE: ci/deps/azure-windows-38.yaml
+        CONDA_PY: "38"
+        PATTERN: "not slow and not network"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[j-z]*"
+
+      py39_1:
+        ENV_FILE: ci/deps/azure-windows-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "not slow and not network and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[a-i]*"
+
+      py39_2:
         ENV_FILE: ci/deps/azure-windows-39.yaml
         CONDA_PY: "39"
         PATTERN: "not slow and not network and not high_memory"
         PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[j-z]*"
 
   steps:
     - powershell: |
@@ -39,6 +55,7 @@ jobs:
       displayName: 'Build'
     - bash: |
         source activate pandas-dev
+        wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed
         ci/run_tests.sh
       displayName: 'Test'
     - task: PublishTestResults@2

ci/run_tests.sh

@@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas"
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     # GH#37455 windows py38 build appears to be running out of memory

doc/source/development/contributing.rst

@@ -331,7 +331,12 @@ can comment::
 
     @github-actions pre-commit
 
-on that pull request. This will trigger a workflow which will autofix formatting errors.
+on that pull request. This will trigger a workflow which will autofix formatting
+errors.
+
+To automatically fix formatting errors on each commit you make, you can
+set up pre-commit yourself. First, create a Python :ref:`environment
+<contributing_environment>` and then set up :ref:`pre-commit <contributing.pre-commit>`.
 
 Delete your merged branch (optional)
 ------------------------------------

doc/source/development/contributing_environment.rst

@@ -133,7 +133,6 @@ compiler installation instructions.
 
 Let us know if you have any difficulties by opening an issue or reaching out on `Gitter <https://gitter.im/pydata/pandas/>`_.
 
-
 Creating a Python environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/ecosystem.rst

@@ -575,3 +575,17 @@ Library            Accessor     Classes                              Description
 .. _composeml: https://github.com/alteryx/compose
 .. _datatest: https://datatest.readthedocs.io/
 .. _woodwork: https://github.com/alteryx/woodwork
+
+Development tools
+----------------------------
+
+`pandas-stubs <https://github.com/VirtusLab/pandas-stubs>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+While pandas repository is partially typed, the package itself doesn't expose this information for external use.
+Install pandas-stubs to enable basic type coverage of pandas API.
+
+Learn more by reading through these issues `14468 <https://github.com/pandas-dev/pandas/issues/14468>`_,
+`26766 <https://github.com/pandas-dev/pandas/issues/26766>`_, `28142 <https://github.com/pandas-dev/pandas/issues/28142>`_.
+
+See installation and usage instructions on the `github page <https://github.com/VirtusLab/pandas-stubs>`__.

doc/source/user_guide/io.rst

@@ -1208,6 +1208,10 @@ Returning Series
 Using the ``squeeze`` keyword, the parser will return output with a single column
 as a ``Series``:
 
+.. deprecated:: 1.4.0
+   Users should append ``.squeeze("columns")`` to the DataFrame returned by
+   ``read_csv`` instead.
+
 .. ipython:: python
    :suppress:
 
@@ -1217,6 +1221,7 @@ as a ``Series``:
        fh.write(data)
 
 .. ipython:: python
+   :okwarning:
 
    print(open("tmp.csv").read())
 

doc/source/user_guide/options.rst

@@ -38,11 +38,11 @@ and so passing in a substring will work - as long as it is unambiguous:
 
 .. ipython:: python
 
-   pd.get_option("display.max_rows")
-   pd.set_option("display.max_rows", 101)
-   pd.get_option("display.max_rows")
-   pd.set_option("max_r", 102)
-   pd.get_option("display.max_rows")
+   pd.get_option("display.chop_threshold")
+   pd.set_option("display.chop_threshold", 2)
+   pd.get_option("display.chop_threshold")
+   pd.set_option("chop", 4)
+   pd.get_option("display.chop_threshold")
 
 
 The following will **not work** because it matches multiple option names, e.g.
@@ -52,7 +52,7 @@ The following will **not work** because it matches multiple option names, e.g.
    :okexcept:
 
    try:
-       pd.get_option("column")
+       pd.get_option("max")
    except KeyError as e:
        print(e)
 
@@ -153,27 +153,27 @@ lines are replaced by an ellipsis.
 .. ipython:: python
 
    df = pd.DataFrame(np.random.randn(7, 2))
-   pd.set_option("max_rows", 7)
+   pd.set_option("display.max_rows", 7)
    df
-   pd.set_option("max_rows", 5)
+   pd.set_option("display.max_rows", 5)
    df
-   pd.reset_option("max_rows")
+   pd.reset_option("display.max_rows")
 
 Once the ``display.max_rows`` is exceeded, the ``display.min_rows`` options
 determines how many rows are shown in the truncated repr.
 
 .. ipython:: python
 
-   pd.set_option("max_rows", 8)
-   pd.set_option("min_rows", 4)
+   pd.set_option("display.max_rows", 8)
+   pd.set_option("display.min_rows", 4)
    # below max_rows -> all rows shown
    df = pd.DataFrame(np.random.randn(7, 2))
    df
    # above max_rows -> only min_rows (4) rows shown
    df = pd.DataFrame(np.random.randn(9, 2))
    df
-   pd.reset_option("max_rows")
-   pd.reset_option("min_rows")
+   pd.reset_option("display.max_rows")
+   pd.reset_option("display.min_rows")
 
 ``display.expand_frame_repr`` allows for the representation of
 dataframes to stretch across pages, wrapped over the full column vs row-wise.
@@ -193,13 +193,13 @@ dataframes to stretch across pages, wrapped over the full column vs row-wise.
 .. ipython:: python
 
    df = pd.DataFrame(np.random.randn(10, 10))
-   pd.set_option("max_rows", 5)
+   pd.set_option("display.max_rows", 5)
    pd.set_option("large_repr", "truncate")
    df
    pd.set_option("large_repr", "info")
    df
    pd.reset_option("large_repr")
-   pd.reset_option("max_rows")
+   pd.reset_option("display.max_rows")
 
 ``display.max_colwidth`` sets the maximum width of columns.  Cells
 of this length or longer will be truncated with an ellipsis.
@@ -491,6 +491,10 @@ styler.render.repr                      html         Standard output format for
                                                      Should be one of "html" or "latex".
 styler.render.max_elements              262144       Maximum number of datapoints that Styler will render
                                                      trimming either rows, columns or both to fit.
+styler.render.max_rows                  None         Maximum number of rows that Styler will render. By default
+                                                     this is dynamic based on ``max_elements``.
+styler.render.max_columns               None         Maximum number of columns that Styler will render. By default
+                                                     this is dynamic based on ``max_elements``.
 styler.render.encoding                  utf-8        Default encoding for output HTML or LaTeX files.
 styler.format.formatter                 None         Object to specify formatting functions to ``Styler.format``.
 styler.format.na_rep                    None         String representation for missing data.

doc/source/whatsnew/v1.3.3.rst

@@ -24,8 +24,13 @@ Fixed regressions
 - Fixed regression in :meth:`read_parquet` where the ``fastparquet`` engine would not work properly with fastparquet 0.7.0 (:issue:`43075`)
 - Fixed regression in :meth:`DataFrame.loc.__setitem__` raising ``ValueError`` when setting array as cell value (:issue:`43422`)
 - Fixed regression in :func:`is_list_like` where objects with ``__iter__`` set to ``None`` would be identified as iterable (:issue:`43373`)
+- Fixed regression in :meth:`DataFrame.__getitem__` raising error for slice of :class:`DatetimeIndex` when index is non monotonic (:issue:`43223`)
 - Fixed regression in :meth:`.Resampler.aggregate` when used after column selection would raise if ``func`` is a list of aggregation functions (:issue:`42905`)
 - Fixed regression in :meth:`DataFrame.corr` where Kendall correlation would produce incorrect results for columns with repeated values (:issue:`43401`)
+- Fixed regression in :meth:`DataFrame.groupby` where aggregation on columns with object types dropped results on those columns (:issue:`42395`, :issue:`43108`)
+- Fixed regression in :meth:`Series.fillna` raising ``TypeError`` when filling ``float`` ``Series`` with list-like fill value having a dtype which couldn't cast lostlessly (like ``float32`` filled with ``float64``) (:issue:`43424`)
+- Fixed regression in :func:`read_csv` throwing an ``AttributeError`` when the file handle is an ``tempfile.SpooledTemporaryFile`` object (:issue:`43439`)
+-
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -75,7 +75,7 @@ Styler
   - Styling of indexing has been added, with :meth:`.Styler.apply_index` and :meth:`.Styler.applymap_index`. These mirror the signature of the methods already used to style data values, and work with both HTML and LaTeX format (:issue:`41893`).
   - :meth:`.Styler.bar` introduces additional arguments to control alignment and display (:issue:`26070`, :issue:`36419`), and it also validates the input arguments ``width`` and ``height`` (:issue:`42511`).
   - :meth:`.Styler.to_latex` introduces keyword argument ``environment``, which also allows a specific "longtable" entry through a separate jinja2 template (:issue:`41866`).
-  - :meth:`.Styler.to_html` introduces keyword arguments ``sparse_index``, ``sparse_columns``, ``bold_headers``, ``caption`` (:issue:`41946`, :issue:`43149`).
+  - :meth:`.Styler.to_html` introduces keyword arguments ``sparse_index``, ``sparse_columns``, ``bold_headers``, ``caption``, ``max_rows`` and ``max_columns`` (:issue:`41946`, :issue:`43149`, :issue:`42972`).
   - Keyword arguments ``level`` and ``names`` added to :meth:`.Styler.hide_index` and :meth:`.Styler.hide_columns` for additional control of visibility of MultiIndexes and index names (:issue:`25475`, :issue:`43404`, :issue:`43346`)
   - Global options have been extended to configure default ``Styler`` properties including formatting and encoding and mathjax options and LaTeX (:issue:`41395`)
   - Naive sparsification is now possible for LaTeX without the multirow package (:issue:`43369`)
@@ -104,9 +104,11 @@ Other enhancements
 - :meth:`Series.ewm`, :meth:`DataFrame.ewm`, now support a ``method`` argument with a ``'table'`` option that performs the windowing operation over an entire :class:`DataFrame`. See :ref:`Window Overview <window.overview>` for performance and functional benefits (:issue:`42273`)
 - :meth:`.GroupBy.cummin` and :meth:`.GroupBy.cummax` now support the argument ``skipna`` (:issue:`34047`)
 - :meth:`read_table` now supports the argument ``storage_options`` (:issue:`39167`)
+- :meth:`DataFrame.to_stata` and :meth:`StataWriter` now accept the keyword only argument ``value_labels`` to save labels for non-categorical columns
 - Methods that relied on hashmap based algos such as :meth:`DataFrameGroupBy.value_counts`, :meth:`DataFrameGroupBy.count` and :func:`factorize` ignored imaginary component for complex numbers (:issue:`17927`)
 - Add :meth:`Series.str.removeprefix` and :meth:`Series.str.removesuffix` introduced in Python 3.9 to remove pre-/suffixes from string-type :class:`Series` (:issue:`36944`)
 - :meth:`.GroupBy.mean` now supports `NaT` values (:issue:`43132`)
+
 .. ---------------------------------------------------------------------------
 
 .. _whatsnew_140.notable_bug_fixes:
@@ -277,6 +279,8 @@ Other Deprecations
 - Deprecated :meth:`Index.reindex` with a non-unique index (:issue:`42568`)
 - Deprecated :meth:`.Styler.render` in favour of :meth:`.Styler.to_html` (:issue:`42140`)
 - Deprecated passing in a string column label into ``times`` in :meth:`DataFrame.ewm` (:issue:`43265`)
+- Deprecated the 'include_start' and 'include_end' arguments in :meth:`DataFrame.between_time`; in a future version passing 'include_start' or 'include_end' will raise (:issue:`40245`)
+- Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
 
 .. ---------------------------------------------------------------------------
 
@@ -294,6 +298,8 @@ Performance improvements
 - Performance improvement in :meth:`to_datetime` with ``uint`` dtypes (:issue:`42606`)
 - Performance improvement in :meth:`Series.sparse.to_coo` (:issue:`42880`)
 - Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
+- Performance improvement in :meth:`GroupBy.quantile` (:issue:`43469`)
+-
 
 .. ---------------------------------------------------------------------------
 
@@ -367,6 +373,7 @@ Indexing
 Missing
 ^^^^^^^
 - Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`)
+- Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 -
 
 MultiIndex
@@ -424,6 +431,7 @@ Reshaping
 
 Sparse
 ^^^^^^
+- Bug in :meth:`DataFrame.sparse.to_coo` raising ``AttributeError`` when column names are not unique (:issue:`29564`)
 -
 -
 
@@ -440,7 +448,7 @@ Styler
 - Bug in :meth:`Styler.apply` where functions which returned Series objects were not correctly handled in terms of aligning their index labels (:issue:`13657`, :issue:`42014`)
 - Bug when rendering an empty DataFrame with a named index (:issue:`43305`).
 - Bug when rendering a single level MultiIndex (:issue:`43383`).
-- Bug when combining non-sparse rendering and :meth:`.Styler.hide_columns` (:issue:`43464`)
+- Bug when combining non-sparse rendering and :meth:`.Styler.hide_columns` or :meth:`.Styler.hide_index` (:issue:`43464`)
 
 Other
 ^^^^^

pandas/_libs/groupby.pyi

@@ -85,11 +85,11 @@ def group_ohlc(
     min_count: int = ...,
 ) -> None: ...
 def group_quantile(
-    out: np.ndarray,  # ndarray[float64_t]
+    out: np.ndarray,  # ndarray[float64_t, ndim=2]
     values: np.ndarray,  # ndarray[numeric, ndim=1]
     labels: np.ndarray,  # ndarray[int64_t]
     mask: np.ndarray,  # ndarray[uint8_t]
-    q: float,  # float64_t
+    qs: np.ndarray,  # const float64_t[:]
     interpolation: Literal["linear", "lower", "higher", "nearest", "midpoint"],
 ) -> None: ...
 def group_last(