Merge remote-tracking branch 'upstream/master' into gh-36562-typeerror-comparison-not-supported-between-float-and-str

# Conflicts:
#	doc/source/whatsnew/v1.2.0.rst

Author: ssche

.github/CODE_OF_CONDUCT.md

@@ -60,4 +60,3 @@ and the [Swift Code of Conduct][swift].
 [homepage]: https://www.contributor-covenant.org
 [version]: https://www.contributor-covenant.org/version/1/3/0/
 [swift]: https://swift.org/community/#code-of-conduct
-

.github/workflows/ci.yml

@@ -125,7 +125,7 @@ jobs:
     # This can be removed when the ipython directive fails when there are errors,
     # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
     - name: Check ipython directive errors
-      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
+      run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
 
     - name: Install ssh key
       run: |

.pre-commit-config.yaml

@@ -4,27 +4,28 @@ repos:
     hooks:
     -   id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.8.4
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-comprehensions>=3.1.0]
     -   id: flake8
-        name: flake8-pyx
-        files: \.(pyx|pxd)$
-        types:
-          - file
+        name: flake8 (cython)
+        types: [cython]
         args: [--append-config=flake8/cython.cfg]
     -   id: flake8
-        name: flake8-pxd
+        name: flake8 (cython template)
         files: \.pxi\.in$
         types:
           - file
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.2.2
+    rev: 5.6.3
     hooks:
     -   id: isort
-        exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
+        name: isort (python)
+    -   id: isort
+        name: isort (cython)
+        types: [cython]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.2
     hooks:
@@ -39,7 +40,25 @@ repos:
     -   id: pip_to_conda
         name: Generate pip dependency from conda
         description: This hook checks if the conda environment.yml and requirements-dev.txt are equal
-        language: system
+        language: python
         entry: python -m scripts.generate_pip_deps_from_conda
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
+        additional_dependencies: [pyyaml]
+    -   id: flake8-rst
+        name: flake8-rst
+        description: Run flake8 on code snippets in docstrings or RST files
+        language: python
+        entry: flake8-rst
+        types: [rst]
+        args: [--filename=*.rst]
+        additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
+-   repo: https://github.com/asottile/yesqa
+    rev: v1.2.2
+    hooks:
+    -   id: yesqa
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: end-of-file-fixer
+        exclude: '.html$|^LICENSES/|.csv$|.txt$|.svg$|.py$'

.travis.yml

@@ -41,10 +41,10 @@ matrix:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
-
-    - env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
+      - JOB="3.8, slow" ENV_FILE="ci/deps/travis-38-slow.yaml" PATTERN="slow" SQL="1"
+      services:
+        - mysql
+        - postgresql
 
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"

AUTHORS.md

@@ -54,4 +54,3 @@ pandas is distributed under a 3-clause ("Simplified" or "New") BSD
 license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have
 BSD-compatible licenses, are included. Their licenses follow the pandas
 license.
-

asv_bench/benchmarks/dtypes.py

@@ -1,5 +1,9 @@
+import string
+
 import numpy as np
 
+from pandas import DataFrame
+import pandas._testing as tm
 from pandas.api.types import pandas_dtype
 
 from .pandas_vb_common import (
@@ -62,4 +66,57 @@ def time_infer(self, dtype):
         lib.infer_dtype(self.data_dict[dtype], skipna=False)
 
 
+class SelectDtypes:
+
+    params = [
+        tm.ALL_INT_DTYPES
+        + tm.ALL_EA_INT_DTYPES
+        + tm.FLOAT_DTYPES
+        + tm.COMPLEX_DTYPES
+        + tm.DATETIME64_DTYPES
+        + tm.TIMEDELTA64_DTYPES
+        + tm.BOOL_DTYPES
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N, K = 5000, 50
+        self.index = tm.makeStringIndex(N)
+        self.columns = tm.makeStringIndex(K)
+
+        def create_df(data):
+            return DataFrame(data, index=self.index, columns=self.columns)
+
+        self.df_int = create_df(np.random.randint(low=100, size=(N, K)))
+        self.df_float = create_df(np.random.randn(N, K))
+        self.df_bool = create_df(np.random.choice([True, False], size=(N, K)))
+        self.df_string = create_df(
+            np.random.choice(list(string.ascii_letters), size=(N, K))
+        )
+
+    def time_select_dtype_int_include(self, dtype):
+        self.df_int.select_dtypes(include=dtype)
+
+    def time_select_dtype_int_exclude(self, dtype):
+        self.df_int.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_float_include(self, dtype):
+        self.df_float.select_dtypes(include=dtype)
+
+    def time_select_dtype_float_exclude(self, dtype):
+        self.df_float.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_bool_include(self, dtype):
+        self.df_bool.select_dtypes(include=dtype)
+
+    def time_select_dtype_bool_exclude(self, dtype):
+        self.df_bool.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_string_include(self, dtype):
+        self.df_string.select_dtypes(include=dtype)
+
+    def time_select_dtype_string_exclude(self, dtype):
+        self.df_string.select_dtypes(exclude=dtype)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -191,7 +191,7 @@ def setup(self, index):
         }
         index = indexes[index]
         self.s = Series(np.random.rand(N), index=index)
-        self.indexer = [True, False, True, True, False] * 20000
+        self.indexer = np.random.randint(0, N, size=N)
 
     def time_take(self, index):
         self.s.take(self.indexer)

asv_bench/benchmarks/pandas_vb_common.py

@@ -15,7 +15,7 @@
 
 # Compatibility import for the testing module
 try:
-    import pandas._testing as tm  # noqa
+    import pandas._testing as tm
 except ImportError:
     import pandas.util.testing as tm  # noqa
 

asv_bench/benchmarks/rolling.py

@@ -76,12 +76,21 @@ class ExpandingMethods:
 
     def setup(self, constructor, dtype, method):
         N = 10 ** 5
+        N_groupby = 100
         arr = (100 * np.random.random(N)).astype(dtype)
         self.expanding = getattr(pd, constructor)(arr).expanding()
+        self.expanding_groupby = (
+            pd.DataFrame({"A": arr[:N_groupby], "B": range(N_groupby)})
+            .groupby("B")
+            .expanding()
+        )
 
     def time_expanding(self, constructor, dtype, method):
         getattr(self.expanding, method)()
 
+    def time_expanding_groupby(self, constructor, dtype, method):
+        getattr(self.expanding_groupby, method)()
+
 
 class EWMMethods:
 

asv_bench/benchmarks/tslibs/offsets.py

@@ -9,7 +9,7 @@
 from pandas import offsets
 
 try:
-    import pandas.tseries.holiday  # noqa
+    import pandas.tseries.holiday
 except ImportError:
     pass
 

ci/azure/posix.yml

@@ -20,39 +20,35 @@ jobs:
           CONDA_PY: "37"
           PATTERN: "not slow and not network and not clipboard"
 
+        py37:
+          ENV_FILE: ci/deps/azure-37.yaml
+          CONDA_PY: "37"
+          PATTERN: "not slow and not network and not clipboard"
+
         py37_locale_slow:
           ENV_FILE: ci/deps/azure-37-locale_slow.yaml
           CONDA_PY: "37"
           PATTERN: "slow"
-          # pandas does not use the language (zh_CN), but should support different encodings (utf8)
-          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
-          LANG: "zh_CN.utf8"
-          LC_ALL: "zh_CN.utf8"
-          EXTRA_APT: "language-pack-zh-hans"
+          LANG: "it_IT.utf8"
+          LC_ALL: "it_IT.utf8"
+          EXTRA_APT: "language-pack-it xsel"
 
         py37_slow:
           ENV_FILE: ci/deps/azure-37-slow.yaml
           CONDA_PY: "37"
           PATTERN: "slow"
 
-        py37_locale:
-          ENV_FILE: ci/deps/azure-37-locale.yaml
-          CONDA_PY: "37"
-          PATTERN: "not slow and not network"
-          LANG: "it_IT.utf8"
-          LC_ALL: "it_IT.utf8"
-          EXTRA_APT: "language-pack-it xsel"
-
-#        py37_32bit:
-#          ENV_FILE: ci/deps/azure-37-32bit.yaml
-#          CONDA_PY: "37"
-#          PATTERN: "not slow and not network and not clipboard"
-#          BITS32: "yes"
+        py38:
+          ENV_FILE: ci/deps/azure-38.yaml
+          CONDA_PY: "38"
+          PATTERN: "not slow and not network and not clipboard"
 
         py38_locale:
           ENV_FILE: ci/deps/azure-38-locale.yaml
           CONDA_PY: "38"
           PATTERN: "not slow and not network"
+          # pandas does not use the language (zh_CN), but should support different encodings (utf8)
+          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans xsel"

ci/code_checks.sh

@@ -73,13 +73,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     flake8 --format="$FLAKE8_FORMAT" pandas/_libs --append-config=flake8/cython-template.cfg
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    echo "flake8-rst --version"
-    flake8-rst --version
-
-    MSG='Linting code-blocks in .rst documentation' ; echo $MSG
-    flake8-rst doc/source --filename=*.rst --format="$FLAKE8_FORMAT"
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
     # it doesn't make a difference, but we want to be internally consistent.
     # Note: this grep pattern is (intended to be) equivalent to the python

ci/deps/azure-37-32bit.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/azure-37-slow.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - botocore>=1.11

ci/deps/travis-37.yaml renamed to ci/deps/azure-37.yaml

@@ -10,11 +10,11 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - numpy
   - python-dateutil
   - nomkl
   - pytz
-  - pip
   - tabulate==0.8.3

ci/deps/travis-38.yaml renamed to ci/deps/azure-38.yaml

@@ -32,15 +32,15 @@ dependencies:
   - google-cloud-bigquery>=1.27.2 # GH 36436
   - psycopg2
   - pyarrow>=0.15.0
-  - pymysql=0.7.11
+  - pymysql<0.10.0 # temporary pin, GH 36465
   - pytables
   - python-snappy
   - python-dateutil
   - pytz
   - s3fs>=0.4.0
   - scikit-learn
   - scipy
-  - sqlalchemy=1.3.0
+  - sqlalchemy
   - statsmodels
   - xarray
   - xlrd