Merge remote-tracking branch 'upstream/master' into arrow-string-array-dtype

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -22,7 +22,9 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
@@ -94,7 +96,9 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Set up pandas
       uses: ./.github/actions/setup
@@ -147,7 +151,9 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Set up pandas
       uses: ./.github/actions/setup

.github/workflows/database.yml

@@ -56,10 +56,12 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Cache conda
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
         CACHE_NUMBER: 0
       with:
@@ -70,7 +72,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: strict
+        channel-priority: flexible
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

.github/workflows/posix.yml

@@ -44,10 +44,12 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Cache conda
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
         CACHE_NUMBER: 0
       with:

.pre-commit-config.yaml

@@ -19,13 +19,14 @@ repos:
         types_or: [python, rst, markdown]
         files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
+    -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
     -   id: trailing-whitespace
 -   repo: https://github.com/cpplint/cpplint
-    rev: f7061b1 # the latest tag does not have the hook
+    rev: 1.5.5
     hooks:
     -   id: cpplint
         # We don't lint all C files because we don't want to lint any that are built
@@ -35,7 +36,7 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.1
+    rev: 3.9.2
     hooks:
     -   id: flake8
         additional_dependencies:
@@ -56,7 +57,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.12.0
+    rev: v2.18.3
     hooks:
     -   id: pyupgrade
         args: [--py37-plus]
@@ -71,11 +72,11 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/asottile/yesqa
-    rev: v1.2.2
+    rev: v1.2.3
     hooks:
     -   id: yesqa
         additional_dependencies:
-            - flake8==3.9.1
+            - flake8==3.9.2
             - flake8-comprehensions==3.1.0
             - flake8-bugbear==21.3.2
             - pandas-dev-flaker==0.2.0

asv_bench/benchmarks/io/style.py

@@ -20,19 +20,19 @@ def setup(self, cols, rows):
 
     def time_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_format_render(self, cols, rows):
         self._style_format()

asv_bench/benchmarks/strings.py

@@ -11,6 +11,19 @@
 from .pandas_vb_common import tm
 
 
+class Dtypes:
+    params = ["str", "string", "arrow_string"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        from pandas.core.arrays.string_arrow import ArrowStringDtype  # noqa: F401
+
+        try:
+            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
+        except ImportError:
+            raise NotImplementedError
+
+
 class Construction:
 
     params = ["str", "string"]
@@ -49,16 +62,7 @@ def peakmem_cat_frame_construction(self, dtype):
         DataFrame(self.frame_cat_arr, dtype=dtype)
 
 
-class Methods:
-    params = ["str", "string[python]", "string[pyarrow]"]
-    param_names = ["dtype"]
-
-    def setup(self, dtype):
-        try:
-            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
-        except ImportError:
-            raise NotImplementedError
-
+class Methods(Dtypes):
     def time_center(self, dtype):
         self.s.str.center(100)
 
@@ -81,6 +85,9 @@ def time_find(self, dtype):
     def time_rfind(self, dtype):
         self.s.str.rfind("[A-Z]+")
 
+    def time_fullmatch(self, dtype):
+        self.s.str.fullmatch("A")
+
     def time_get(self, dtype):
         self.s.str.get(0)
 
@@ -209,31 +216,26 @@ def time_cat(self, other_cols, sep, na_rep, na_frac):
         self.s.str.cat(others=self.others, sep=sep, na_rep=na_rep)
 
 
-class Contains:
+class Contains(Dtypes):
 
-    params = (["str", "string[python]", "string[pyarrow]"], [True, False])
+    params = (Dtypes.params, [True, False])
     param_names = ["dtype", "regex"]
 
     def setup(self, dtype, regex):
-        try:
-            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
-        except ImportError:
-            raise NotImplementedError
+        super().setup(dtype)
 
     def time_contains(self, dtype, regex):
         self.s.str.contains("A", regex=regex)
 
 
-class Split:
+class Split(Dtypes):
 
-    params = (["str", "string[python]", "string[pyarrow]"], [True, False])
+    params = (Dtypes.params, [True, False])
     param_names = ["dtype", "expand"]
 
     def setup(self, dtype, expand):
-        try:
-            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype).str.join("--")
-        except ImportError:
-            raise NotImplementedError
+        super().setup(dtype)
+        self.s = self.s.str.join("--")
 
     def time_split(self, dtype, expand):
         self.s.str.split("--", expand=expand)
@@ -242,11 +244,25 @@ def time_rsplit(self, dtype, expand):
         self.s.str.rsplit("--", expand=expand)
 
 
-class Dummies:
-    def setup(self):
-        self.s = Series(tm.makeStringIndex(10 ** 5)).str.join("|")
+class Extract(Dtypes):
+
+    params = (Dtypes.params, [True, False])
+    param_names = ["dtype", "expand"]
 
-    def time_get_dummies(self):
+    def setup(self, dtype, expand):
+        super().setup(dtype)
+
+    def time_extract_single_group(self, dtype, expand):
+        with warnings.catch_warnings(record=True):
+            self.s.str.extract("(\\w*)A", expand=expand)
+
+
+class Dummies(Dtypes):
+    def setup(self, dtype):
+        super().setup(dtype)
+        self.s = self.s.str.join("|")
+
+    def time_get_dummies(self, dtype):
         self.s.str.get_dummies("|")
 
 
@@ -265,3 +281,9 @@ def setup(self):
     def time_vector_slice(self):
         # GH 2602
         self.s.str[:5]
+
+
+class Iter(Dtypes):
+    def time_iter(self, dtype):
+        for i in self.s:
+            pass

ci/deps/actions-37-db-min.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # tools
   - cython>=0.29.21
-  - pytest>=5.0.1
+  - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
@@ -31,7 +31,8 @@ dependencies:
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
-  - pyarrow=0.17 # GH 38803
+  - protobuf>=3.12.4
+  - pyarrow=0.17.1 # GH 38803
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3

ci/deps/actions-37-db.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # tools
   - cython>=0.29.21
-  - pytest>=5.0.1
+  - pytest>=6.0
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
@@ -15,7 +15,7 @@ dependencies:
   - beautifulsoup4
   - botocore>=1.11
   - dask
-  - fastparquet>=0.4.0, <=0.5.0
+  - fastparquet>=0.4.0
   - fsspec>=0.7.4
   - gcsfs>=0.6.0
   - geopandas
@@ -25,13 +25,13 @@ dependencies:
   - flask
   - nomkl
   - numexpr
-  - numpy=1.16.*
+  - numpy=1.17.*
   - odfpy
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
   - psycopg2
-  - pyarrow>=0.15.0
+  - pyarrow>=0.17.0
   - pymysql
   - pytables
   - python-snappy

ci/deps/actions-37-locale_slow.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.21
-  - pytest>=5.0.1
+  - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
@@ -17,13 +17,13 @@ dependencies:
   - bottleneck=1.2.*
   - lxml
   - matplotlib=3.0.0
-  - numpy=1.16.*
+  - numpy=1.17.*
   - openpyxl=3.0.0
   - python-dateutil
   - python-blosc
   - pytz=2017.3
   - scipy
-  - sqlalchemy=1.2.8
+  - sqlalchemy=1.3.0
   - xlrd=1.2.0
   - xlsxwriter=1.0.2
   - xlwt=1.3.0

ci/deps/actions-37-minimum_versions.yaml

@@ -17,13 +17,13 @@ dependencies:
   - bottleneck=1.2.1
   - jinja2=2.10
   - numba=0.46.0
-  - numexpr=2.6.8
+  - numexpr=2.7.0
   - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1
   - python-dateutil=2.7.3
   - pytz=2017.3
-  - pyarrow=0.15
+  - pyarrow=0.17.0
   - scipy=1.2
   - xlrd=1.2.0
   - xlsxwriter=1.0.2

ci/deps/actions-37-slow.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.21
-  - pytest>=5.0.1
+  - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0

ci/deps/actions-37.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.21
-  - pytest>=5.0.1
+  - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
@@ -18,7 +18,7 @@ dependencies:
   - numpy=1.19
   - python-dateutil
   - nomkl
-  - pyarrow=0.15.1
+  - pyarrow
   - pytz
   - s3fs>=0.4.0
   - moto>=1.3.14