Merge remote-tracking branch 'upstream/master' into conda-4.7.1

Author: TomAugspurger

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://pandas.pydata.org/donate.html

.travis.yml

@@ -103,10 +103,5 @@ script:
 after_script:
   - echo "after_script start"
   - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - if [ -e test-data-single.xml ]; then
-        ci/print_skipped.py test-data-single.xml;
-    fi
-  - if [ -e test-data-multiple.xml ]; then
-        ci/print_skipped.py test-data-multiple.xml;
-    fi
+  - ci/print_skipped.py 
   - echo "after_script done"

asv_bench/asv.conf.json

@@ -107,7 +107,7 @@
     // `asv` will cache wheels of the recent builds in each
     // environment, making them faster to install next time.  This is
     // number of builds to keep, per environment.
-    "wheel_cache_size": 8,
+    "build_cache_size": 8,
 
     // The commits after which the regression search in `asv publish`
     // should start looking for regressions. Dictionary whose keys are

asv_bench/benchmarks/index_object.py

@@ -52,7 +52,6 @@ def time_is_dates_only(self):
 
 class Ops:
 
-    sample_time = 0.2
     params = ['float', 'int']
     param_names = ['dtype']
 
@@ -95,6 +94,12 @@ def time_min(self):
     def time_min_trivial(self):
         self.idx_inc.min()
 
+    def time_get_loc_inc(self):
+        self.idx_inc.get_loc(900000)
+
+    def time_get_loc_dec(self):
+        self.idx_dec.get_loc(100000)
+
 
 class IndexAppend:
 
@@ -183,19 +188,34 @@ def time_get_loc(self):
 
 class IntervalIndexMethod:
     # GH 24813
-    params = [10**3, 10**5, 10**7]
+    params = [10**3, 10**5]
 
     def setup(self, N):
         left = np.append(np.arange(N), np.array(0))
         right = np.append(np.arange(1, N + 1), np.array(1))
         self.intv = IntervalIndex.from_arrays(left, right)
         self.intv._engine
 
+        self.intv2 = IntervalIndex.from_arrays(left + 1, right + 1)
+        self.intv2._engine
+
+        self.left = IntervalIndex.from_breaks(np.arange(N))
+        self.right = IntervalIndex.from_breaks(np.arange(N - 3, 2 * N - 3))
+
     def time_monotonic_inc(self, N):
         self.intv.is_monotonic_increasing
 
     def time_is_unique(self, N):
         self.intv.is_unique
 
+    def time_intersection(self, N):
+        self.left.intersection(self.right)
+
+    def time_intersection_one_duplicate(self, N):
+        self.intv.intersection(self.right)
+
+    def time_intersection_both_duplicate(self, N):
+        self.intv.intersection(self.intv2)
+
 
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/multiindex_object.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 import pandas.util.testing as tm
-from pandas import date_range, MultiIndex
+from pandas import date_range, MultiIndex, DataFrame
 
 
 class GetLoc:
@@ -126,4 +126,18 @@ def time_datetime_level_values_sliced(self, mi):
         mi[:10].values
 
 
+class CategoricalLevel:
+
+    def setup(self):
+
+        self.df = DataFrame({
+            'a': np.arange(1_000_000, dtype=np.int32),
+            'b': np.arange(1_000_000, dtype=np.int64),
+            'c': np.arange(1_000_000, dtype=float),
+        }).astype({'a': 'category', 'b': 'category'})
+
+    def time_categorical_level(self):
+        self.df.set_index(['a', 'b'])
+
+
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/rolling.py

@@ -4,7 +4,6 @@
 
 class Methods:
 
-    sample_time = 0.2
     params = (['DataFrame', 'Series'],
               [10, 1000],
               ['int', 'float'],
@@ -23,7 +22,6 @@ def time_rolling(self, constructor, window, dtype, method):
 
 class ExpandingMethods:
 
-    sample_time = 0.2
     params = (['DataFrame', 'Series'],
               ['int', 'float'],
               ['median', 'mean', 'max', 'min', 'std', 'count', 'skew', 'kurt',
@@ -41,7 +39,6 @@ def time_expanding(self, constructor, dtype, method):
 
 class EWMMethods:
 
-    sample_time = 0.2
     params = (['DataFrame', 'Series'],
               [10, 1000],
               ['int', 'float'],
@@ -58,7 +55,6 @@ def time_ewm(self, constructor, window, dtype, method):
 
 
 class VariableWindowMethods(Methods):
-    sample_time = 0.2
     params = (['DataFrame', 'Series'],
               ['50s', '1h', '1d'],
               ['int', 'float'],
@@ -75,7 +71,6 @@ def setup(self, constructor, window, dtype, method):
 
 class Pairwise:
 
-    sample_time = 0.2
     params = ([10, 1000, None],
               ['corr', 'cov'],
               [True, False])
@@ -95,7 +90,6 @@ def time_pairwise(self, window, method, pairwise):
 
 
 class Quantile:
-    sample_time = 0.2
     params = (['DataFrame', 'Series'],
               [10, 1000],
               ['int', 'float'],

asv_bench/benchmarks/timeseries.py

@@ -5,7 +5,7 @@
 from pandas import to_datetime, date_range, Series, DataFrame, period_range
 from pandas.tseries.frequencies import infer_freq
 try:
-    from pandas.plotting._converter import DatetimeConverter
+    from pandas.plotting._matplotlib.converter import DatetimeConverter
 except ImportError:
     from pandas.tseries.converter import DatetimeConverter
 

azure-pipelines.yml

@@ -15,7 +15,7 @@ jobs:
     name: Windows
     vmImage: vs2017-win2016
 
-- job: 'Checks_and_doc'
+- job: 'Checks'
   pool:
     vmImage: ubuntu-16.04
   timeoutInMinutes: 90
@@ -24,7 +24,6 @@ jobs:
       # XXX next command should avoid redefining the path in every step, but
       # made the process crash as it couldn't find deactivate
       #echo '##vso[task.prependpath]$HOME/miniconda3/bin'
-      echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev'
       echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
       echo '##vso[task.setvariable variable=AZURE]true'
     displayName: 'Setting environment variables'
@@ -116,3 +115,65 @@ jobs:
       fi
     displayName: 'Running benchmarks'
     condition: true
+
+- job: 'Docs'
+  pool:
+    vmImage: ubuntu-16.04
+  timeoutInMinutes: 90
+  steps:
+  - script: |
+      echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml'
+    displayName: 'Setting environment variables'
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      sudo apt-get install -y libc6-dev-i386
+      ci/setup_env.sh
+    displayName: 'Setup environment and build pandas'
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      doc/make.py
+    displayName: 'Build documentation'
+
+  - script: |
+      cd doc/build/html
+      git init
+      touch .nojekyll
+      echo "dev.pandas.io" > CNAME
+      git add --all .
+      git config user.email "[email protected]"
+      git config user.name "pandas-docs-bot"
+      git commit -m "pandas documentation in master"
+    displayName: 'Create git repo for docs build'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  # For `InstallSSHKey@0` to work, next steps are required:
+  # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "[email protected]"`)
+  # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles
+  # 3. Click on "+ Secure file"
+  # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key")
+  # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save
+  # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys
+  - task: InstallSSHKey@0
+    inputs:
+      hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=='
+      sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== [email protected]'
+      sshKeySecureFile: 'pandas_docs_key'
+    displayName: 'Install GitHub ssh deployment key'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  - script: |
+      cd doc/build/html
+      git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
+      git push -f origin master
+      exit 0  # FIXME this will leave the build green even if the step fails. To be removed when we are confident with this.
+    displayName: 'Publish docs to GitHub pages'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))

ci/azure/posix.yml

@@ -89,4 +89,9 @@ jobs:
           # note that this will produce $LASTEXITCODE=1
           Write-Error "$($matches[1]) tests failed"
         }
-      displayName: Check for test failures
+      displayName: 'Check for test failures'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
+        python ci/print_skipped.py
+      displayName: 'Print skipped tests'

ci/azure/windows.yml

@@ -18,13 +18,11 @@ jobs:
 
   steps:
     - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
-      displayName: Add conda to PATH
+      displayName: 'Add conda to PATH'
     - script: conda install -c conda-canary conda
       displayName: Update conda
-    - script: |
-        call activate
-        conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
-      displayName: Create anaconda environment
+    - script: conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+      displayName: 'Create anaconda environment'
     - script: |
         call activate pandas-dev
         call conda list
@@ -50,4 +48,9 @@ jobs:
           # note that this will produce $LASTEXITCODE=1
           Write-Error "$($matches[1]) tests failed"
         }
-      displayName: Check for test failures
+      displayName: 'Check for test failures'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
+        python ci/print_skipped.py
+      displayName: 'Print skipped tests'

ci/deps/azure-35-compat.yaml

@@ -4,28 +4,27 @@ channels:
   - conda-forge
   - free
 dependencies:
-  - beautifulsoup4=4.4.1
+  - beautifulsoup4=4.6.0
   - bottleneck=1.2.1
   - jinja2=2.8
   - numexpr=2.6.2
   - numpy=1.13.3
-  - openpyxl=2.4.0
+  - openpyxl=2.4.8
   - pytables=3.4.2
-  - python-dateutil=2.5.0
+  - python-dateutil=2.6.1
   - python=3.5.*
-  - pytz=2015.4
+  - pytz=2017.2
   - scipy=0.19.0
-  - xlrd=1.0.0
-  - xlsxwriter=0.7.7
-  - xlwt=1.0.0
+  - xlrd=1.1.0
+  - xlsxwriter=0.9.8
+  - xlwt=1.2.0
   # universal
   - cython=0.28.2
   - hypothesis>=3.58.0
   - pytest-xdist
   - pytest-mock
-  - isort
   - pip
   - pip:
     # for python 3.5, pytest>=4.0.2 is not available in conda
-    - pytest>=4.0.2
+    - pytest==4.5.0
     - html5lib==1.0b2

ci/deps/azure-36-locale.yaml

@@ -3,28 +3,27 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - beautifulsoup4==4.5.1
+  - beautifulsoup4==4.6.0
   - bottleneck=1.2.*
   - cython=0.28.2
   - lxml
   - matplotlib=2.2.2
   - numpy=1.14.*
-  - openpyxl=2.4.0
+  - openpyxl=2.4.8
   - python-dateutil
   - python-blosc
   - python=3.6.*
-  - pytz=2016.10
+  - pytz=2017.2
   - scipy
   - sqlalchemy=1.1.4
-  - xlrd=1.0.0
-  - xlsxwriter=0.9.4
+  - xlrd=1.1.0
+  - xlsxwriter=0.9.8
   - xlwt=1.2.0
   # universal
   - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
-  - isort
   - pip
   - pip:
     - html5lib==1.0b2

ci/deps/azure-36-locale_slow.yaml

@@ -30,7 +30,6 @@ dependencies:
   - pytest-xdist
   - pytest-mock
   - moto
-  - isort
   - pip
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

@@ -28,7 +28,6 @@ dependencies:
   - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
-  - isort
   - pip
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-37-numpydev.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
-  - isort
   - pip
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"