Merge branch 'master' into fix-15582

Author: albertvillanova

.travis.yml

@@ -8,7 +8,6 @@ language: python
 # The cash directories will be deleted if anything in ci/ changes in a commit
 cache:
  directories:
-  - $HOME/miniconda # miniconda cache
   - $HOME/.cache # cython cache
   - $HOME/.ccache # compiler cache
 
@@ -75,7 +74,7 @@ matrix:
         - CLIPBOARD=xsel
         - COVERAGE=true
         - CACHE_NAME="35_nslow"
-#        - USE_CACHE=true # Don't use cache for 35_nslow
+        - USE_CACHE=true
       addons:
         apt:
           packages:
@@ -86,6 +85,8 @@ matrix:
         - JOB_NAME: "36"
         - TEST_ARGS="--skip-slow --skip-network"
         - PANDAS_TESTING_MODE="deprecate"
+        - CONDA_FORGE=true
+        - USE_CACHE=true
       addons:
         apt:
           packages:
@@ -122,11 +123,6 @@ matrix:
         - PANDAS_TESTING_MODE="deprecate"
         - CACHE_NAME="35_numpy_dev"
         - USE_CACHE=true
-      addons:
-        apt:
-          packages:
-          - libatlas-base-dev
-          - gfortran
 #    In allow_failures
     - python: 3.5
       env:
@@ -159,18 +155,13 @@ matrix:
         - USE_CACHE=true
       - python: 3.5
         env:
-          - PYTHON_VERSION=3.5
-          - JOB_NAME: "35_numpy_dev"
-          - JOB_TAG=_NUMPY_DEV
-          - TEST_ARGS="--skip-slow --skip-network"
-          - PANDAS_TESTING_MODE="deprecate"
-          - CACHE_NAME="35_numpy_dev"
-          - USE_CACHE=true
-        addons:
-          apt:
-            packages:
-            - libatlas-base-dev
-            - gfortran
+        - PYTHON_VERSION=3.5
+        - JOB_NAME: "35_numpy_dev"
+        - JOB_TAG=_NUMPY_DEV
+        - TEST_ARGS="--skip-slow --skip-network"
+        - PANDAS_TESTING_MODE="deprecate"
+        - CACHE_NAME="35_numpy_dev"
+        - USE_CACHE=true
       - python: 3.5
         env:
         - PYTHON_VERSION=3.5

Makefile

@@ -1,4 +1,4 @@
-tseries: pandas/lib.pyx pandas/tslib.pyx pandas/hashtable.pyx
+tseries: pandas/_libs/lib.pyx pandas/_libs/tslib.pyx pandas/_libs/hashtable.pyx
 	python setup.py build_ext --inplace
 
 .PHONY : develop build clean clean_pyc tseries doc
@@ -9,9 +9,6 @@ clean:
 clean_pyc:
 	-find . -name '*.py[co]' -exec rm {} \;
 
-sparse: pandas/src/sparse.pyx
-	python setup.py build_ext --inplace
-
 build: clean_pyc
 	python setup.py build_ext --inplace
 

README.md

@@ -41,8 +41,8 @@
 <tr>
   <td></td>
   <td>
-    <a href="https://ci.appveyor.com/project/jreback/pandas-465">
-    <img src="https://ci.appveyor.com/api/projects/status/iblk29s98quexwxi/branch/master?svg=true" alt="appveyor build status" />
+    <a href="https://ci.appveyor.com/project/pandas-dev/pandas">
+    <img src="https://ci.appveyor.com/api/projects/status/86vn83mxgnl4xf1s/branch/master?svg=true" alt="appveyor build status" />
     </a>
   </td>
 </tr>
@@ -54,7 +54,15 @@
   <td>Conda</td>
   <td>
     <a href="http://pandas.pydata.org">
-    <img src="http://pubbadges.s3-website-us-east-1.amazonaws.com/pkgs-downloads-pandas.png" alt="conda downloads" />
+    <img src="http://pubbadges.s3-website-us-east-1.amazonaws.com/pkgs-downloads-pandas.png" alt="conda default downloads" />
+    </a>
+  </td>
+</tr>
+<tr>
+  <td>Conda-forge</td>
+  <td>
+    <a href="http://pandas.pydata.org">
+    <img src="https://anaconda.org/conda-forge/pandas/badges/downloads.svg" alt="conda-forge downloads" />
     </a>
   </td>
 </tr>

asv_bench/benchmarks/binary_ops.py

@@ -107,4 +107,4 @@ def setup(self):
         self.s = Series(date_range('20010101', periods=self.N, freq='T', tz='US/Eastern'))
         self.ts = self.s[self.halfway]
 
-        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))
+        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))

asv_bench/benchmarks/join_merge.py

@@ -6,7 +6,7 @@
     from pandas import ordered_merge as merge_ordered
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Append
 
 class Append(object):
@@ -35,7 +35,7 @@ def time_append_mixed(self):
         self.mdf1.append(self.mdf2)
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Concat
 
 class Concat(object):
@@ -120,7 +120,7 @@ def time_f_ordered_axis1(self):
         concat(self.frames_f, axis=1, ignore_index=True)
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Joins
 
 class Join(object):
@@ -202,7 +202,7 @@ def time_join_non_unique_equal(self):
         (self.fracofday * self.temp[self.fracofday.index])
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Merges
 
 class Merge(object):
@@ -257,7 +257,31 @@ def time_i8merge(self):
         merge(self.left, self.right, how='outer')
 
 
-#----------------------------------------------------------------------
+class MergeCategoricals(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.left_object = pd.DataFrame(
+            {'X': np.random.choice(range(0, 10), size=(10000,)),
+             'Y': np.random.choice(['one', 'two', 'three'], size=(10000,))})
+
+        self.right_object = pd.DataFrame(
+            {'X': np.random.choice(range(0, 10), size=(10000,)),
+             'Z': np.random.choice(['jjj', 'kkk', 'sss'], size=(10000,))})
+
+        self.left_cat = self.left_object.assign(
+            Y=self.left_object['Y'].astype('category'))
+        self.right_cat = self.right_object.assign(
+            Z=self.right_object['Z'].astype('category'))
+
+    def time_merge_object(self):
+        merge(self.left_object, self.right_object, on='X')
+
+    def time_merge_cat(self):
+        merge(self.left_cat, self.right_cat, on='X')
+
+
+# ----------------------------------------------------------------------
 # Ordered merge
 
 class MergeOrdered(object):
@@ -332,7 +356,7 @@ def time_multiby(self):
         merge_asof(self.df1e, self.df2e, on='time', by=['key', 'key2'])
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # data alignment
 
 class Align(object):

asv_bench/benchmarks/pandas_vb_common.py

@@ -8,16 +8,22 @@
 import random
 import numpy as np
 import threading
+from importlib import import_module
+
 try:
     from pandas.compat import range
 except ImportError:
     pass
 
 np.random.seed(1234)
-try:
-    import pandas._tseries as lib
-except:
-    import pandas.lib as lib
+
+# try em until it works!
+for imp in ['pandas_tseries', 'pandas.lib', 'pandas._libs.lib']:
+    try:
+        lib = import_module(imp)
+        break
+    except:
+        pass
 
 try:
     Panel = Panel

asv_bench/benchmarks/panel_methods.py

@@ -21,4 +21,4 @@ def time_shift(self):
         self.panel.shift(1)
 
     def time_shift_minor(self):
-        self.panel.shift(1, axis='minor')
+        self.panel.shift(1, axis='minor')

asv_bench/benchmarks/reshape.py

@@ -59,6 +59,27 @@ def time_reshape_unstack_simple(self):
         self.df.unstack(1)
 
 
+class reshape_unstack_large_single_dtype(object):
+    goal_time = 0.2
+
+    def setup(self):
+        m = 100
+        n = 1000
+
+        levels = np.arange(m)
+        index = pd.MultiIndex.from_product([levels]*2)
+        columns = np.arange(n)
+        values = np.arange(m*m*n).reshape(m*m, n)
+        self.df = pd.DataFrame(values, index, columns)
+        self.df2 = self.df.iloc[:-1]
+
+    def time_unstack_full_product(self):
+        self.df.unstack()
+
+    def time_unstack_with_mask(self):
+        self.df2.unstack()
+
+
 class unstack_sparse_keyspace(object):
     goal_time = 0.2
 

asv_bench/benchmarks/series_methods.py

@@ -68,8 +68,8 @@ def setup(self):
         self.s4 = self.s3.astype('object')
 
     def time_series_nlargest1(self):
-        self.s1.nlargest(3, take_last=True)
-        self.s1.nlargest(3, take_last=False)
+        self.s1.nlargest(3, keep='last')
+        self.s1.nlargest(3, keep='first')
 
 
 class series_nlargest2(object):
@@ -83,8 +83,8 @@ def setup(self):
         self.s4 = self.s3.astype('object')
 
     def time_series_nlargest2(self):
-        self.s2.nlargest(3, take_last=True)
-        self.s2.nlargest(3, take_last=False)
+        self.s2.nlargest(3, keep='last')
+        self.s2.nlargest(3, keep='first')
 
 
 class series_nsmallest2(object):
@@ -98,8 +98,8 @@ def setup(self):
         self.s4 = self.s3.astype('object')
 
     def time_series_nsmallest2(self):
-        self.s2.nsmallest(3, take_last=True)
-        self.s2.nsmallest(3, take_last=False)
+        self.s2.nsmallest(3, keep='last')
+        self.s2.nsmallest(3, keep='first')
 
 
 class series_dropna_int64(object):

ci/build_docs.sh

@@ -23,9 +23,6 @@ if [ x"$DOC_BUILD" != x"" ]; then
 
     source activate pandas
 
-    # install sudo deps
-    time sudo apt-get $APT_ARGS install dvipng texlive-latex-base texlive-latex-extra
-
     mv "$TRAVIS_BUILD_DIR"/doc /tmp
     cd /tmp/doc
 

ci/check_cache.sh

@@ -12,14 +12,12 @@ else
     ci_changes=$(git diff PR_HEAD~2 --numstat | grep -E "ci/"| wc -l)
 fi
 
-MINICONDA_DIR="$HOME/miniconda/"
 CACHE_DIR="$HOME/.cache/"
 CCACHE_DIR="$HOME/.ccache/"
 
 if [ $ci_changes -ne 0 ]
 then
     echo "Files have changed in ci/ deleting all caches"
-    rm -rf "$MINICONDA_DIR"
     rm -rf "$CACHE_DIR"
     rm -rf "$CCACHE_DIR"
-fi
+fi

ci/install_travis.sh

@@ -32,41 +32,53 @@ edit_init
 home_dir=$(pwd)
 echo "[home_dir: $home_dir]"
 
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    echo "[install ccache]"
+    time brew install ccache
+fi
+
 # install miniconda
+MINICONDA_DIR="$HOME/miniconda3"
+
 echo "[Using clean Miniconda install]"
 
-MINICONDA_DIR="$HOME/miniconda3"
 if [ -d "$MINICONDA_DIR" ]; then
     rm -rf "$MINICONDA_DIR"
 fi
 
 # install miniconda
 if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
-    wget http://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1
+    time wget http://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1
 else
-    wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh || exit 1
+    time wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh || exit 1
 fi
-bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1
+time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1
+
+echo "[show conda]"
+which conda
 
 echo "[update conda]"
 conda config --set ssl_verify false || exit 1
 conda config --set always_yes true --set changeps1 false || exit 1
 conda update -q conda
 
+echo "[add channels]"
 # add the pandas channel to take priority
 # to add extra packages
-echo "[add channels]"
 conda config --add channels pandas || exit 1
 conda config --remove channels defaults || exit 1
 conda config --add channels defaults || exit 1
 
-conda install anaconda-client
+if [ "$CONDA_FORGE" ]; then
+    # add conda-forge channel as priority
+    conda config --add channels conda-forge || exit 1
+fi
 
 # Useful for debugging any issues with conda
 conda info -a || exit 1
 
 # set the compiler cache to work
-if [ "$USE_CACHE" ] && "${TRAVIS_OS_NAME}" == "linux" ]; then
+if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then
     echo "[Using ccache]"
     export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
     gcc=$(which gcc)
@@ -78,13 +90,16 @@ else
     echo "[Not using ccache]"
 fi
 
+echo "[create env]"
+
 # may have installation instructions for this build
 INSTALL="ci/install-${PYTHON_VERSION}${JOB_TAG}.sh"
 if [ -e ${INSTALL} ]; then
     time bash $INSTALL || exit 1
 else
     # create new env
-    time conda create -n pandas python=$PYTHON_VERSION pytest || exit 1
+    # this may already exists, in which case our caching worked
+    time conda create -n pandas python=$PYTHON_VERSION pytest nomkl
 fi
 
 # build deps