Merge branch 'master' into bug_issue16770

Author: ri938

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
  - [ ] closes #xxxx
  - [ ] tests added / passed
- - [ ] passes ``git diff upstream/master --name-only -- '*.py' | flake8 --diff``
+ - [ ] passes ``git diff upstream/master --name-only -- '*.py' | flake8 --diff`` (On Windows, ``git diff upstream/master -u -- "*.py" | flake8 --diff`` might work as an alternative.)
  - [ ] whatsnew entry

.travis.yml

@@ -35,61 +35,61 @@ matrix:
       language: generic
       env:
         - JOB="3.5_OSX" TEST_ARGS="--skip-slow --skip-network"
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8"
       addons:
         apt:
           packages:
           - language-pack-zh-hans
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7" TEST_ARGS="--skip-slow" LINT=true
       addons:
         apt:
           packages:
           - python-gtk2
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.5" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true
       addons:
         apt:
           packages:
           - xsel
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.6" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.6_DOC" DOC=true
       addons:
         apt:
           packages:
           - xsel
     allow_failures:
-      - os: linux
+      - dist: trusty
         env:
           - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
-      - os: linux
+      - dist: trusty
         env:
           - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
-      - os: linux
+      - dist: trusty
         env:
           - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
-      - os: linux
+      - dist: trusty
         env:
           - JOB="3.6_DOC" DOC=true
 

ci/requirements-2.7.sh

@@ -4,4 +4,4 @@ source activate pandas
 
 echo "install 27"
 
-conda install -n pandas -c conda-forge feather-format jemalloc=4.4.0
+conda install -n pandas -c conda-forge feather-format

ci/requirements-2.7_BUILD_TEST.sh

@@ -4,4 +4,4 @@ source activate pandas
 
 echo "install 27 BUILD_TEST"
 
-conda install -n pandas -c conda-forge pyarrow dask jemalloc=4.4.0
+conda install -n pandas -c conda-forge pyarrow dask

ci/requirements-3.5.sh

@@ -4,4 +4,4 @@ source activate pandas
 
 echo "install 35"
 
-conda install -n pandas -c conda-forge feather-format jemalloc=4.4.0
+conda install -n pandas -c conda-forge feather-format

ci/requirements-3.6.run

@@ -14,7 +14,6 @@ html5lib
 jinja2
 sqlalchemy
 pymysql
-jemalloc=4.4.0
 feather-format
 # psycopg2 (not avail on defaults ATM)
 beautifulsoup4

ci/requirements-3.6_DOC.sh

@@ -6,6 +6,6 @@ echo "[install DOC_BUILD deps]"
 
 pip install pandas-gbq
 
-conda install -n pandas -c conda-forge feather-format nbsphinx pandoc jemalloc=4.4.0
+conda install -n pandas -c conda-forge feather-format nbsphinx pandoc
 
 conda install -n pandas -c r r rpy2 --yes

doc/source/contributing.rst

@@ -525,6 +525,12 @@ run this slightly modified command::
 
    git diff master --name-only -- '*.py' | grep 'pandas/' | xargs flake8
 
+Note that on Windows, ``grep``, ``xargs``, and other tools are likely
+unavailable. However, this has been shown to work on smaller commits in the
+standard Windows command line::
+
+   git diff master -u -- "*.py" | flake8 --diff
+
 Backwards Compatibility
 ~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/v0.20.3.txt

@@ -37,15 +37,18 @@ Performance Improvements
 Bug Fixes
 ~~~~~~~~~
 - Fixed issue with dataframe scatter plot for categorical data that reports incorrect column key not found when categorical data is used for plotting (:issue:`16199`)
+- Fixed issue with :meth:`DataFrame.style` where element id's were not unique (:issue:`16780`)
+- Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`)
+- Fixed compat with loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`)
+- Fixed bug where computing the rolling covariance of a MultiIndexed ``DataFrame`` improperly raised a ``ValueError`` (:issue:`16789`)
 - Handle reindexing an empty categorical index rather than throwing (:issue:`16770`)
 
-
-
 Conversion
 ^^^^^^^^^^
 
 - Bug in pickle compat prior to the v0.20.x series, when ``UTC`` is a timezone in a Series/DataFrame/Index (:issue:`16608`)
-- Bug in Series construction when passing a Series with ``dtype='category'`` (:issue:`16524`).
+- Bug in ``Series`` construction when passing a ``Series`` with ``dtype='category'`` (:issue:`16524`).
+- Bug in ``DataFrame.astype()`` when passing a ``Series`` as the ``dtype`` kwarg. (:issue:`16717`).
 
 Indexing
 ^^^^^^^^
@@ -55,8 +58,9 @@ Indexing
 I/O
 ^^^
 
-- Bug in :func:`read_csv`` in which files weren't opened as binary files by the C engine on Windows, causing EOF characters mid-field, which would fail (:issue:`16039`, :issue:`16559`, :issue:`16675`)
-- Bug in :func:`read_hdf`` in which reading a ``Series`` saved to an HDF file in 'fixed' format fails when an explicit ``mode='r'`` argument is supplied (:issue:`16583`)
+- Bug in :func:`read_csv` in which files weren't opened as binary files by the C engine on Windows, causing EOF characters mid-field, which would fail (:issue:`16039`, :issue:`16559`, :issue:`16675`)
+- Bug in :func:`read_hdf` in which reading a ``Series`` saved to an HDF file in 'fixed' format fails when an explicit ``mode='r'`` argument is supplied (:issue:`16583`)
+- Bug in :func:`DataFrame.to_latex` where ``bold_rows`` was wrongly specified to be ``True`` by default, whereas in reality row labels remained non-bold whatever parameter provided. (:issue:`16707`)
 
 Plotting
 ^^^^^^^^
@@ -78,6 +82,8 @@ Sparse
 Reshaping
 ^^^^^^^^^
 
+- Bug in joining on a ``MultiIndex`` with a ``category`` dtype for a level (:issue:`16627`).
+- Bug in :func:`merge` when merging/joining with multiple categorical columns (:issue:`16767`)
 
 
 Numeric

doc/source/whatsnew/v0.21.0.txt

@@ -92,6 +92,8 @@ Performance Improvements
 Bug Fixes
 ~~~~~~~~~
 
+- Fixes regression in 0.20, :func:`Series.aggregate` and :func:`DataFrame.aggregate` allow dictionaries as return values again (:issue:`16741`)
+
 Conversion
 ^^^^^^^^^^
 
@@ -133,6 +135,7 @@ Reshaping
 
 Numeric
 ^^^^^^^
+- Bug in ``.clip()`` with ``axis=1`` and a list-like for ``threshold`` is passed; previously this raised ``ValueError`` (:issue:`15390`)
 
 
 Categorical

pandas/_libs/src/reduce.pyx

@@ -419,7 +419,7 @@ cdef class SeriesGrouper:
 cdef inline _extract_result(object res):
     """ extract the result object, it might be a 0-dim ndarray
         or a len-1 0-dim, or a scalar """
-    if hasattr(res, 'values'):
+    if hasattr(res, 'values') and isinstance(res.values, np.ndarray):
         res = res.values
     if not np.isscalar(res):
         if isinstance(res, np.ndarray):

pandas/core/generic.py

@@ -52,6 +52,7 @@
 from pandas.compat.numpy import function as nv
 from pandas.compat import (map, zip, lzip, lrange, string_types,
                            isidentifier, set_function_name, cPickle as pkl)
+from pandas.core.ops import _align_method_FRAME
 import pandas.core.nanops as nanops
 from pandas.util._decorators import Appender, Substitution, deprecate_kwarg
 from pandas.util._validators import validate_bool_kwarg
@@ -1538,7 +1539,7 @@ def to_xarray(self):
 
         `to_latex`-specific options:
 
-        bold_rows : boolean, default True
+        bold_rows : boolean, default False
             Make the row labels bold in the output
         column_format : str, default None
             The columns format as specified in `LaTeX table format
@@ -1587,7 +1588,7 @@ def to_xarray(self):
     @Appender(_shared_docs['to_latex'] % _shared_doc_kwargs)
     def to_latex(self, buf=None, columns=None, col_space=None, header=True,
                  index=True, na_rep='NaN', formatters=None, float_format=None,
-                 sparsify=None, index_names=True, bold_rows=True,
+                 sparsify=None, index_names=True, bold_rows=False,
                  column_format=None, longtable=None, escape=None,
                  encoding=None, decimal='.', multicolumn=None,
                  multicolumn_format=None, multirow=None):
@@ -3507,12 +3508,12 @@ def astype(self, dtype, copy=True, errors='raise', **kwargs):
         -------
         casted : type of caller
         """
-        if isinstance(dtype, collections.Mapping):
+        if is_dict_like(dtype):
             if self.ndim == 1:  # i.e. Series
-                if len(dtype) > 1 or list(dtype.keys())[0] != self.name:
+                if len(dtype) > 1 or self.name not in dtype:
                     raise KeyError('Only the Series name can be used for '
                                    'the key in Series dtype mappings.')
-                new_type = list(dtype.values())[0]
+                new_type = dtype[self.name]
                 return self.astype(new_type, copy, errors, **kwargs)
             elif self.ndim > 2:
                 raise NotImplementedError(
@@ -4413,6 +4414,34 @@ def _clip_with_scalar(self, lower, upper, inplace=False):
         else:
             return result
 
+    def _clip_with_one_bound(self, threshold, method, axis, inplace):
+
+        inplace = validate_bool_kwarg(inplace, 'inplace')
+        if axis is not None:
+            axis = self._get_axis_number(axis)
+
+        if np.any(isnull(threshold)):
+            raise ValueError("Cannot use an NA value as a clip threshold")
+
+        # method is self.le for upper bound and self.ge for lower bound
+        if is_scalar(threshold) and is_number(threshold):
+            if method.__name__ == 'le':
+                return self._clip_with_scalar(None, threshold, inplace=inplace)
+            return self._clip_with_scalar(threshold, None, inplace=inplace)
+
+        subset = method(threshold, axis=axis) | isnull(self)
+
+        # GH #15390
+        # In order for where method to work, the threshold must
+        # be transformed to NDFrame from other array like structure.
+        if (not isinstance(threshold, ABCSeries)) and is_list_like(threshold):
+            if isinstance(self, ABCSeries):
+                threshold = pd.Series(threshold, index=self.index)
+            else:
+                threshold = _align_method_FRAME(self, np.asarray(threshold),
+                                                axis)
+        return self.where(subset, threshold, axis=axis, inplace=inplace)
+
     def clip(self, lower=None, upper=None, axis=None, inplace=False,
              *args, **kwargs):
         """
@@ -4515,16 +4544,8 @@ def clip_upper(self, threshold, axis=None, inplace=False):
         -------
         clipped : same type as input
         """
-        if np.any(isnull(threshold)):
-            raise ValueError("Cannot use an NA value as a clip threshold")
-
-        if is_scalar(threshold) and is_number(threshold):
-            return self._clip_with_scalar(None, threshold, inplace=inplace)
-
-        inplace = validate_bool_kwarg(inplace, 'inplace')
-
-        subset = self.le(threshold, axis=axis) | isnull(self)
-        return self.where(subset, threshold, axis=axis, inplace=inplace)
+        return self._clip_with_one_bound(threshold, method=self.le,
+                                         axis=axis, inplace=inplace)
 
     def clip_lower(self, threshold, axis=None, inplace=False):
         """
@@ -4547,16 +4568,8 @@ def clip_lower(self, threshold, axis=None, inplace=False):
         -------
         clipped : same type as input
         """
-        if np.any(isnull(threshold)):
-            raise ValueError("Cannot use an NA value as a clip threshold")
-
-        if is_scalar(threshold) and is_number(threshold):
-            return self._clip_with_scalar(threshold, None, inplace=inplace)
-
-        inplace = validate_bool_kwarg(inplace, 'inplace')
-
-        subset = self.ge(threshold, axis=axis) | isnull(self)
-        return self.where(subset, threshold, axis=axis, inplace=inplace)
+        return self._clip_with_one_bound(threshold, method=self.ge,
+                                         axis=axis, inplace=inplace)
 
     def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,
                 group_keys=True, squeeze=False, **kwargs):

pandas/core/indexes/category.py

@@ -565,6 +565,9 @@ def take(self, indices, axis=0, allow_fill=True,
                                            na_value=-1)
         return self._create_from_codes(taken)
 
+    def is_dtype_equal(self, other):
+        return self._data.is_dtype_equal(other)
+
     take_nd = take
 
     def map(self, mapper):

pandas/core/reshape/merge.py

@@ -1440,13 +1440,14 @@ def _factorize_keys(lk, rk, sort=True):
         lk = lk.values
         rk = rk.values
 
-    # if we exactly match in categories, allow us to use codes
+    # if we exactly match in categories, allow us to factorize on codes
     if (is_categorical_dtype(lk) and
             is_categorical_dtype(rk) and
             lk.is_dtype_equal(rk)):
-        return lk.codes, rk.codes, len(lk.categories)
-
-    if is_int_or_datetime_dtype(lk) and is_int_or_datetime_dtype(rk):
+        klass = libhashtable.Int64Factorizer
+        lk = _ensure_int64(lk.codes)
+        rk = _ensure_int64(rk.codes)
+    elif is_int_or_datetime_dtype(lk) and is_int_or_datetime_dtype(rk):
         klass = libhashtable.Int64Factorizer
         lk = _ensure_int64(com._values_from_object(lk))
         rk = _ensure_int64(com._values_from_object(rk))

pandas/core/window.py

@@ -1948,7 +1948,7 @@ def dataframe_from_int_dict(data, frame_template):
                 result.columns = Index(result.columns).set_names(
                     arg2.columns.name)
                 result.index = result.index.set_names(
-                    [arg1.index.name, arg1.columns.name])
+                    arg1.index.names + arg1.columns.names)
 
                 return result
 

pandas/io/formats/format.py

@@ -845,6 +845,7 @@ def __init__(self, formatter, column_format=None, longtable=False,
                  multicolumn=False, multicolumn_format=None, multirow=False):
         self.fmt = formatter
         self.frame = self.fmt.frame
+        self.bold_rows = self.fmt.kwds.get('bold_rows', False)
         self.column_format = column_format
         self.longtable = longtable
         self.multicolumn = multicolumn
@@ -943,6 +944,11 @@ def get_col_type(dtype):
                          if x else '{}') for x in row]
             else:
                 crow = [x if x else '{}' for x in row]
+            if self.bold_rows and self.fmt.index:
+                # bold row labels
+                crow = ['\\textbf{%s}' % x
+                        if j < ilevels and x.strip() not in ['', '{}'] else x
+                        for j, x in enumerate(crow)]
             if i < clevels and self.fmt.header and self.multicolumn:
                 # sum up columns to multicolumns
                 crow = self._format_multicolumn(crow, ilevels)

pandas/io/formats/style.py

@@ -281,13 +281,14 @@ def format_attr(pair):
         for r, idx in enumerate(self.data.index):
             row_es = []
             for c, value in enumerate(rlabels[r]):
+                rid = [ROW_HEADING_CLASS, "level%s" % c, "row%s" % r]
                 es = {
                     "type": "th",
                     "is_visible": _is_visible(r, c, idx_lengths),
                     "value": value,
                     "display_value": value,
-                    "class": " ".join([ROW_HEADING_CLASS, "level%s" % c,
-                                       "row%s" % r]),
+                    "id": "_".join(rid[1:]),
+                    "class": " ".join(rid)
                 }
                 rowspan = idx_lengths.get((c, r), 0)
                 if rowspan > 1:

pandas/io/pytables.py

@@ -2591,8 +2591,8 @@ def read_index_node(self, node, start=None, stop=None):
         if 'name' in node._v_attrs:
             name = _ensure_str(node._v_attrs.name)
 
-        index_class = self._alias_to_class(getattr(node._v_attrs,
-                                                   'index_class', ''))
+        index_class = self._alias_to_class(_ensure_decoded(
+            getattr(node._v_attrs, 'index_class', '')))
         factory = self._get_index_factory(index_class)
 
         kwargs = {}