Merge branch 'master' into PR_TOOL_MERGE_PR_18852

Author: jreback

.travis.yml

@@ -49,6 +49,7 @@ matrix:
         apt:
           packages:
           - python-gtk2
+    # In allow_failures
     - dist: trusty
       env:
         - JOB="3.5_CONDA_BUILD_TEST" TEST_ARGS="--skip-slow --skip-network" CONDA_BUILD_TEST=true
@@ -76,6 +77,9 @@ matrix:
       env:
         - JOB="3.6_DOC" DOC=true
     allow_failures:
+      - dist: trusty
+        env:
+          - JOB="3.5_CONDA_BUILD_TEST" TEST_ARGS="--skip-slow --skip-network" CONDA_BUILD_TEST=true
       - dist: trusty
         env:
           - JOB="2.7_SLOW" SLOW=true
@@ -95,6 +99,9 @@ matrix:
 
 before_install:
   - echo "before_install"
+  # set non-blocking IO on travis
+  # https://github.com/travis-ci/travis-ci/issues/8920#issuecomment-352661024
+  - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
   - source ci/travis_process_gbq_encryption.sh
   - export PATH="$HOME/miniconda3/bin:$PATH"
   - df -h

asv_bench/benchmarks/frame_methods.py

@@ -4,7 +4,7 @@
 from pandas import (DataFrame, Series, MultiIndex, date_range, period_range,
                     isnull, NaT)
 
-from .pandas_vb_common import setup # noqa
+from .pandas_vb_common import setup  # noqa
 
 
 class GetNumericData(object):
@@ -127,7 +127,7 @@ class ToHTML(object):
     def setup(self):
         nrows = 500
         self.df2 = DataFrame(np.random.randn(nrows, 10))
-        self.df2[0] = period_range('2000', '2010', nrows)
+        self.df2[0] = period_range('2000', periods=nrows)
         self.df2[1] = range(nrows)
 
     def time_to_html_mixed(self):

asv_bench/benchmarks/groupby.py

@@ -1,4 +1,4 @@
-from string import ascii_letters, digits
+from string import ascii_letters
 from itertools import product
 from functools import partial
 
@@ -275,18 +275,12 @@ class GroupStrings(object):
 
     def setup(self):
         n = 2 * 10**5
-        alpha = list(map(''.join, product((ascii_letters + digits), repeat=4)))
-        self.df = DataFrame({'a': np.repeat(np.random.choice(alpha,
-                                                             (n // 11)), 11),
-                             'b': np.repeat(np.random.choice(alpha,
-                                                             (n // 7)), 7),
-                             'c': np.repeat(np.random.choice(alpha,
-                                                             (n // 5)), 5),
-                             'd': np.repeat(np.random.choice(alpha,
-                                                             (n // 1)), 1)})
+        alpha = list(map(''.join, product(ascii_letters, repeat=4)))
+        data = np.random.choice(alpha, (n // 5, 4), replace=False)
+        data = np.repeat(data, 5, axis=0)
+        self.df = DataFrame(data, columns=list('abcd'))
         self.df['joe'] = (np.random.randn(len(self.df)) * 10).round(3)
-        i = np.random.permutation(len(self.df))
-        self.df = self.df.iloc[i].reset_index(drop=True)
+        self.df = self.df.sample(frac=1).reset_index(drop=True)
 
     def time_multi_columns(self):
         self.df.groupby(list('abcd')).max()
@@ -356,10 +350,16 @@ class GroupByMethods(object):
 
     goal_time = 0.2
 
-    param_names = ['dtype', 'ngroups']
-    params = [['int', 'float'], [100, 10000]]
+    param_names = ['dtype', 'method']
+    params = [['int', 'float'],
+              ['all', 'any', 'count', 'cumcount', 'cummax', 'cummin',
+               'cumprod', 'cumsum', 'describe', 'first', 'head', 'last', 'mad',
+               'max', 'min', 'median', 'mean', 'nunique', 'pct_change', 'prod',
+               'rank', 'sem', 'shift', 'size', 'skew', 'std', 'sum', 'tail',
+               'unique', 'value_counts', 'var']]
 
-    def setup(self, dtype, ngroups):
+    def setup(self, dtype, method):
+        ngroups = 1000
         size = ngroups * 2
         rng = np.arange(ngroups)
         values = rng.take(np.random.randint(0, ngroups, size=size))
@@ -369,104 +369,11 @@ def setup(self, dtype, ngroups):
             key = np.concatenate([np.random.random(ngroups) * 0.1,
                                   np.random.random(ngroups) * 10.0])
 
-        self.df = DataFrame({'values': values,
-                             'key': key})
+        df = DataFrame({'values': values, 'key': key})
+        self.df_groupby_method = getattr(df.groupby('key')['values'], method)
 
-    def time_all(self, dtype, ngroups):
-        self.df.groupby('key')['values'].all()
-
-    def time_any(self, dtype, ngroups):
-        self.df.groupby('key')['values'].any()
-
-    def time_count(self, dtype, ngroups):
-        self.df.groupby('key')['values'].count()
-
-    def time_cumcount(self, dtype, ngroups):
-        self.df.groupby('key')['values'].cumcount()
-
-    def time_cummax(self, dtype, ngroups):
-        self.df.groupby('key')['values'].cummax()
-
-    def time_cummin(self, dtype, ngroups):
-        self.df.groupby('key')['values'].cummin()
-
-    def time_cumprod(self, dtype, ngroups):
-        self.df.groupby('key')['values'].cumprod()
-
-    def time_cumsum(self, dtype, ngroups):
-        self.df.groupby('key')['values'].cumsum()
-
-    def time_describe(self, dtype, ngroups):
-        self.df.groupby('key')['values'].describe()
-
-    def time_diff(self, dtype, ngroups):
-        self.df.groupby('key')['values'].diff()
-
-    def time_first(self, dtype, ngroups):
-        self.df.groupby('key')['values'].first()
-
-    def time_head(self, dtype, ngroups):
-        self.df.groupby('key')['values'].head()
-
-    def time_last(self, dtype, ngroups):
-        self.df.groupby('key')['values'].last()
-
-    def time_mad(self, dtype, ngroups):
-        self.df.groupby('key')['values'].mad()
-
-    def time_max(self, dtype, ngroups):
-        self.df.groupby('key')['values'].max()
-
-    def time_mean(self, dtype, ngroups):
-        self.df.groupby('key')['values'].mean()
-
-    def time_median(self, dtype, ngroups):
-        self.df.groupby('key')['values'].median()
-
-    def time_min(self, dtype, ngroups):
-        self.df.groupby('key')['values'].min()
-
-    def time_nunique(self, dtype, ngroups):
-        self.df.groupby('key')['values'].nunique()
-
-    def time_pct_change(self, dtype, ngroups):
-        self.df.groupby('key')['values'].pct_change()
-
-    def time_prod(self, dtype, ngroups):
-        self.df.groupby('key')['values'].prod()
-
-    def time_rank(self, dtype, ngroups):
-        self.df.groupby('key')['values'].rank()
-
-    def time_sem(self, dtype, ngroups):
-        self.df.groupby('key')['values'].sem()
-
-    def time_shift(self, dtype, ngroups):
-        self.df.groupby('key')['values'].shift()
-
-    def time_size(self, dtype, ngroups):
-        self.df.groupby('key')['values'].size()
-
-    def time_skew(self, dtype, ngroups):
-        self.df.groupby('key')['values'].skew()
-
-    def time_std(self, dtype, ngroups):
-        self.df.groupby('key')['values'].std()
-
-    def time_sum(self, dtype, ngroups):
-        self.df.groupby('key')['values'].sum()
-
-    def time_tail(self, dtype, ngroups):
-        self.df.groupby('key')['values'].tail()
-
-    def time_unique(self, dtype, ngroups):
-        self.df.groupby('key')['values'].unique()
-
-    def time_value_counts(self, dtype, ngroups):
-        self.df.groupby('key')['values'].value_counts()
-
-    def time_var(self, dtype, ngroups):
-        self.df.groupby('key')['values'].var()
+    def time_method(self, dtype, method):
+        self.df_groupby_method()
 
 
 class Float32(object):

asv_bench/benchmarks/timestamp.py

@@ -1,10 +1,10 @@
+import datetime
+
 from pandas import Timestamp
 import pytz
-import datetime
 
 
 class TimestampConstruction(object):
-    # TODO: classmethod constructors: fromordinal, fromtimestamp...
 
     def time_parse_iso8601_no_tz(self):
         Timestamp('2017-08-25 08:16:14')
@@ -21,6 +21,12 @@ def time_parse_today(self):
     def time_parse_now(self):
         Timestamp('now')
 
+    def time_fromordinal(self):
+        Timestamp.fromordinal(730120)
+
+    def time_fromtimestamp(self):
+        Timestamp.fromtimestamp(1515448538)
+
 
 class TimestampProperties(object):
     goal_time = 0.2
@@ -36,9 +42,6 @@ def setup(self, tz, freq):
     def time_tz(self, tz, freq):
         self.ts.tz
 
-    def time_offset(self, tz, freq):
-        self.ts.offset
-
     def time_dayofweek(self, tz, freq):
         self.ts.dayofweek
 

ci/install_travis.sh

@@ -101,9 +101,6 @@ time conda create -n pandas --file=${REQ} || exit 1
 
 source activate pandas
 
-# https://github.com/travis-ci/travis-ci/issues/8920#issuecomment-352661024
-python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
-
 # may have addtl installation instructions for this build
 echo
 echo "[build addtl installs]"

ci/requirements-2.7.build

@@ -2,5 +2,5 @@ python=2.7*
 python-dateutil=2.5.0
 pytz=2013b
 nomkl
-numpy
+numpy=1.13*
 cython=0.24

ci/requirements-3.6_DOC.build

@@ -1,5 +1,5 @@
 python=3.6*
 python-dateutil
 pytz
-numpy
+numpy=1.13*
 cython

conda.recipe/meta.yaml

@@ -14,14 +14,14 @@ requirements:
   build:
     - python
     - cython
-    - {{ pin_compatible('numpy') }}
+    - {{ pin_compatible('numpy', upper_bound='1.14') }}
     - setuptools >=3.3
     - python-dateutil >=2.5.0
     - pytz
 
   run:
     - python
-    - {{ pin_compatible('numpy') }}
+    - {{ pin_compatible('numpy', upper_bound='1.14') }}
     - python-dateutil >=2.5.0
     - pytz
 

doc/source/contributing.rst

@@ -858,9 +858,9 @@ takes a regular expression.  For example, this will only run tests from a
 If you want to only run a specific group of tests from a file, you can do it
 using ``.`` as a separator. For example::
 
-    asv continuous -f 1.1 upstream/master HEAD -b groupby.groupby_agg_builtins
+    asv continuous -f 1.1 upstream/master HEAD -b groupby.GroupByMethods
 
-will only run the ``groupby_agg_builtins`` benchmark defined in ``groupby.py``.
+will only run the ``GroupByMethods`` benchmark defined in ``groupby.py``.
 
 You can also run the benchmark suite using the version of ``pandas``
 already installed in your current Python environment. This can be

doc/source/io.rst

@@ -149,7 +149,7 @@ squeeze : boolean, default ``False``
 prefix : str, default ``None``
   Prefix to add to column numbers when no header, e.g. 'X' for X0, X1, ...
 mangle_dupe_cols : boolean, default ``True``
-  Duplicate columns will be specified as 'X.0'...'X.N', rather than 'X'...'X'.
+  Duplicate columns will be specified as 'X', 'X.1'...'X.N', rather than 'X'...'X'.
   Passing in False will cause data to be overwritten if there are duplicate
   names in the columns.
 
@@ -548,7 +548,7 @@ these names so as to prevent data overwrite:
    pd.read_csv(StringIO(data))
 
 There is no more duplicate data because ``mangle_dupe_cols=True`` by default, which modifies
-a series of duplicate columns 'X'...'X' to become 'X.0'...'X.N'.  If ``mangle_dupe_cols
+a series of duplicate columns 'X'...'X' to become 'X', 'X.1',...'X.N'.  If ``mangle_dupe_cols
 =False``, duplicate data can arise:
 
 .. code-block :: python

doc/source/whatsnew/v0.23.0.txt

@@ -241,6 +241,7 @@ Other Enhancements
 - ``Resampler`` objects now have a functioning :attr:`~pandas.core.resample.Resampler.pipe` method.
   Previously, calls to ``pipe`` were diverted to  the ``mean`` method (:issue:`17905`).
 - :func:`~pandas.api.types.is_scalar` now returns ``True`` for ``DateOffset`` objects (:issue:`18943`).
+- ``IntervalIndex.astype`` now supports conversions between subtypes when passed an ``IntervalDtype`` (:issue:`19197`)
 
 .. _whatsnew_0230.api_breaking:
 
@@ -309,6 +310,8 @@ Other API Changes
 - Subtraction of :class:`Series` with timezone-aware ``dtype='datetime64[ns]'`` with mis-matched timezones will raise ``TypeError`` instead of ``ValueError`` (issue:`18817`)
 - :class:`IntervalIndex` and ``IntervalDtype`` no longer support categorical, object, and string subtypes (:issue:`19016`)
 - The default ``Timedelta`` constructor now accepts an ``ISO 8601 Duration`` string as an argument (:issue:`19040`)
+- ``IntervalDtype`` now returns ``True`` when compared against ``'interval'`` regardless of subtype, and ``IntervalDtype.name`` now returns ``'interval'`` regardless of subtype (:issue:`18980`)
+- :func:`Series.to_csv` now accepts a ``compression`` argument that works in the same way as the ``compression`` argument in :func:`DataFrame.to_csv` (:issue:`18958`)
 
 .. _whatsnew_0230.deprecations:
 
@@ -353,6 +356,9 @@ Removal of prior version deprecations/changes
 - The ``Panel4D`` and ``PanelND`` classes have been removed (:issue:`13776`)
 - The ``Panel``class has dropped the ``to_long``and ``toLong`` methods (:issue:`19077`)
 - The options ``display.line_with`` and ``display.height`` are removed in favor of ``display.width`` and ``display.max_rows`` respectively (:issue:`4391`, :issue:`19107`)
+- The ``labels`` attribute of the ``Categorical`` class has been removed in favor of :attribute:`Categorical.codes` (:issue:`7768`)
+- The ``flavor`` parameter have been removed from func:`to_sql` method (:issue:`13611`)
+- The modules `pandas.tools.hashing` and `pandas.util.hashing` have been removed (:issue:`16223`)
 
 .. _whatsnew_0230.performance:
 
@@ -416,7 +422,14 @@ Conversion
 - Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`)
 - Bug in :class:`DatetimeIndex` where the repr was not showing high-precision time values at the end of a day (e.g., 23:59:59.999999999) (:issue:`19030`)
 - Bug where dividing a scalar timedelta-like object with :class:`TimedeltaIndex` performed the reciprocal operation (:issue:`19125`)
--
+- Bug in :class:`WeekOfMonth` and :class:`LastWeekOfMonth` where default keyword arguments for constructor raised ``ValueError`` (:issue:`19142`)
+- Bug in localization of a naive, datetime string in a ``Series`` constructor with a ``datetime64[ns, tz]`` dtype (:issue:`174151`)
+- :func:`Timestamp.replace` will now handle Daylight Savings transitions gracefully (:issue:`18319`)
+
+
+
+- Bug in ``.astype()`` to non-ns timedelta units would hold the incorrect dtype (:issue:`19176`, :issue:`19223`, :issue:`12425`)
+
 
 Indexing
 ^^^^^^^^
@@ -437,8 +450,10 @@ Indexing
 - Bug in indexing non-scalar value from ``Series`` having non-unique ``Index`` will return value flattened (:issue:`17610`)
 - Bug in :func:`DatetimeIndex.insert` where inserting ``NaT`` into a timezone-aware index incorrectly raised (:issue:`16357`)
 - Bug in ``__setitem__`` when indexing a :class:`DataFrame` with a 2-d boolean ndarray (:issue:`18582`)
+- Bug in :func:`MultiIndex.__contains__` where non-tuple keys would return ``True`` even if they had been dropped (:issue:`19027`)
 - Bug in :func:`MultiIndex.set_labels` which would cause casting (and potentially clipping) of the new labels if the ``level`` argument is not 0 or a list like [0, 1, ... ]  (:issue:`19057`)
 - Bug in ``str.extractall`` when there were no matches empty :class:`Index` was returned instead of appropriate :class:`MultiIndex` (:issue:`19034`)
+- Bug in :class:`IntervalIndex` where set operations that returned an empty ``IntervalIndex`` had the wrong dtype (:issue:`19101`)
 
 I/O
 ^^^
@@ -483,6 +498,8 @@ Reshaping
 - Bug in :func:`cut` which fails when using readonly arrays (:issue:`18773`)
 - Bug in :func:`Dataframe.pivot_table` which fails when the ``aggfunc`` arg is of type string.  The behavior is now consistent with other methods like ``agg`` and ``apply`` (:issue:`18713`)
 - Bug in :func:`DataFrame.merge` in which merging using ``Index`` objects as vectors raised an Exception (:issue:`19038`)
+- Bug in :func:`DataFrame.stack`, :func:`DataFrame.unstack`, :func:`Series.unstack` which were not returning subclasses (:issue:`15563`)
+-
 
 Numeric
 ^^^^^^^
@@ -507,4 +524,3 @@ Other
 ^^^^^
 
 - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)
-- :func:`Timestamp.replace` will now handle Daylight Savings transitions gracefully (:issue:`18319`)