Merge remote-tracking branch 'upstream/master' into 26760-converter

Author: TomAugspurger

.travis.yml

@@ -103,10 +103,5 @@ script:
 after_script:
   - echo "after_script start"
   - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - if [ -e test-data-single.xml ]; then
-        ci/print_skipped.py test-data-single.xml;
-    fi
-  - if [ -e test-data-multiple.xml ]; then
-        ci/print_skipped.py test-data-multiple.xml;
-    fi
+  - ci/print_skipped.py 
   - echo "after_script done"

asv_bench/asv.conf.json

@@ -107,7 +107,7 @@
     // `asv` will cache wheels of the recent builds in each
     // environment, making them faster to install next time.  This is
     // number of builds to keep, per environment.
-    "wheel_cache_size": 8,
+    "build_cache_size": 8,
 
     // The commits after which the regression search in `asv publish`
     // should start looking for regressions. Dictionary whose keys are

asv_bench/benchmarks/index_object.py

@@ -196,6 +196,9 @@ def setup(self, N):
         self.intv = IntervalIndex.from_arrays(left, right)
         self.intv._engine
 
+        self.intv2 = IntervalIndex.from_arrays(left + 1, right + 1)
+        self.intv2._engine
+
         self.left = IntervalIndex.from_breaks(np.arange(N))
         self.right = IntervalIndex.from_breaks(np.arange(N - 3, 2 * N - 3))
 
@@ -208,8 +211,11 @@ def time_is_unique(self, N):
     def time_intersection(self, N):
         self.left.intersection(self.right)
 
-    def time_intersection_duplicate(self, N):
+    def time_intersection_one_duplicate(self, N):
         self.intv.intersection(self.right)
 
+    def time_intersection_both_duplicate(self, N):
+        self.intv.intersection(self.intv2)
+
 
 from .pandas_vb_common import setup  # noqa: F401

ci/azure/posix.yml

@@ -89,4 +89,9 @@ jobs:
           # note that this will produce $LASTEXITCODE=1
           Write-Error "$($matches[1]) tests failed"
         }
-      displayName: Check for test failures
+      displayName: 'Check for test failures'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
+        python ci/print_skipped.py
+      displayName: 'Print skipped tests'

ci/azure/windows.yml

@@ -18,11 +18,11 @@ jobs:
 
   steps:
     - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
-      displayName: Add conda to PATH
+      displayName: 'Add conda to PATH'
     - script: conda update -q -n base conda
       displayName: Update conda
     - script: conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
-      displayName: Create anaconda environment
+      displayName: 'Create anaconda environment'
     - script: |
         call activate pandas-dev
         call conda list
@@ -48,4 +48,9 @@ jobs:
           # note that this will produce $LASTEXITCODE=1
           Write-Error "$($matches[1]) tests failed"
         }
-      displayName: Check for test failures
+      displayName: 'Check for test failures'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
+        python ci/print_skipped.py
+      displayName: 'Print skipped tests'

ci/print_skipped.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import os
 import sys
 import math
 import xml.etree.ElementTree as et
@@ -36,19 +37,19 @@ def parse_results(filename):
     return '\n'.join(skipped)
 
 
-def main(args):
+def main():
+    test_files = [
+        'test-data-single.xml',
+        'test-data-multiple.xml',
+        'test-data.xml',
+    ]
+
     print('SKIPPED TESTS:')
-    for fn in args.filename:
-        print(parse_results(fn))
+    for fn in test_files:
+        if os.path.isfile(fn):
+            print(parse_results(fn))
     return 0
 
 
-def parse_args():
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument('filename', nargs='+', help='XUnit file to parse')
-    return parser.parse_args()
-
-
 if __name__ == '__main__':
-    sys.exit(main(parse_args()))
+    sys.exit(main())

doc/source/reference/indexing.rst

@@ -403,6 +403,13 @@ Conversion
    DatetimeIndex.to_series
    DatetimeIndex.to_frame
 
+Methods
+~~~~~~~
+.. autosummary::
+    :toctree: api/
+
+    DatetimeIndex.mean
+
 TimedeltaIndex
 --------------
 .. autosummary::
@@ -435,6 +442,13 @@ Conversion
    TimedeltaIndex.ceil
    TimedeltaIndex.to_frame
 
+Methods
+~~~~~~~
+.. autosummary::
+    :toctree: api/
+
+    TimedeltaIndex.mean
+
 .. currentmodule:: pandas
 
 PeriodIndex

doc/source/user_guide/groupby.rst

@@ -595,7 +595,7 @@ accepts the special syntax in :meth:`GroupBy.agg`, known as "named aggregation",
    animals.groupby("kind").agg(
        min_height=pd.NamedAgg(column='height', aggfunc='min'),
        max_height=pd.NamedAgg(column='height', aggfunc='max'),
-       average_weight=pd.NamedAgg(column='height', aggfunc=np.mean),
+       average_weight=pd.NamedAgg(column='weight', aggfunc=np.mean),
    )
 
 
@@ -606,7 +606,7 @@ accepts the special syntax in :meth:`GroupBy.agg`, known as "named aggregation",
    animals.groupby("kind").agg(
        min_height=('height', 'min'),
        max_height=('height', 'max'),
-       average_weight=('height', np.mean),
+       average_weight=('weight', np.mean),
    )
 
 
@@ -630,6 +630,16 @@ requires additional arguments, partially apply them with :meth:`functools.partia
    consistent. To ensure consistent ordering, the keys (and so output columns)
    will always be sorted for Python 3.5.
 
+Named aggregation is also valid for Series groupby aggregations. In this case there's
+no column selection, so the values are just the functions.
+
+.. ipython:: python
+
+   animals.groupby("kind").height.agg(
+       min_height='min',
+       max_height='max',
+   )
+
 Applying different functions to DataFrame columns
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/v0.25.0.rst

@@ -28,7 +28,7 @@ Groupby Aggregation with Relabeling
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Pandas has added special groupby behavior, known as "named aggregation", for naming the
-output columns when applying multiple aggregation functions to specific columns (:issue:`18366`).
+output columns when applying multiple aggregation functions to specific columns (:issue:`18366`, :issue:`26512`).
 
 .. ipython:: python
 
@@ -39,7 +39,7 @@ output columns when applying multiple aggregation functions to specific columns
    animals.groupby("kind").agg(
        min_height=pd.NamedAgg(column='height', aggfunc='min'),
        max_height=pd.NamedAgg(column='height', aggfunc='max'),
-       average_weight=pd.NamedAgg(column='height', aggfunc=np.mean),
+       average_weight=pd.NamedAgg(column='weight', aggfunc=np.mean),
    )
 
 Pass the desired columns names as the ``**kwargs`` to ``.agg``. The values of ``**kwargs``
@@ -52,12 +52,26 @@ what the arguments to the function are, but plain tuples are accepted as well.
    animals.groupby("kind").agg(
        min_height=('height', 'min'),
        max_height=('height', 'max'),
-       average_weight=('height', np.mean),
+       average_weight=('weight', np.mean),
    )
 
 Named aggregation is the recommended replacement for the deprecated "dict-of-dicts"
 approach to naming the output of column-specific aggregations (:ref:`whatsnew_0200.api_breaking.deprecate_group_agg_dict`).
 
+A similar approach is now available for Series groupby objects as well. Because there's no need for
+column selection, the values can just be the functions to apply
+
+.. ipython:: python
+
+   animals.groupby("kind").height.agg(
+       min_height="min",
+       max_height="max",
+   )
+
+
+This type of aggregation is the recommended alternative to the deprecated behavior when passing
+a dict to a Series groupby aggregation (:ref:`whatsnew_0200.api_breaking.deprecate_group_agg_dict`).
+
 See :ref:`_groupby.aggregate.named` for more.
 
 .. _whatsnew_0250.enhancements.other:
@@ -82,7 +96,9 @@ Other Enhancements
 - :meth:`DataFrame.query` and :meth:`DataFrame.eval` now supports quoting column names with backticks to refer to names with spaces (:issue:`6508`)
 - :func:`merge_asof` now gives a more clear error message when merge keys are categoricals that are not equal (:issue:`26136`)
 - :meth:`pandas.core.window.Rolling` supports exponential (or Poisson) window type (:issue:`21303`)
--
+- Error message for missing required imports now includes the original import error's text (:issue:`23868`)
+- :class:`DatetimeIndex` and :class:`TimedeltaIndex` now have a ``mean`` method (:issue:`24757`)
+- :meth:`DataFrame.describe` now formats integer percentiles without decimal point (:issue:`26660`)
 
 .. _whatsnew_0250.api_breaking:
 
@@ -478,6 +494,8 @@ Other Deprecations
 - The :meth:`DataFrame.compound` and :meth:`Series.compound` methods are deprecated and will be removed in a future version (:issue:`26405`).
 - The internal attributes ``_start``, ``_stop`` and ``_step`` attributes of :class:`RangeIndex` have been deprecated.
   Use the public attributes :attr:`~RangeIndex.start`, :attr:`~RangeIndex.stop` and :attr:`~RangeIndex.step` instead (:issue:`26581`).
+- The :meth:`Series.ftype`, :meth:`Series.ftypes` and :meth:`DataFrame.ftypes` methods are deprecated and will be removed in a future version.
+  Instead, use :meth:`Series.dtype` and :meth:`DataFrame.dtypes` (:issue:`26705`).
 
 
 .. _whatsnew_0250.prior_deprecations:
@@ -650,6 +668,8 @@ I/O
 - Bug in :func:`read_json` where date strings with ``Z`` were not converted to a UTC timezone (:issue:`26168`)
 - Added ``cache_dates=True`` parameter to :meth:`read_csv`, which allows to cache unique dates when they are parsed (:issue:`25990`)
 - :meth:`DataFrame.to_excel` now raises a ``ValueError`` when the caller's dimensions exceed the limitations of Excel (:issue:`26051`)
+- :func:`read_excel` now raises a ``ValueError`` when input is of type :class:`pandas.io.excel.ExcelFile` and ``engine`` param is passed since :class:`pandas.io.excel.ExcelFile` has an engine defined (:issue:`26566`)
+- Bug while selecting from :class:`HDFStore` with ``where=''`` specified (:issue:`26610`).
 
 Plotting
 ^^^^^^^^

pandas/__init__.py

@@ -10,11 +10,10 @@
     try:
         __import__(dependency)
     except ImportError as e:
-        missing_dependencies.append(dependency)
+        missing_dependencies.append("{0}: {1}".format(dependency, str(e)))
 
 if missing_dependencies:
-    raise ImportError(
-        "Missing required dependencies {0}".format(missing_dependencies))
+    raise ImportError("Unable to import required dependencies:\n" + "\n".join(missing_dependencies))
 del hard_dependencies, dependency, missing_dependencies
 
 # numpy compat

pandas/_libs/tslib.pyx

@@ -127,15 +127,15 @@ def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
         for i in range(n):
             value = arr[i]
             if value == NPY_NAT:
-                result[i] = NaT
+                result[i] = <object>NaT
             else:
                 dt64_to_dtstruct(value, &dts)
                 result[i] = func_create(value, dts, tz, freq)
     elif is_tzlocal(tz):
         for i in range(n):
             value = arr[i]
             if value == NPY_NAT:
-                result[i] = NaT
+                result[i] = <object>NaT
             else:
                 # Python datetime objects do not support nanosecond
                 # resolution (yet, PEP 564). Need to compute new value
@@ -152,7 +152,7 @@ def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
             for i in range(n):
                 value = arr[i]
                 if value == NPY_NAT:
-                    result[i] = NaT
+                    result[i] = <object>NaT
                 else:
                     # Adjust datetime64 timestamp, recompute datetimestruct
                     dt64_to_dtstruct(value + delta, &dts)
@@ -164,7 +164,7 @@ def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
             for i in range(n):
                 value = arr[i]
                 if value == NPY_NAT:
-                    result[i] = NaT
+                    result[i] = <object>NaT
                 else:
                     # Adjust datetime64 timestamp, recompute datetimestruct
                     pos = trans.searchsorted(value, side='right') - 1
@@ -175,7 +175,7 @@ def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
             for i in range(n):
                 value = arr[i]
                 if value == NPY_NAT:
-                    result[i] = NaT
+                    result[i] = <object>NaT
                 else:
                     # Adjust datetime64 timestamp, recompute datetimestruct
                     pos = trans.searchsorted(value, side='right') - 1
@@ -439,11 +439,11 @@ def array_with_unit_to_datetime(ndarray values, object unit,
         val = values[i]
 
         if checknull_with_nat(val):
-            oresult[i] = NaT
+            oresult[i] = <object>NaT
         elif is_integer_object(val) or is_float_object(val):
 
             if val != val or val == NPY_NAT:
-                oresult[i] = NaT
+                oresult[i] = <object>NaT
             else:
                 try:
                     oresult[i] = Timestamp(cast_from_unit(val, unit))
@@ -452,7 +452,7 @@ def array_with_unit_to_datetime(ndarray values, object unit,
 
         elif isinstance(val, str):
             if len(val) == 0 or val in nat_strings:
-                oresult[i] = NaT
+                oresult[i] = <object>NaT
 
             else:
                 oresult[i] = val
@@ -816,7 +816,7 @@ cdef array_to_datetime_object(ndarray[object] values, str errors,
                 check_dts_bounds(&dts)
             except (ValueError, OverflowError):
                 if is_coerce:
-                    oresult[i] = NaT
+                    oresult[i] = <object>NaT
                     continue
                 if is_raise:
                     raise

pandas/_libs/tslibs/timedeltas.pyx

@@ -117,7 +117,7 @@ def ints_to_pytimedelta(int64_t[:] arr, box=False):
 
         value = arr[i]
         if value == NPY_NAT:
-            result[i] = NaT
+            result[i] = <object>NaT
         else:
             if box:
                 result[i] = Timedelta(value)