Merge pull request #1 from brute4s99/master

rebasing

Author: tm9k1

.travis.yml

@@ -53,18 +53,20 @@ matrix:
     - dist: trusty
       env:
         - JOB="3.6, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true DOCTEST=true
-    # In allow_failures
-    - dist: trusty
-      env:
-        - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
-    # In allow_failures
+
     - dist: trusty
       env:
         - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
           - xsel
+
+    # In allow_failures
+    - dist: trusty
+      env:
+        - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
+
     # In allow_failures
     - dist: trusty
       env:
@@ -73,13 +75,6 @@ matrix:
       - dist: trusty
         env:
           - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
-      - dist: trusty
-        env:
-          - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
-        addons:
-          apt:
-            packages:
-            - xsel
       - dist: trusty
         env:
           - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true

README.md

@@ -97,7 +97,7 @@ easy and intuitive. It aims to be the fundamental high-level building block for
 doing practical, **real world** data analysis in Python. Additionally, it has
 the broader goal of becoming **the most powerful and flexible open source data
 analysis / manipulation tool available in any language**. It is already well on
-its way toward this goal.
+its way towards this goal.
 
 ## Main Features
 Here are just a few of the things that pandas does well:

asv_bench/benchmarks/indexing.py

@@ -2,10 +2,10 @@
 
 import numpy as np
 import pandas.util.testing as tm
-from pandas import (Series, DataFrame, MultiIndex, Int64Index, Float64Index,
-                    IntervalIndex, CategoricalIndex,
-                    IndexSlice, concat, date_range)
-from .pandas_vb_common import setup, Panel  # noqa
+from pandas import (Series, DataFrame, MultiIndex, Panel,
+                    Int64Index, Float64Index, IntervalIndex,
+                    CategoricalIndex, IndexSlice, concat, date_range)
+from .pandas_vb_common import setup  # noqa
 
 
 class NumericSeriesIndexing(object):

asv_bench/benchmarks/join_merge.py

@@ -3,14 +3,15 @@
 
 import numpy as np
 import pandas.util.testing as tm
-from pandas import (DataFrame, Series, MultiIndex, date_range, concat, merge,
-                    merge_asof)
+from pandas import (DataFrame, Series, Panel, MultiIndex,
+                    date_range, concat, merge, merge_asof)
+
 try:
     from pandas import merge_ordered
 except ImportError:
     from pandas import ordered_merge as merge_ordered
 
-from .pandas_vb_common import Panel, setup  # noqa
+from .pandas_vb_common import setup  # noqa
 
 
 class Append(object):

asv_bench/benchmarks/panel_ctor.py

@@ -1,9 +1,9 @@
 import warnings
 from datetime import datetime, timedelta
 
-from pandas import DataFrame, DatetimeIndex, date_range
+from pandas import DataFrame, Panel, DatetimeIndex, date_range
 
-from .pandas_vb_common import Panel, setup  # noqa
+from .pandas_vb_common import setup  # noqa
 
 
 class DifferentIndexes(object):

asv_bench/benchmarks/panel_methods.py

@@ -1,8 +1,9 @@
 import warnings
 
 import numpy as np
+from pandas import Panel
 
-from .pandas_vb_common import Panel, setup  # noqa
+from .pandas_vb_common import setup  # noqa
 
 
 class PanelMethods(object):

ci/travis-27.yaml

@@ -44,7 +44,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
-  - moto
+  - moto==1.3.4
   - hypothesis>=3.58.0
   - pip:
     - backports.lzma

ci/travis-36-doc.yaml

@@ -12,7 +12,7 @@ dependencies:
   - html5lib
   - hypothesis>=3.58.0
   - ipykernel
-  - ipython
+  - ipython==6.5.0
   - ipywidgets
   - lxml
   - matplotlib

doc/source/contributing.rst

@@ -880,7 +880,7 @@ If your change involves checking that a warning is actually emitted, use
 
 .. code-block:: python
 
-   with tm.assert_prodcues_warning(FutureWarning):
+   with tm.assert_produces_warning(FutureWarning):
        df.some_operation()
 
 We prefer this to the ``pytest.warns`` context manager because ours checks that the warning's

doc/source/whatsnew/v0.24.0.txt

@@ -199,6 +199,7 @@ Other Enhancements
 
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- A newly constructed empty :class:`DataFrame` with integer as the ``dtype`` will now only be cast to ``float64`` if ``index`` is specified (:issue:`22858`)
 
 
 .. _whatsnew_0240.api_breaking.interval_values:
@@ -506,6 +507,7 @@ ExtensionType Changes
 - :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`).
 - Slicing a single row of a ``DataFrame`` with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`)
 - Added :meth:`pandas.api.types.register_extension_dtype` to register an extension type with pandas (:issue:`22664`)
+- Updated the ``.type`` attribute for ``PeriodDtype``, ``DatetimeTZDtype``, and ``IntervalDtype`` to be instances of the dtype (``Period``, ``Timestamp``, and ``Interval`` respectively) (:issue:`22938`)
 
 .. _whatsnew_0240.api.incompatibilities:
 
@@ -533,6 +535,35 @@ Current Behavior:
     ...
     OverflowError: Trying to coerce negative values to unsigned integers
 
+.. _whatsnew_0240.api.crosstab_dtypes
+
+Crosstab Preserves Dtypes
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:func:`crosstab` will preserve now dtypes in some cases that previously would
+cast from integer dtype to floating dtype (:issue:`22019`)
+
+Previous Behavior:
+
+.. code-block:: ipython
+
+    In [3]: df = pd.DataFrame({'a': [1, 2, 2, 2, 2], 'b': [3, 3, 4, 4, 4],
+       ...:                    'c': [1, 1, np.nan, 1, 1]})
+    In [4]: pd.crosstab(df.a, df.b, normalize='columns')
+    Out[4]:
+    b    3    4
+    a
+    1  0.5  0.0
+    2  0.5  1.0
+
+Current Behavior:
+
+.. code-block:: ipython
+
+    In [3]: df = pd.DataFrame({'a': [1, 2, 2, 2, 2], 'b': [3, 3, 4, 4, 4],
+       ...:                    'c': [1, 1, np.nan, 1, 1]})
+    In [4]: pd.crosstab(df.a, df.b, normalize='columns')
+
 Datetimelike API Changes
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -795,6 +826,7 @@ Groupby/Resample/Rolling
 - Bug in :meth:`Resampler.asfreq` when frequency of ``TimedeltaIndex`` is a subperiod of a new frequency (:issue:`13022`).
 - Bug in :meth:`SeriesGroupBy.mean` when values were integral but could not fit inside of int64, overflowing instead. (:issue:`22487`)
 - :func:`RollingGroupby.agg` and :func:`ExpandingGroupby.agg` now support multiple aggregation functions as parameters (:issue:`15072`)
+- Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` when resampling by a weekly offset (``'W'``) across a DST transition (:issue:`9119`, :issue:`21459`)
 
 Sparse
 ^^^^^^
@@ -818,6 +850,7 @@ Reshaping
 - Bug in :meth:`DataFrame.drop_duplicates` for empty ``DataFrame`` which incorrectly raises an error (:issue:`20516`)
 - Bug in :func:`pandas.wide_to_long` when a string is passed to the stubnames argument and a column name is a substring of that stubname (:issue:`22468`)
 - Bug in :func:`merge` when merging ``datetime64[ns, tz]`` data that contained a DST transition (:issue:`18885`)
+- Bug in :func:`merge_asof` when merging on float values within defined tolerance (:issue:`22981`)
 
 Build Changes
 ^^^^^^^^^^^^^

pandas/_libs/algos_common_helper.pxi.in

@@ -2,7 +2,6 @@
 Template for each `dtype` helper function using 1-d template
 
 # 1-d template
-- map_indices
 - pad
 - pad_1d
 - pad_2d

pandas/_libs/algos_rank_helper.pxi.in

@@ -24,17 +24,8 @@ dtypes = [('object', 'object', 'Infinity()', 'NegInfinity()'),
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-{{if dtype == 'object'}}
-
-
 def rank_1d_{{dtype}}(object in_arr, ties_method='average',
                       ascending=True, na_option='keep', pct=False):
-{{else}}
-
-
-def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
-                      na_option='keep', pct=False):
-{{endif}}
     """
     Fast NaN-friendly version of scipy.stats.rankdata
     """

pandas/_libs/algos_take_helper.pxi.in

@@ -260,33 +260,39 @@ def take_2d_multi_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=2] values,
 
 {{endfor}}
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # take_2d internal function
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 
-{{py:
-
-# dtype, ctype, init_result
-dtypes = [('float64', 'float64_t', 'np.empty_like(values)'),
-          ('uint64', 'uint64_t', 'np.empty_like(values)'),
-          ('object', 'object', 'values.copy()'),
-          ('int64', 'int64_t', 'np.empty_like(values)')]
-}}
+ctypedef fused take_t:
+    float64_t
+    uint64_t
+    int64_t
+    object
 
-{{for dtype, ctype, init_result in dtypes}}
 
-cdef _take_2d_{{dtype}}(ndarray[{{ctype}}, ndim=2] values, object idx):
+cdef _take_2d(ndarray[take_t, ndim=2] values, object idx):
     cdef:
         Py_ssize_t i, j, N, K
         ndarray[Py_ssize_t, ndim=2, cast=True] indexer = idx
-        ndarray[{{ctype}}, ndim=2] result
+        ndarray[take_t, ndim=2] result
         object val
 
     N, K = (<object> values).shape
-    result = {{init_result}}
+
+    if take_t is object:
+        # evaluated at compile-time
+        result = values.copy()
+    else:
+        result = np.empty_like(values)
+
     for i in range(N):
         for j in range(K):
             result[i, j] = values[i, indexer[i, j]]
     return result
 
-{{endfor}}
+
+_take_2d_object = _take_2d[object]
+_take_2d_float64 = _take_2d[float64_t]
+_take_2d_int64 = _take_2d[int64_t]
+_take_2d_uint64 = _take_2d[uint64_t]

pandas/_libs/join_func_helper.pxi.in

@@ -68,21 +68,21 @@ def asof_join_backward_{{on_dtype}}_by_{{by_dtype}}(
 
         # find last position in right whose value is less than left's
         if allow_exact_matches:
-            while right_pos < right_size and\
-                right_values[right_pos] <= left_values[left_pos]:
+            while (right_pos < right_size and
+                   right_values[right_pos] <= left_values[left_pos]):
                 hash_table.set_item(right_by_values[right_pos], right_pos)
                 right_pos += 1
         else:
-            while right_pos < right_size and\
-                right_values[right_pos] < left_values[left_pos]:
+            while (right_pos < right_size and
+                   right_values[right_pos] < left_values[left_pos]):
                 hash_table.set_item(right_by_values[right_pos], right_pos)
                 right_pos += 1
         right_pos -= 1
 
         # save positions as the desired index
         by_value = left_by_values[left_pos]
-        found_right_pos = hash_table.get_item(by_value)\
-                          if by_value in hash_table else -1
+        found_right_pos = (hash_table.get_item(by_value)
+                           if by_value in hash_table else -1)
         left_indexer[left_pos] = left_pos
         right_indexer[left_pos] = found_right_pos
 
@@ -133,21 +133,21 @@ def asof_join_forward_{{on_dtype}}_by_{{by_dtype}}(
 
         # find first position in right whose value is greater than left's
         if allow_exact_matches:
-            while right_pos >= 0 and\
-                right_values[right_pos] >= left_values[left_pos]:
+            while (right_pos >= 0 and
+                   right_values[right_pos] >= left_values[left_pos]):
                 hash_table.set_item(right_by_values[right_pos], right_pos)
                 right_pos -= 1
         else:
-            while right_pos >= 0 and\
-                right_values[right_pos] > left_values[left_pos]:
+            while (right_pos >= 0 and
+                   right_values[right_pos] > left_values[left_pos]):
                 hash_table.set_item(right_by_values[right_pos], right_pos)
                 right_pos -= 1
         right_pos += 1
 
         # save positions as the desired index
         by_value = left_by_values[left_pos]
-        found_right_pos = hash_table.get_item(by_value)\
-                          if by_value in hash_table else -1
+        found_right_pos = (hash_table.get_item(by_value)
+                           if by_value in hash_table else -1)
         left_indexer[left_pos] = left_pos
         right_indexer[left_pos] = found_right_pos
 
@@ -259,12 +259,12 @@ def asof_join_backward_{{on_dtype}}(
 
         # find last position in right whose value is less than left's
         if allow_exact_matches:
-            while right_pos < right_size and\
-                right_values[right_pos] <= left_values[left_pos]:
+            while (right_pos < right_size and
+                   right_values[right_pos] <= left_values[left_pos]):
                 right_pos += 1
         else:
-            while right_pos < right_size and\
-                right_values[right_pos] < left_values[left_pos]:
+            while (right_pos < right_size and
+                   right_values[right_pos] < left_values[left_pos]):
                 right_pos += 1
         right_pos -= 1
 
@@ -313,19 +313,19 @@ def asof_join_forward_{{on_dtype}}(
 
         # find first position in right whose value is greater than left's
         if allow_exact_matches:
-            while right_pos >= 0 and\
-                right_values[right_pos] >= left_values[left_pos]:
+            while (right_pos >= 0 and
+                   right_values[right_pos] >= left_values[left_pos]):
                 right_pos -= 1
         else:
-            while right_pos >= 0 and\
-                right_values[right_pos] > left_values[left_pos]:
+            while (right_pos >= 0 and
+                   right_values[right_pos] > left_values[left_pos]):
                 right_pos -= 1
         right_pos += 1
 
         # save positions as the desired index
         left_indexer[left_pos] = left_pos
-        right_indexer[left_pos] = right_pos\
-                                  if right_pos != right_size else -1
+        right_indexer[left_pos] = (right_pos
+                                   if right_pos != right_size else -1)
 
         # if needed, verify that tolerance is met
         if has_tolerance and right_pos != right_size: