Merge pull request #86 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

doc/redirects.csv

@@ -49,7 +49,25 @@ internals,development/internals
 # api moved function
 reference/api/pandas.io.json.json_normalize,pandas.json_normalize
 
-# api rename
+# rename due to refactors
+reference/api/pandas.core.window.Rolling,pandas.core.window.rolling.Rolling
+reference/api/pandas.core.window.Rolling.aggregate,pandas.core.window.rolling.Rolling.aggregate
+reference/api/pandas.core.window.Rolling.apply,pandas.core.window.rolling.Rolling.apply
+reference/api/pandas.core.window.Rolling.corr,pandas.core.window.rolling.Rolling.corr
+reference/api/pandas.core.window.Rolling.count,pandas.core.window.rolling.Rolling.count
+reference/api/pandas.core.window.Rolling.cov,pandas.core.window.rolling.Rolling.cov
+reference/api/pandas.core.window.Rolling.kurt,pandas.core.window.rolling.Rolling.kurt
+reference/api/pandas.core.window.Rolling.max,pandas.core.window.rolling.Rolling.max
+reference/api/pandas.core.window.Rolling.mean,pandas.core.window.rolling.Rolling.mean
+reference/api/pandas.core.window.Rolling.median,pandas.core.window.rolling.Rolling.median
+reference/api/pandas.core.window.Rolling.min,pandas.core.window.rolling.Rolling.min
+reference/api/pandas.core.window.Rolling.quantile,pandas.core.window.rolling.Rolling.quantile
+reference/api/pandas.core.window.Rolling.skew,pandas.core.window.rolling.Rolling.skew
+reference/api/pandas.core.window.Rolling.std,pandas.core.window.rolling.Rolling.std
+reference/api/pandas.core.window.Rolling.sum,pandas.core.window.rolling.Rolling.sum
+reference/api/pandas.core.window.Rolling.var,pandas.core.window.rolling.Rolling.var
+
+# api url change (generated -> reference/api rename)
 api,reference/index
 generated/pandas.api.extensions.ExtensionArray.argsort,../reference/api/pandas.api.extensions.ExtensionArray.argsort
 generated/pandas.api.extensions.ExtensionArray.astype,../reference/api/pandas.api.extensions.ExtensionArray.astype

doc/source/_static/question_mark_noback.svg

@@ -25,7 +25,7 @@ This calendar shows all the developer meetings.
 You can subscribe to this calendar with the following links:
 
 * `iCal <https://calendar.google.com/calendar/ical/pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com/public/basic.ics>`__
-* `Google calendar <https://calendar.google.com/calendar/embed?src=pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com>`__
+* `Google calendar <https://calendar.google.com/calendar/r?cid=pgbn14p6poja8a1cf2dv2jhrmg@group.calendar.google.com>`__
 
 Additionally, we'll sometimes have one-off meetings on specific topics.
 These will be published on the same calendar.

doc/source/development/meeting.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_102:
 
-What's new in 1.0.2 (February ??, 2020)
----------------------------------------
+What's new in 1.0.2 (March 11, 2020)
+------------------------------------
 
 These are the changes in pandas 1.0.2. See :ref:`release` for a full changelog
 including other versions of pandas.
@@ -18,13 +18,14 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`)
 - Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`)
 - Fixed regression in :meth:`pandas.core.groupby.RollingGroupby.apply` where the ``raw`` parameter was ignored (:issue:`31754`)
-- Fixed regression in :meth:`rolling(..).corr() <pandas.core.window.Rolling.corr>` when using a time offset (:issue:`31789`)
-- Fixed regression in :meth:`DataFrameGroupBy.nunique` which was modifying the original values if ``NaN`` values were present (:issue:`31950`)
+- Fixed regression in :meth:`pandas.core.window.Rolling.corr` when using a time offset (:issue:`31789`)
+- Fixed regression in :meth:`pandas.core.groupby.DataFrameGroupBy.nunique` which was modifying the original values if ``NaN`` values were present (:issue:`31950`)
 - Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`).
 - Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`)
-- Fixed regression in :meth:`GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`)
+- Fixed regression in :meth:`pandas.core.groupby.GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`)
 - Joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` will preserve ``freq`` in simple cases (:issue:`32166`)
-- Fixed bug in the repr of an object-dtype ``Index`` with bools and missing values (:issue:`32146`)
+- Fixed bug in the repr of an object-dtype :class:`Index` with bools and missing values (:issue:`32146`)
+- Fixed regression in :meth:`read_csv` in which the ``encoding`` option was not recognized with certain file-like objects (:issue:`31819`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -64,7 +65,9 @@ Bug fixes
 **Datetimelike**
 
 - Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`)
+- Bug in :meth:`Series.astype` not copying for tz-naive and tz-aware datetime64 dtype (:issue:`32490`)
 - Bug where :func:`to_datetime` would raise when passed ``pd.NA`` (:issue:`32213`)
+- Improved error message when subtracting two :class:`Timestamp` that result in an out-of-bounds :class:`Timedelta` (:issue:`31774`)
 
 **Categorical**
 
@@ -82,8 +85,17 @@ Bug fixes
 
 - Fix bug in :meth:`DataFrame.convert_dtypes` for columns that were already using the ``"string"`` dtype (:issue:`31731`).
 - Fixed bug in setting values using a slice indexer with string dtype (:issue:`31772`)
-- Fixed bug where :meth:`GroupBy.first` and :meth:`GroupBy.last` would raise a ``TypeError`` when groups contained ``pd.NA`` in a column of object dtype (:issue:`32123`)
+- Fixed bug where :meth:`pandas.core.groupby.GroupBy.first` and :meth:`pandas.core.groupby.GroupBy.last` would raise a ``TypeError`` when groups contained ``pd.NA`` in a column of object dtype (:issue:`32123`)
 - Fix bug in :meth:`Series.convert_dtypes` for series with mix of integers and strings (:issue:`32117`)
+- Fixed bug in :meth:`DataFrame.convert_dtypes`, where ``BooleanDtype`` columns were converted to ``Int64`` (:issue:`32287`)
+
+**Strings**
+
+- Using ``pd.NA`` with :meth:`Series.str.repeat` now correctly outputs a null value instead of raising error for vector inputs (:issue:`31632`)
+
+**Rolling**
+
+- Fixed rolling operations with variable window (defined by time duration) on decreasing time index (:issue:`32385`).
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.0.2.rst

@@ -64,6 +64,7 @@ Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
 - :class:`Styler` may now render CSS more efficiently where multiple cells have the same styling (:issue:`30876`)
+- :meth:`Styler.highlight_null` now accepts ``subset`` argument (:issue:`31345`)
 - When writing directly to a sqlite connection :func:`to_sql` now supports the ``multi`` method (:issue:`29921`)
 - `OptionError` is now exposed in `pandas.errors` (:issue:`27553`)
 - :func:`timedelta_range` will now infer a frequency when passed ``start``, ``stop``, and ``periods`` (:issue:`32377`)
@@ -187,7 +188,9 @@ Performance improvements
 - Performance improvement in :class:`Timedelta` constructor (:issue:`30543`)
 - Performance improvement in :class:`Timestamp` constructor (:issue:`30543`)
 - Performance improvement in flex arithmetic ops between :class:`DataFrame` and :class:`Series` with ``axis=0`` (:issue:`31296`)
--
+- The internal :meth:`Index._shallow_copy` now copies cached attributes over to the new index,
+  avoiding creating these again on the new index. This can speed up many operations
+  that depend on creating copies of existing indexes (:issue:`28584`)
 
 .. ---------------------------------------------------------------------------
 
@@ -262,7 +265,7 @@ Indexing
 - Bug in :meth:`Series.xs` incorrectly returning ``Timestamp`` instead of ``datetime64`` in some object-dtype cases (:issue:`31630`)
 - Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`)
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
--
+- Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 
 Missing
 ^^^^^^^

doc/source/whatsnew/v1.1.0.rst

@@ -638,7 +638,8 @@ cdef class TextReader:
                 raise ValueError(f'Unrecognized compression type: '
                                  f'{self.compression}')
 
-            if self.encoding and isinstance(source, (io.BufferedIOBase, io.RawIOBase)):
+            if (self.encoding and hasattr(source, "read") and
+                    not hasattr(source, "encoding")):
                 source = io.TextIOWrapper(
                     source, self.encoding.decode('utf-8'), newline='')
 

pandas/_libs/parsers.pyx

@@ -237,9 +237,9 @@ static PyObject *get_values(PyObject *obj) {
         }
     }
 
-    if ((values == NULL) && PyObject_HasAttrString(obj, "get_block_values")) {
+    if ((values == NULL) && PyObject_HasAttrString(obj, "get_block_values_for_json")) {
         PRINTMARK();
-        values = PyObject_CallMethod(obj, "get_block_values", NULL);
+        values = PyObject_CallMethod(obj, "get_block_values_for_json", NULL);
 
         if (values == NULL) {
             // Clear so we can subsequently try another method

pandas/_libs/src/ujson/python/objToJSON.c

@@ -286,6 +286,10 @@ cdef class _Timestamp(datetime):
         # coerce if necessary if we are a Timestamp-like
         if (PyDateTime_Check(self)
                 and (PyDateTime_Check(other) or is_datetime64_object(other))):
+            # both_timestamps is to determine whether Timedelta(self - other)
+            # should raise the OOB error, or fall back returning a timedelta.
+            both_timestamps = (isinstance(other, _Timestamp) and
+                               isinstance(self, _Timestamp))
             if isinstance(self, _Timestamp):
                 other = type(self)(other)
             else:
@@ -301,7 +305,14 @@ cdef class _Timestamp(datetime):
             from pandas._libs.tslibs.timedeltas import Timedelta
             try:
                 return Timedelta(self.value - other.value)
-            except (OverflowError, OutOfBoundsDatetime):
+            except (OverflowError, OutOfBoundsDatetime) as err:
+                if isinstance(other, _Timestamp):
+                    if both_timestamps:
+                        raise OutOfBoundsDatetime(
+                            "Result is too large for pandas.Timedelta. Convert inputs "
+                            "to datetime.datetime with 'Timestamp.to_pydatetime()' "
+                            "before subtracting."
+                        ) from err
                 pass
         elif is_datetime64_object(self):
             # GH#28286 cython semantics for __rsub__, `other` is actually

pandas/_libs/tslibs/c_timestamp.pyx

@@ -1013,7 +1013,7 @@ def roll_max_variable(ndarray[float64_t] values, ndarray[int64_t] start,
 def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
                    ndarray[int64_t] end, int64_t minp, int64_t win):
     """
-    Moving max of 1d array of any numeric type along axis=0 ignoring NaNs.
+    Moving min of 1d array of any numeric type along axis=0 ignoring NaNs.
 
     Parameters
     ----------
@@ -1030,7 +1030,7 @@ def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
 def roll_min_variable(ndarray[float64_t] values, ndarray[int64_t] start,
                       ndarray[int64_t] end, int64_t minp):
     """
-    Moving max of 1d array of any numeric type along axis=0 ignoring NaNs.
+    Moving min of 1d array of any numeric type along axis=0 ignoring NaNs.
 
     Parameters
     ----------

pandas/_libs/window/aggregations.pyx

@@ -44,6 +44,7 @@ def calculate_variable_window_bounds(
     cdef:
         bint left_closed = False
         bint right_closed = False
+        int index_growth_sign = 1
         ndarray[int64_t, ndim=1] start, end
         int64_t start_bound, end_bound
         Py_ssize_t i, j
@@ -58,6 +59,9 @@ def calculate_variable_window_bounds(
     if closed in ['left', 'both']:
         left_closed = True
 
+    if index[num_values - 1] < index[0]:
+        index_growth_sign = -1
+
     start = np.empty(num_values, dtype='int64')
     start.fill(-1)
     end = np.empty(num_values, dtype='int64')
@@ -78,7 +82,7 @@ def calculate_variable_window_bounds(
         # end is end of slice interval (not including)
         for i in range(1, num_values):
             end_bound = index[i]
-            start_bound = index[i] - window_size
+            start_bound = index[i] - index_growth_sign * window_size
 
             # left endpoint is closed
             if left_closed:
@@ -88,13 +92,13 @@ def calculate_variable_window_bounds(
             # within the constraint
             start[i] = i
             for j in range(start[i - 1], i):
-                if index[j] > start_bound:
+                if (index[j] - start_bound) * index_growth_sign > 0:
                     start[i] = j
                     break
 
             # end bound is previous end
             # or current index
-            if index[end[i - 1]] <= end_bound:
+            if (index[end[i - 1]] - end_bound) * index_growth_sign <= 0:
                 end[i] = i + 1
             else:
                 end[i] = end[i - 1]