Merge pull request #39 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

doc/redirects.csv

@@ -271,21 +271,21 @@ generated/pandas.core.window.Expanding.skew,../reference/api/pandas.core.window.
 generated/pandas.core.window.Expanding.std,../reference/api/pandas.core.window.Expanding.std
 generated/pandas.core.window.Expanding.sum,../reference/api/pandas.core.window.Expanding.sum
 generated/pandas.core.window.Expanding.var,../reference/api/pandas.core.window.Expanding.var
-generated/pandas.core.window.Rolling.aggregate,../reference/api/pandas.core.window.Rolling.aggregate
-generated/pandas.core.window.Rolling.apply,../reference/api/pandas.core.window.Rolling.apply
-generated/pandas.core.window.Rolling.corr,../reference/api/pandas.core.window.Rolling.corr
-generated/pandas.core.window.Rolling.count,../reference/api/pandas.core.window.Rolling.count
-generated/pandas.core.window.Rolling.cov,../reference/api/pandas.core.window.Rolling.cov
-generated/pandas.core.window.Rolling.kurt,../reference/api/pandas.core.window.Rolling.kurt
-generated/pandas.core.window.Rolling.max,../reference/api/pandas.core.window.Rolling.max
-generated/pandas.core.window.Rolling.mean,../reference/api/pandas.core.window.Rolling.mean
-generated/pandas.core.window.Rolling.median,../reference/api/pandas.core.window.Rolling.median
-generated/pandas.core.window.Rolling.min,../reference/api/pandas.core.window.Rolling.min
-generated/pandas.core.window.Rolling.quantile,../reference/api/pandas.core.window.Rolling.quantile
-generated/pandas.core.window.Rolling.skew,../reference/api/pandas.core.window.Rolling.skew
-generated/pandas.core.window.Rolling.std,../reference/api/pandas.core.window.Rolling.std
-generated/pandas.core.window.Rolling.sum,../reference/api/pandas.core.window.Rolling.sum
-generated/pandas.core.window.Rolling.var,../reference/api/pandas.core.window.Rolling.var
+generated/pandas.core.window.Rolling.aggregate,../reference/api/pandas.core.window.rolling.Rolling.aggregate
+generated/pandas.core.window.Rolling.apply,../reference/api/pandas.core.window.rolling.Rolling.apply
+generated/pandas.core.window.Rolling.corr,../reference/api/pandas.core.window.rolling.Rolling.corr
+generated/pandas.core.window.Rolling.count,../reference/api/pandas.core.window.rolling.Rolling.count
+generated/pandas.core.window.Rolling.cov,../reference/api/pandas.core.window.rolling.Rolling.cov
+generated/pandas.core.window.Rolling.kurt,../reference/api/pandas.core.window.rolling.Rolling.kurt
+generated/pandas.core.window.Rolling.max,../reference/api/pandas.core.window.rolling.Rolling.max
+generated/pandas.core.window.Rolling.mean,../reference/api/pandas.core.window.rolling.Rolling.mean
+generated/pandas.core.window.Rolling.median,../reference/api/pandas.core.window.rolling.Rolling.median
+generated/pandas.core.window.Rolling.min,../reference/api/pandas.core.window.rolling.Rolling.min
+generated/pandas.core.window.Rolling.quantile,../reference/api/pandas.core.window.rolling.Rolling.quantile
+generated/pandas.core.window.Rolling.skew,../reference/api/pandas.core.window.rolling.Rolling.skew
+generated/pandas.core.window.Rolling.std,../reference/api/pandas.core.window.rolling.Rolling.std
+generated/pandas.core.window.Rolling.sum,../reference/api/pandas.core.window.rolling.Rolling.sum
+generated/pandas.core.window.Rolling.var,../reference/api/pandas.core.window.rolling.Rolling.var
 generated/pandas.core.window.Window.mean,../reference/api/pandas.core.window.Window.mean
 generated/pandas.core.window.Window.sum,../reference/api/pandas.core.window.Window.sum
 generated/pandas.crosstab,../reference/api/pandas.crosstab

doc/source/development/code_style.rst

@@ -119,14 +119,14 @@ For example:
 .. code-block:: python
 
     value = str
-    f"Unknown recived value, got: {repr(value)}"
+    f"Unknown received value, got: {repr(value)}"
 
 **Good:**
 
 .. code-block:: python
 
     value = str
-    f"Unknown recived type, got: '{type(value).__name__}'"
+    f"Unknown received type, got: '{type(value).__name__}'"
 
 
 Imports (aim for absolute)
@@ -135,11 +135,11 @@ Imports (aim for absolute)
 In Python 3, absolute imports are recommended. In absolute import doing something
 like ``import string`` will import the string module rather than ``string.py``
 in the same directory. As much as possible, you should try to write out
-absolute imports that show the whole import chain from toplevel pandas.
+absolute imports that show the whole import chain from top-level pandas.
 
-Explicit relative imports are also supported in Python 3. But it is not
-recommended to use it. Implicit relative imports should never be used
-and is removed in Python 3.
+Explicit relative imports are also supported in Python 3 but it is not
+recommended to use them. Implicit relative imports should never be used
+and are removed in Python 3.
 
 For example:
 

doc/source/getting_started/10min.rst

@@ -70,17 +70,17 @@ will be completed:
    df2.abs                df2.boxplot
    df2.add                df2.C
    df2.add_prefix         df2.clip
-   df2.add_suffix         df2.clip_lower
-   df2.align              df2.clip_upper
-   df2.all                df2.columns
+   df2.add_suffix         df2.columns
+   df2.align              df2.copy
+   df2.all                df2.count
    df2.any                df2.combine
-   df2.append             df2.combine_first
-   df2.apply              df2.consolidate
-   df2.applymap
-   df2.D
+   df2.append             df2.D
+   df2.apply              df2.describe
+   df2.applymap           df2.diff
+   df2.B                  df2.duplicated
 
 As you can see, the columns ``A``, ``B``, ``C``, and ``D`` are automatically
-tab completed. ``E`` is there as well; the rest of the attributes have been
+tab completed. ``E`` and ``F`` are there as well; the rest of the attributes have been
 truncated for brevity.
 
 Viewing data

doc/source/whatsnew/v1.0.2.rst

@@ -16,6 +16,7 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
 - Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`)
+- Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.1.0.rst

@@ -72,7 +72,7 @@ Backwards incompatible API changes
 
 Deprecations
 ~~~~~~~~~~~~
-
+- Lookups on a :class:`Series` with a single-item list containing a slice (e.g. ``ser[[slice(0, 4)]]``) are deprecated, will raise in a future version.  Either convert the list to tuple, or pass the slice directly instead (:issue:`31333`)
 -
 -
 
@@ -185,6 +185,7 @@ I/O
 - Bug in :meth:`DataFrame.to_parquet` overwriting pyarrow's default for
   ``coerce_timestamps``; following pyarrow's default allows writing nanosecond
   timestamps with ``version="2.0"`` (:issue:`31652`).
+- Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`)
 
 Plotting
 ^^^^^^^^
@@ -207,8 +208,10 @@ Reshaping
 - Bug in :meth:`DataFrame.pivot_table` when ``margin`` is ``True`` and only ``column`` is defined (:issue:`31016`)
 - Fix incorrect error message in :meth:`DataFrame.pivot` when ``columns`` is set to ``None``. (:issue:`30924`)
 - Bug in :func:`crosstab` when inputs are two Series and have tuple names, the output will keep dummy MultiIndex as columns. (:issue:`18321`)
+- :meth:`DataFrame.pivot` can now take lists for ``index`` and ``columns`` arguments (:issue:`21425`)
 - Bug in :func:`concat` where the resulting indices are not copied when ``copy=True`` (:issue:`29879`)
 
+
 Sparse
 ^^^^^^
 
@@ -227,6 +230,8 @@ Other
 - Appending a dictionary to a :class:`DataFrame` without passing ``ignore_index=True`` will raise ``TypeError: Can only append a dict if ignore_index=True``
   instead of ``TypeError: Can only append a Series if ignore_index=True or if the Series has a name`` (:issue:`30871`)
 - Set operations on an object-dtype :class:`Index` now always return object-dtype results (:issue:`31401`)
+- Bug in :meth:`AbstractHolidayCalendar.holidays` when no rules were defined (:issue:`31415`)
+-
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/index.pyx

@@ -12,6 +12,7 @@ cnp.import_array()
 
 cimport pandas._libs.util as util
 
+from pandas._libs.tslibs import Period
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
 from pandas._libs.tslibs.c_timestamp cimport _Timestamp
 
@@ -466,6 +467,28 @@ cdef class TimedeltaEngine(DatetimeEngine):
 
 cdef class PeriodEngine(Int64Engine):
 
+    cdef int64_t _unbox_scalar(self, scalar) except? -1:
+        if scalar is NaT:
+            return scalar.value
+        if isinstance(scalar, Period):
+            # NB: we assume that we have the correct freq here.
+            # TODO: potential optimize by checking for _Period?
+            return scalar.ordinal
+        raise TypeError(scalar)
+
+    cpdef get_loc(self, object val):
+        # NB: the caller is responsible for ensuring that we are called
+        #  with either a Period or NaT
+        cdef:
+            int64_t conv
+
+        try:
+            conv = self._unbox_scalar(val)
+        except TypeError:
+            raise KeyError(val)
+
+        return Int64Engine.get_loc(self, conv)
+
     cdef _get_index_values(self):
         return super(PeriodEngine, self).vgetter().view("i8")
 

pandas/_libs/reduction.pyx

@@ -309,8 +309,7 @@ cdef class SeriesGrouper(_BaseGrouper):
     def __init__(self, object series, object f, object labels,
                  Py_ssize_t ngroups, object dummy):
 
-        # in practice we always pass either obj[:0] or the
-        #  safer obj._get_values(slice(None, 0))
+        # in practice we always pass obj.iloc[:0] or equivalent
         assert dummy is not None
 
         if len(series) == 0:

pandas/core/arrays/datetimelike.py

@@ -702,12 +702,31 @@ def take(self, indices, allow_fill=False, fill_value=None):
 
     @classmethod
     def _concat_same_type(cls, to_concat):
-        dtypes = {x.dtype for x in to_concat}
-        assert len(dtypes) == 1
-        dtype = list(dtypes)[0]
+
+        # do not pass tz to set because tzlocal cannot be hashed
+        dtypes = {str(x.dtype) for x in to_concat}
+        if len(dtypes) != 1:
+            raise ValueError("to_concat must have the same dtype (tz)", dtypes)
+
+        obj = to_concat[0]
+        dtype = obj.dtype
 
         values = np.concatenate([x.asi8 for x in to_concat])
-        return cls(values, dtype=dtype)
+
+        if is_period_dtype(to_concat[0].dtype):
+            new_freq = obj.freq
+        else:
+            # GH 3232: If the concat result is evenly spaced, we can retain the
+            # original frequency
+            new_freq = None
+            to_concat = [x for x in to_concat if len(x)]
+
+            if obj.freq is not None and all(x.freq == obj.freq for x in to_concat):
+                pairs = zip(to_concat[:-1], to_concat[1:])
+                if all(pair[0][-1] + obj.freq == pair[1][0] for pair in pairs):
+                    new_freq = obj.freq
+
+        return cls._simple_new(values, dtype=dtype, freq=new_freq)
 
     def copy(self):
         values = self.asi8.copy()

pandas/core/arrays/timedeltas.py

@@ -195,9 +195,12 @@ def __init__(self, values, dtype=_TD_DTYPE, freq=None, copy=False):
     def _simple_new(cls, values, freq=None, dtype=_TD_DTYPE):
         assert dtype == _TD_DTYPE, dtype
         assert isinstance(values, np.ndarray), type(values)
+        if values.dtype != _TD_DTYPE:
+            assert values.dtype == "i8"
+            values = values.view(_TD_DTYPE)
 
         result = object.__new__(cls)
-        result._data = values.view(_TD_DTYPE)
+        result._data = values
         result._freq = to_offset(freq)
         result._dtype = _TD_DTYPE
         return result

pandas/core/frame.py

@@ -3011,17 +3011,12 @@ def _set_value(self, index, col, value, takeable: bool = False):
         col : column label
         value : scalar
         takeable : interpret the index/col as indexers, default False
-
-        Returns
-        -------
-        DataFrame
-            If label pair is contained, will be reference to calling DataFrame,
-            otherwise a new object.
         """
         try:
             if takeable is True:
                 series = self._iget_item_cache(col)
-                return series._set_value(index, value, takeable=True)
+                series._set_value(index, value, takeable=True)
+                return
 
             series = self._get_item_cache(col)
             engine = self.index._engine
@@ -3031,7 +3026,6 @@ def _set_value(self, index, col, value, takeable: bool = False):
             series._values[loc] = value
             # Note: trying to use series._set_value breaks tests in
             #  tests.frame.indexing.test_indexing and tests.indexing.test_partial
-            return self
         except (KeyError, TypeError):
             # set using a non-recursive method & reset the cache
             if takeable:
@@ -3040,8 +3034,6 @@ def _set_value(self, index, col, value, takeable: bool = False):
                 self.loc[index, col] = value
             self._item_cache.pop(col, None)
 
-            return self
-
     def _ensure_valid_index(self, value):
         """
         Ensure that if we don't have an index, that we can create one from the
@@ -5897,11 +5889,19 @@ def groupby(
 
         Parameters
         ----------%s
-        index : str or object, optional
+        index : str or object or a list of str, optional
             Column to use to make new frame's index. If None, uses
             existing index.
-        columns : str or object
+
+            .. versionchanged:: 1.1.0
+               Also accept list of index names.
+
+        columns : str or object or a list of str
             Column to use to make new frame's columns.
+
+            .. versionchanged:: 1.1.0
+               Also accept list of columns names.
+
         values : str, object or a list of the previous, optional
             Column(s) to use for populating new frame's values. If not
             specified, all remaining columns will be used and the result will
@@ -5968,6 +5968,38 @@ def groupby(
         one   1  2  3   x  y  z
         two   4  5  6   q  w  t
 
+        You could also assign a list of column names or a list of index names.
+
+        >>> df = pd.DataFrame({
+        ...        "lev1": [1, 1, 1, 2, 2, 2],
+        ...        "lev2": [1, 1, 2, 1, 1, 2],
+        ...        "lev3": [1, 2, 1, 2, 1, 2],
+        ...        "lev4": [1, 2, 3, 4, 5, 6],
+        ...        "values": [0, 1, 2, 3, 4, 5]})
+        >>> df
+            lev1 lev2 lev3 lev4 values
+        0   1    1    1    1    0
+        1   1    1    2    2    1
+        2   1    2    1    3    2
+        3   2    1    2    4    3
+        4   2    1    1    5    4
+        5   2    2    2    6    5
+
+        >>> df.pivot(index="lev1", columns=["lev2", "lev3"],values="values")
+        lev2    1         2
+        lev3    1    2    1    2
+        lev1
+        1     0.0  1.0  2.0  NaN
+        2     4.0  3.0  NaN  5.0
+
+        >>> df.pivot(index=["lev1", "lev2"], columns=["lev3"],values="values")
+              lev3    1    2
+        lev1  lev2
+           1     1  0.0  1.0
+                 2  2.0  NaN
+           2     1  4.0  3.0
+                 2  NaN  5.0
+
         A ValueError is raised if there are any duplicates.
 
         >>> df = pd.DataFrame({"foo": ['one', 'one', 'two', 'two'],

pandas/core/generic.py

@@ -8360,9 +8360,7 @@ def _align_frame(
             left = self._ensure_type(
                 left.fillna(method=method, axis=fill_axis, limit=limit)
             )
-            right = self._ensure_type(
-                right.fillna(method=method, axis=fill_axis, limit=limit)
-            )
+            right = right.fillna(method=method, axis=fill_axis, limit=limit)
 
         # if DatetimeIndex have different tz, convert to UTC
         if is_datetime64tz_dtype(left.index):
@@ -9961,7 +9959,7 @@ def _add_numeric_operations(cls):
             see_also="",
             examples="",
         )
-        @Appender(_num_doc)
+        @Appender(_num_doc_mad)
         def mad(self, axis=None, skipna=None, level=None):
             if skipna is None:
                 skipna = True
@@ -10330,6 +10328,26 @@ def _doc_parms(cls):
 %(examples)s
 """
 
+_num_doc_mad = """
+%(desc)s
+
+Parameters
+----------
+axis : %(axis_descr)s
+    Axis for the function to be applied on.
+skipna : bool, default None
+    Exclude NA/null values when computing the result.
+level : int or level name, default None
+    If the axis is a MultiIndex (hierarchical), count along a
+    particular level, collapsing into a %(name1)s.
+
+Returns
+-------
+%(name1)s or %(name2)s (if level specified)\
+%(see_also)s\
+%(examples)s
+"""
+
 _num_ddof_doc = """
 %(desc)s