Merge pull request #65 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

asv_bench/benchmarks/categoricals.py

@@ -258,9 +258,6 @@ def setup(self):
     def time_get_loc(self):
         self.index.get_loc(self.category)
 
-    def time_shape(self):
-        self.index.shape
-
     def time_shallow_copy(self):
         self.index._shallow_copy()
 

asv_bench/benchmarks/index_cached_properties.py

@@ -7,6 +7,7 @@ class IndexCache:
 
     params = [
         [
+            "CategoricalIndex",
             "DatetimeIndex",
             "Float64Index",
             "IntervalIndex",
@@ -42,6 +43,8 @@ def setup(self, index_type):
             self.idx = pd.Float64Index(range(N))
         elif index_type == "UInt64Index":
             self.idx = pd.UInt64Index(range(N))
+        elif index_type == "CategoricalIndex":
+            self.idx = pd.CategoricalIndex(range(N), range(N))
         else:
             raise ValueError
         assert len(self.idx) == N

asv_bench/benchmarks/index_object.py

@@ -55,14 +55,6 @@ def time_datetime_difference_disjoint(self):
         self.datetime_left.difference(self.datetime_right)
 
 
-class Datetime:
-    def setup(self):
-        self.dr = date_range("20000101", freq="D", periods=10000)
-
-    def time_is_dates_only(self):
-        self.dr._is_dates_only
-
-
 class Range:
     def setup(self):
         self.idx_inc = RangeIndex(start=0, stop=10 ** 7, step=3)

asv_bench/benchmarks/indexing.py

@@ -1,3 +1,8 @@
+"""
+These benchmarks are for Series and DataFrame indexing methods.  For the
+lower-level methods directly on Index and subclasses, see index_object.py,
+indexing_engine.py, and index_cached.py
+"""
 import warnings
 
 import numpy as np

asv_bench/benchmarks/period.py

@@ -85,9 +85,6 @@ def setup(self):
     def time_get_loc(self):
         self.index.get_loc(self.period)
 
-    def time_shape(self):
-        self.index.shape
-
     def time_shallow_copy(self):
         self.index._shallow_copy()
 

asv_bench/benchmarks/timedelta.py

@@ -73,9 +73,6 @@ def setup(self):
     def time_get_loc(self):
         self.index.get_loc(self.timedelta)
 
-    def time_shape(self):
-        self.index.shape
-
     def time_shallow_copy(self):
         self.index._shallow_copy()
 

asv_bench/benchmarks/timeseries.py

@@ -57,6 +57,9 @@ def time_to_date(self, index_type):
     def time_to_pydatetime(self, index_type):
         self.index.to_pydatetime()
 
+    def time_is_dates_only(self, index_type):
+        self.index._is_dates_only
+
 
 class TzLocalize:
 

doc/source/ecosystem.rst

@@ -56,6 +56,11 @@ joining paths, replacing file extensions, and checking if files exist are also a
 Statistics and machine learning
 -------------------------------
 
+`pandas-tfrecords <https://pypi.org/project/pandas-tfrecords/>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Easy saving pandas dataframe to tensorflow tfrecords format and reading tfrecords to pandas.
+
 `Statsmodels <https://www.statsmodels.org/>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/getting_started/basics.rst

@@ -689,6 +689,17 @@ of a 1D array of values. It can also be used as a function on regular arrays:
    s.value_counts()
    pd.value_counts(data)
 
+.. versionadded:: 1.1.0
+
+The :meth:`~DataFrame.value_counts` method can be used to count combinations across multiple columns.
+By default all columns are used but a subset can be selected using the ``subset`` argument.
+
+.. ipython:: python
+
+    data = {"a": [1, 2, 3, 4], "b": ["x", "x", "y", "y"]}
+    frame = pd.DataFrame(data)
+    frame.value_counts()
+
 Similarly, you can get the most frequently occurring value(s) (the mode) of the values in a Series or DataFrame:
 
 .. ipython:: python

doc/source/reference/frame.rst

@@ -170,6 +170,7 @@ Computations / descriptive stats
    DataFrame.std
    DataFrame.var
    DataFrame.nunique
+   DataFrame.value_counts
 
 Reindexing / selection / label manipulation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/user_guide/timeseries.rst

@@ -2297,6 +2297,35 @@ To remove time zone information, use ``tz_localize(None)`` or ``tz_convert(None)
    # tz_convert(None) is identical to tz_convert('UTC').tz_localize(None)
    didx.tz_convert('UTC').tz_localize(None)
 
+.. _timeseries.fold:
+
+Fold
+~~~~
+
+.. versionadded:: 1.1.0
+
+For ambiguous times, pandas supports explicitly specifying the keyword-only fold argument.
+Due to daylight saving time, one wall clock time can occur twice when shifting
+from summer to winter time; fold describes whether the datetime-like corresponds
+to the first (0) or the second time (1) the wall clock hits the ambiguous time.
+Fold is supported only for constructing from naive ``datetime.datetime``
+(see `datetime documentation <https://docs.python.org/3/library/datetime.html>`__ for details) or from :class:`Timestamp`
+or for constructing from components (see below). Only ``dateutil`` timezones are supported
+(see `dateutil documentation <https://dateutil.readthedocs.io/en/stable/tz.html#dateutil.tz.enfold>`__
+for ``dateutil`` methods that deal with ambiguous datetimes) as ``pytz``
+timezones do not support fold (see `pytz documentation <http://pytz.sourceforge.net/index.html>`__
+for details on how ``pytz`` deals with ambiguous datetimes). To localize an ambiguous datetime
+with ``pytz``, please use :meth:`Timestamp.tz_localize`. In general, we recommend to rely
+on :meth:`Timestamp.tz_localize` when localizing ambiguous datetimes if you need direct
+control over how they are handled.
+
+.. ipython:: python
+
+   pd.Timestamp(datetime.datetime(2019, 10, 27, 1, 30, 0, 0),
+                tz='dateutil/Europe/London', fold=0)
+   pd.Timestamp(year=2019, month=10, day=27, hour=1, minute=30,
+                tz='dateutil/Europe/London', fold=1)
+
 .. _timeseries.timezone_ambiguous:
 
 Ambiguous times when localizing

doc/source/whatsnew/v1.0.2.rst

@@ -23,6 +23,7 @@ Fixed regressions
 - Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`).
 - Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`)
 - Fixed regression in :meth:`GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`)
+- Joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` will preserve ``freq`` in simple cases (:issue:`32166`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -62,6 +63,7 @@ Bug fixes
 **Datetimelike**
 
 - Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`)
+- Bug where :func:`to_datetime` would raise when passed ``pd.NA`` (:issue:`32213`)
 
 **Categorical**
 

doc/source/whatsnew/v1.1.0.rst

@@ -36,6 +36,28 @@ For example:
    ser["2014"]
    ser.loc["May 2015"]
 
+.. _whatsnew_110.timestamp_fold_support:
+
+Fold argument support in Timestamp constructor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:class:`Timestamp:` now supports the keyword-only fold argument according to `PEP 495 <https://www.python.org/dev/peps/pep-0495/#the-fold-attribute>`_ similar to parent ``datetime.datetime`` class. It supports both accepting fold as an initialization argument and inferring fold from other constructor arguments (:issue:`25057`, :issue:`31338`). Support is limited to ``dateutil`` timezones as ``pytz`` doesn't support fold.
+
+For example:
+
+.. ipython:: python
+
+    ts = pd.Timestamp("2019-10-27 01:30:00+00:00")
+    ts.fold
+
+.. ipython:: python
+
+    ts = pd.Timestamp(year=2019, month=10, day=27, hour=1, minute=30,
+                      tz="dateutil/Europe/London", fold=1)
+    ts
+
+For more on working with fold, see :ref:`Fold subsection <timeseries.fold>` in the user guide.
+
 .. _whatsnew_110.enhancements.other:
 
 Other enhancements
@@ -55,6 +77,7 @@ Other API changes
 
 - :meth:`Series.describe` will now show distribution percentiles for ``datetime`` dtypes, statistics ``first`` and ``last``
   will now be ``min`` and ``max`` to match with numeric dtypes in :meth:`DataFrame.describe` (:issue:`30164`)
+- Added :meth:`DataFrame.value_counts` (:issue:`5377`)
 - :meth:`Groupby.groups` now returns an abbreviated representation when called on large dataframes (:issue:`1135`)
 - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`)
 -
@@ -114,6 +137,7 @@ Datetimelike
 - :meth:`DatetimeArray.searchsorted`, :meth:`TimedeltaArray.searchsorted`, :meth:`PeriodArray.searchsorted` not recognizing non-pandas scalars and incorrectly raising ``ValueError`` instead of ``TypeError`` (:issue:`30950`)
 - Bug in :class:`Timestamp` where constructing :class:`Timestamp` with dateutil timezone less than 128 nanoseconds before daylight saving time switch from winter to summer would result in nonexistent time (:issue:`31043`)
 - Bug in :meth:`Period.to_timestamp`, :meth:`Period.start_time` with microsecond frequency returning a timestamp one nanosecond earlier than the correct time (:issue:`31475`)
+- :class:`Timestamp` raising confusing error message when year, month or day is missing (:issue:`31200`)
 
 Timedelta
 ^^^^^^^^^

pandas/_libs/tslib.pyx

@@ -49,30 +49,31 @@ from pandas._libs.tslibs.tzconversion cimport (
 
 cdef inline object create_datetime_from_ts(
         int64_t value, npy_datetimestruct dts,
-        object tz, object freq):
+        object tz, object freq, bint fold):
     """ convenience routine to construct a datetime.datetime from its parts """
     return datetime(dts.year, dts.month, dts.day, dts.hour,
-                    dts.min, dts.sec, dts.us, tz)
+                    dts.min, dts.sec, dts.us, tz, fold=fold)
 
 
 cdef inline object create_date_from_ts(
         int64_t value, npy_datetimestruct dts,
-        object tz, object freq):
+        object tz, object freq, bint fold):
     """ convenience routine to construct a datetime.date from its parts """
+    # GH 25057 add fold argument to match other func_create signatures
     return date(dts.year, dts.month, dts.day)
 
 
 cdef inline object create_time_from_ts(
         int64_t value, npy_datetimestruct dts,
-        object tz, object freq):
+        object tz, object freq, bint fold):
     """ convenience routine to construct a datetime.time from its parts """
-    return time(dts.hour, dts.min, dts.sec, dts.us, tz)
+    return time(dts.hour, dts.min, dts.sec, dts.us, tz, fold=fold)
 
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
-                       str box="datetime"):
+                       bint fold=0, str box="datetime"):
     """
     Convert an i8 repr to an ndarray of datetimes, date, time or Timestamp
 
@@ -83,6 +84,13 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
          convert to this timezone
     freq : str/Offset, default None
          freq to convert
+    fold : bint, default is 0
+        Due to daylight saving time, one wall clock time can occur twice
+        when shifting from summer to winter time; fold describes whether the
+        datetime-like corresponds  to the first (0) or the second time (1)
+        the wall clock hits the ambiguous time
+
+        .. versionadded:: 1.1.0
     box  : {'datetime', 'timestamp', 'date', 'time'}, default 'datetime'
          If datetime, convert to datetime.datetime
          If date, convert to datetime.date
@@ -104,7 +112,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
         str typ
         int64_t value, delta, local_value
         ndarray[object] result = np.empty(n, dtype=object)
-        object (*func_create)(int64_t, npy_datetimestruct, object, object)
+        object (*func_create)(int64_t, npy_datetimestruct, object, object, bint)
 
     if box == "date":
         assert (tz is None), "tz should be None when converting to date"
@@ -129,7 +137,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                 result[i] = <object>NaT
             else:
                 dt64_to_dtstruct(value, &dts)
-                result[i] = func_create(value, dts, tz, freq)
+                result[i] = func_create(value, dts, tz, freq, fold)
     elif is_tzlocal(tz):
         for i in range(n):
             value = arr[i]
@@ -141,7 +149,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                 # using the i8 representation.
                 local_value = tz_convert_utc_to_tzlocal(value, tz)
                 dt64_to_dtstruct(local_value, &dts)
-                result[i] = func_create(value, dts, tz, freq)
+                result[i] = func_create(value, dts, tz, freq, fold)
     else:
         trans, deltas, typ = get_dst_info(tz)
 
@@ -155,7 +163,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                 else:
                     # Adjust datetime64 timestamp, recompute datetimestruct
                     dt64_to_dtstruct(value + delta, &dts)
-                    result[i] = func_create(value, dts, tz, freq)
+                    result[i] = func_create(value, dts, tz, freq, fold)
 
         elif typ == 'dateutil':
             # no zone-name change for dateutil tzs - dst etc
@@ -168,7 +176,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                     # Adjust datetime64 timestamp, recompute datetimestruct
                     pos = trans.searchsorted(value, side='right') - 1
                     dt64_to_dtstruct(value + deltas[pos], &dts)
-                    result[i] = func_create(value, dts, tz, freq)
+                    result[i] = func_create(value, dts, tz, freq, fold)
         else:
             # pytz
             for i in range(n):
@@ -182,7 +190,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                     new_tz = tz._tzinfos[tz._transition_info[pos]]
 
                     dt64_to_dtstruct(value + deltas[pos], &dts)
-                    result[i] = func_create(value, dts, new_tz, freq)
+                    result[i] = func_create(value, dts, new_tz, freq, fold)
 
     return result
 

pandas/_libs/tslibs/conversion.pxd

@@ -12,6 +12,7 @@ cdef class _TSObject:
         npy_datetimestruct dts      # npy_datetimestruct
         int64_t value               # numpy dt64
         object tzinfo
+        bint fold
 
 
 cdef convert_to_tsobject(object ts, object tz, object unit,