Merge branch 'master' of git://github.com/pandas-dev/pandas into ValueErrorInPy3

Author: radekbenesinno

doc/source/whatsnew/v0.24.1.rst

@@ -22,8 +22,10 @@ Fixed Regressions
 
 - Bug in :meth:`DataFrame.itertuples` with ``records`` orient raising an ``AttributeError`` when the ``DataFrame`` contained more than 255 columns (:issue:`24939`)
 - Bug in :meth:`DataFrame.itertuples` orient converting integer column names to strings prepended with an underscore (:issue:`24940`)
+- Fixed regression in :func:`read_sql` when passing certain queries with MySQL/pymysql (:issue:`24988`).
 - Fixed regression in :class:`Index.intersection` incorrectly sorting the values by default (:issue:`24959`).
 - Fixed regression in :func:`merge` when merging an empty ``DataFrame`` with multiple timezone-aware columns on one of the timezone-aware columns (:issue:`25014`).
+- Fixed regression in :meth:`Series.rename_axis` and :meth:`DataFrame.rename_axis` where passing ``None`` failed to remove the axis name (:issue:`25034`)
 
 .. _whatsnew_0241.enhancements:
 

doc/source/whatsnew/v0.24.2.rst

@@ -0,0 +1,99 @@
+:orphan:
+
+.. _whatsnew_0242:
+
+Whats New in 0.24.2 (February XX, 2019)
+---------------------------------------
+
+.. warning::
+
+   The 0.24.x series of releases will be the last to support Python 2. Future feature
+   releases will support Python 3 only. See :ref:`install.dropping-27` for more.
+
+{{ header }}
+
+These are the changes in pandas 0.24.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+.. _whatsnew_0242.regressions:
+
+Fixed Regressions
+^^^^^^^^^^^^^^^^^
+
+-
+-
+-
+
+.. _whatsnew_0242.enhancements:
+
+Enhancements
+^^^^^^^^^^^^
+
+-
+-
+
+.. _whatsnew_0242.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+**Conversion**
+
+-
+-
+-
+
+**Indexing**
+
+-
+-
+-
+
+**I/O**
+
+-
+-
+-
+
+**Categorical**
+
+-
+-
+-
+
+**Timezones**
+
+-
+-
+-
+
+**Timedelta**
+
+-
+-
+-
+
+**Reshaping**
+
+-
+-
+-
+
+**Visualization**
+
+-
+-
+-
+
+**Other**
+
+-
+-
+-
+
+.. _whatsnew_0.242.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v0.24.1..v0.24.2

doc/source/whatsnew/v0.25.0.rst

@@ -19,7 +19,7 @@ including other versions of pandas.
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 
--
+- :meth:`Timestamp.replace` now supports the ``fold`` argument to disambiguate DST transition times (:issue:`25017`)
 -
 -
 

pandas/_libs/tslibs/nattype.pyx

@@ -669,7 +669,6 @@ class NaTType(_NaT):
         nanosecond : int, optional
         tzinfo : tz-convertible, optional
         fold : int, optional, default is 0
-            added in 3.6, NotImplemented
 
         Returns
         -------

pandas/_libs/tslibs/timestamps.pyx

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import sys
 import warnings
 
 from cpython cimport (PyObject_RichCompareBool, PyObject_RichCompare,
@@ -43,10 +44,11 @@ from pandas._libs.tslibs.timezones import UTC
 # Constants
 _zero_time = datetime_time(0, 0)
 _no_input = object()
-
+PY36 = sys.version_info >= (3, 6)
 
 # ----------------------------------------------------------------------
 
+
 def maybe_integer_op_deprecated(obj):
     # GH#22535 add/sub of integers and int-arrays is deprecated
     if obj.freq is not None:
@@ -1195,7 +1197,6 @@ class Timestamp(_Timestamp):
         nanosecond : int, optional
         tzinfo : tz-convertible, optional
         fold : int, optional, default is 0
-            added in 3.6, NotImplemented
 
         Returns
         -------
@@ -1252,12 +1253,16 @@ class Timestamp(_Timestamp):
             # see GH#18319
             ts_input = _tzinfo.localize(datetime(dts.year, dts.month, dts.day,
                                                  dts.hour, dts.min, dts.sec,
-                                                 dts.us))
+                                                 dts.us),
+                                        is_dst=not bool(fold))
             _tzinfo = ts_input.tzinfo
         else:
-            ts_input = datetime(dts.year, dts.month, dts.day,
-                                dts.hour, dts.min, dts.sec, dts.us,
-                                tzinfo=_tzinfo)
+            kwargs = {'year': dts.year, 'month': dts.month, 'day': dts.day,
+                      'hour': dts.hour, 'minute': dts.min, 'second': dts.sec,
+                      'microsecond': dts.us, 'tzinfo': _tzinfo}
+            if PY36:
+                kwargs['fold'] = fold
+            ts_input = datetime(**kwargs)
 
         ts = convert_datetime_to_tsobject(ts_input, _tzinfo)
         value = ts.value + (dts.ps // 1000)

pandas/compat/pickle_compat.py

@@ -201,7 +201,7 @@ def load_newobj_ex(self):
     pass
 
 
-def load(fh, encoding=None, compat=False, is_verbose=False):
+def load(fh, encoding=None, is_verbose=False):
     """load a pickle, with a provided encoding
 
     if compat is True:
@@ -212,7 +212,6 @@ def load(fh, encoding=None, compat=False, is_verbose=False):
     ----------
     fh : a filelike object
     encoding : an optional encoding
-    compat : provide Series compatibility mode, boolean, default False
     is_verbose : show exception output
     """
 

pandas/core/base.py

@@ -1234,7 +1234,7 @@ def value_counts(self, normalize=False, sort=True, ascending=False,
             If True then the object returned will contain the relative
             frequencies of the unique values.
         sort : boolean, default True
-            Sort by values.
+            Sort by frequencies.
         ascending : boolean, default False
             Sort in ascending order.
         bins : integer, optional

pandas/core/frame.py

@@ -4625,7 +4625,8 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None,
     def drop_duplicates(self, subset=None, keep='first', inplace=False):
         """
         Return DataFrame with duplicate rows removed, optionally only
-        considering certain columns.
+        considering certain columns. Indexes, including time indexes
+        are ignored.
 
         Parameters
         ----------

pandas/core/generic.py

@@ -61,6 +61,10 @@
         by : str or list of str
             Name or list of names to sort by""")
 
+# sentinel value to use as kwarg in place of None when None has special meaning
+# and needs to be distinguished from a user explicitly passing None.
+sentinel = object()
+
 
 def _single_replace(self, to_replace, method, inplace, limit):
     """
@@ -290,11 +294,16 @@ def _construct_axes_dict_for_slice(self, axes=None, **kwargs):
         d.update(kwargs)
         return d
 
-    def _construct_axes_from_arguments(self, args, kwargs, require_all=False):
+    def _construct_axes_from_arguments(
+            self, args, kwargs, require_all=False, sentinel=None):
         """Construct and returns axes if supplied in args/kwargs.
 
         If require_all, raise if all axis arguments are not supplied
         return a tuple of (axes, kwargs).
+
+        sentinel specifies the default parameter when an axis is not
+        supplied; useful to distinguish when a user explicitly passes None
+        in scenarios where None has special meaning.
         """
 
         # construct the args
@@ -322,7 +331,7 @@ def _construct_axes_from_arguments(self, args, kwargs, require_all=False):
                         raise TypeError("not enough/duplicate arguments "
                                         "specified!")
 
-        axes = {a: kwargs.pop(a, None) for a in self._AXIS_ORDERS}
+        axes = {a: kwargs.pop(a, sentinel) for a in self._AXIS_ORDERS}
         return axes, kwargs
 
     @classmethod
@@ -1089,7 +1098,7 @@ def rename(self, *args, **kwargs):
 
     @rewrite_axis_style_signature('mapper', [('copy', True),
                                              ('inplace', False)])
-    def rename_axis(self, mapper=None, **kwargs):
+    def rename_axis(self, mapper=sentinel, **kwargs):
         """
         Set the name of the axis for the index or columns.
 
@@ -1218,7 +1227,8 @@ class  name
                cat            4         0
                monkey         2         2
         """
-        axes, kwargs = self._construct_axes_from_arguments((), kwargs)
+        axes, kwargs = self._construct_axes_from_arguments(
+            (), kwargs, sentinel=sentinel)
         copy = kwargs.pop('copy', True)
         inplace = kwargs.pop('inplace', False)
         axis = kwargs.pop('axis', 0)
@@ -1231,7 +1241,7 @@ class  name
 
         inplace = validate_bool_kwarg(inplace, 'inplace')
 
-        if (mapper is not None):
+        if (mapper is not sentinel):
             # Use v0.23 behavior if a scalar or list
             non_mapper = is_scalar(mapper) or (is_list_like(mapper) and not
                                                is_dict_like(mapper))
@@ -1254,7 +1264,7 @@ class  name
 
             for axis in lrange(self._AXIS_LEN):
                 v = axes.get(self._AXIS_NAMES[axis])
-                if v is None:
+                if v is sentinel:
                     continue
                 non_mapper = is_scalar(v) or (is_list_like(v) and not
                                               is_dict_like(v))

pandas/io/pickle.py

@@ -1,8 +1,7 @@
 """ pickle compat """
 import warnings
 
-import numpy as np
-from numpy.lib.format import read_array, write_array
+from numpy.lib.format import read_array
 
 from pandas.compat import PY3, BytesIO, cPickle as pkl, pickle_compat as pc
 
@@ -76,6 +75,7 @@ def to_pickle(obj, path, compression='infer', protocol=pkl.HIGHEST_PROTOCOL):
     try:
         f.write(pkl.dumps(obj, protocol=protocol))
     finally:
+        f.close()
         for _f in fh:
             _f.close()
 
@@ -138,63 +138,32 @@ def read_pickle(path, compression='infer'):
     >>> os.remove("./dummy.pkl")
     """
     path = _stringify_path(path)
+    f, fh = _get_handle(path, 'rb', compression=compression, is_text=False)
+
+    # 1) try with cPickle
+    # 2) try with the compat pickle to handle subclass changes
+    # 3) pass encoding only if its not None as py2 doesn't handle the param
 
-    def read_wrapper(func):
-        # wrapper file handle open/close operation
-        f, fh = _get_handle(path, 'rb',
-                            compression=compression,
-                            is_text=False)
-        try:
-            return func(f)
-        finally:
-            for _f in fh:
-                _f.close()
-
-    def try_read(path, encoding=None):
-        # try with cPickle
-        # try with current pickle, if we have a Type Error then
-        # try with the compat pickle to handle subclass changes
-        # pass encoding only if its not None as py2 doesn't handle
-        # the param
-
-        # cpickle
-        # GH 6899
-        try:
-            with warnings.catch_warnings(record=True):
-                # We want to silence any warnings about, e.g. moved modules.
-                warnings.simplefilter("ignore", Warning)
-                return read_wrapper(lambda f: pkl.load(f))
-        except Exception:  # noqa: E722
-            # reg/patched pickle
-            # compat not used in pandas/compat/pickle_compat.py::load
-            # TODO: remove except block OR modify pc.load to use compat
-            try:
-                return read_wrapper(
-                    lambda f: pc.load(f, encoding=encoding, compat=False))
-            # compat pickle
-            except Exception:  # noqa: E722
-                return read_wrapper(
-                    lambda f: pc.load(f, encoding=encoding, compat=True))
     try:
-        return try_read(path)
+        with warnings.catch_warnings(record=True):
+            # We want to silence any warnings about, e.g. moved modules.
+            warnings.simplefilter("ignore", Warning)
+            return pkl.load(f)
     except Exception:  # noqa: E722
-        if PY3:
-            return try_read(path, encoding='latin1')
-        raise
-
+        try:
+            return pc.load(f, encoding=None)
+        except Exception:  # noqa: E722
+            if PY3:
+                return pc.load(f, encoding='latin1')
+            raise
+    finally:
+        f.close()
+        for _f in fh:
+            _f.close()
 
 # compat with sparse pickle / unpickle
 
 
-def _pickle_array(arr):
-    arr = arr.view(np.ndarray)
-
-    buf = BytesIO()
-    write_array(buf, arr)
-
-    return buf.getvalue()
-
-
 def _unpickle_array(bytes):
     arr = read_array(BytesIO(bytes))
 

pandas/io/sql.py

@@ -381,7 +381,8 @@ def read_sql(sql, con, index_col=None, coerce_float=True, params=None,
 
     try:
         _is_table_name = pandas_sql.has_table(sql)
-    except (ImportError, AttributeError):
+    except Exception:
+        # using generic exception to catch errors from sql drivers (GH24988)
         _is_table_name = False
 
     if _is_table_name: