Merge branch 'master' of https://github.com/pandas-dev/pandas into tslibs-offsets-inits

Author: jbrockmendel

.travis.yml

@@ -102,8 +102,6 @@ before_install:
   - uname -a
   - git --version
   - git tag
-  - ci/before_install_travis.sh
-  - export DISPLAY=":99.0"
 
 install:
   - echo "install start"
@@ -114,6 +112,8 @@ install:
 
 before_script:
   - ci/install_db_travis.sh
+  - export DISPLAY=":99.0"
+  - ci/before_script_travis.sh
 
 script:
   - echo "script start"

ci/before_install_travis.sh renamed to ci/before_script_travis.sh

@@ -4,6 +4,7 @@ echo "inside $0"
 
 if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
    sh -e /etc/init.d/xvfb start
+   sleep 3
 fi
 
 # Never fail because bad things happened here.

doc/source/whatsnew/v0.21.1.txt

@@ -73,7 +73,7 @@ Conversion
 Indexing
 ^^^^^^^^
 
--
+- Bug where a ``MultiIndex`` with more than a million records was not raising ``AttributeError`` when trying to access a missing attribute (:issue:`18165`)
 -
 -
 

doc/source/whatsnew/v0.22.0.txt

@@ -61,7 +61,7 @@ Deprecations
 Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
--
+- Warnings against the obsolete usage ``Categorical(codes, categories)``, which were emitted for instance when the first two arguments to ``Categorical()`` had different dtypes, and recommended the use of ``Categorical.from_codes``, have now been removed (:issue:`8074`)
 -
 -
 
@@ -72,6 +72,7 @@ Performance Improvements
 
 - Indexers on ``Series`` or ``DataFrame`` no longer create a reference cycle (:issue:`17956`)
 - Added a keyword argument, ``cache``, to :func:`to_datetime` that improved the performance of converting duplicate datetime arguments (:issue:`11665`)
+- :class`DateOffset` arithmetic performance is improved (:issue:`18218`)
 -
 
 .. _whatsnew_0220.docs:

pandas/_libs/tslibs/offsets.pyx

@@ -4,7 +4,7 @@
 cimport cython
 
 import time
-from cpython.datetime cimport timedelta, time as dt_time
+from cpython.datetime cimport datetime, timedelta, time as dt_time
 
 from dateutil.relativedelta import relativedelta
 
@@ -13,9 +13,9 @@ cimport numpy as np
 np.import_array()
 
 
-from util cimport is_string_object
+from util cimport is_string_object, is_integer_object
 
-from pandas._libs.tslib import pydt_to_i8
+from pandas._libs.tslib import pydt_to_i8, monthrange
 
 from frequencies cimport get_freq_code
 from conversion cimport tz_convert_single
@@ -375,3 +375,56 @@ class BaseOffset(_BaseOffset):
             # i.e. isinstance(other, (ABCDatetimeIndex, ABCSeries))
             return other - self
         return -self + other
+
+
+# ----------------------------------------------------------------------
+# RelativeDelta Arithmetic
+
+
+cpdef datetime shift_month(datetime stamp, int months, object day_opt=None):
+    """
+    Given a datetime (or Timestamp) `stamp`, an integer `months` and an
+    option `day_opt`, return a new datetimelike that many months later,
+    with day determined by `day_opt` using relativedelta semantics.
+
+    Scalar analogue of tslib.shift_months
+
+    Parameters
+    ----------
+    stamp : datetime or Timestamp
+    months : int
+    day_opt : None, 'start', 'end', or an integer
+        None: returned datetimelike has the same day as the input, or the
+              last day of the month if the new month is too short
+        'start': returned datetimelike has day=1
+        'end': returned datetimelike has day on the last day of the month
+        int: returned datetimelike has day equal to day_opt
+
+    Returns
+    -------
+    shifted : datetime or Timestamp (same as input `stamp`)
+    """
+    cdef:
+        int year, month, day
+        int dim, dy
+
+    dy = (stamp.month + months) // 12
+    month = (stamp.month + months) % 12
+
+    if month == 0:
+        month = 12
+        dy -= 1
+    year = stamp.year + dy
+
+    dim = monthrange(year, month)[1]
+    if day_opt is None:
+        day = min(stamp.day, dim)
+    elif day_opt == 'start':
+        day = 1
+    elif day_opt == 'end':
+        day = dim
+    elif is_integer_object(day_opt):
+        day = min(day_opt, dim)
+    else:
+        raise ValueError(day_opt)
+    return stamp.replace(year=year, month=month, day=day)

pandas/core/api.py

@@ -24,8 +24,8 @@
 from pandas.core.panel import Panel, WidePanel
 from pandas.core.panel4d import Panel4D
 from pandas.core.reshape.reshape import (
-    pivot_simple as pivot, get_dummies,
-    lreshape, wide_to_long)
+    pivot_simple as pivot, get_dummies)
+from pandas.core.reshape.melt import lreshape, wide_to_long
 
 from pandas.core.indexing import IndexSlice
 from pandas.core.tools.numeric import to_numeric

pandas/core/categorical.py

@@ -25,7 +25,6 @@
     is_timedelta64_dtype,
     is_categorical,
     is_categorical_dtype,
-    is_integer_dtype,
     is_list_like, is_sequence,
     is_scalar,
     is_dict_like)
@@ -261,6 +260,7 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
         #    c.) infer from values
 
         if dtype is not None:
+            # The dtype argument takes precedence over values.dtype (if any)
             if isinstance(dtype, compat.string_types):
                 if dtype == 'category':
                     dtype = CategoricalDtype(categories, ordered)
@@ -275,9 +275,12 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
             ordered = dtype.ordered
 
         elif is_categorical(values):
+            # If no "dtype" was passed, use the one from "values", but honor
+            # the "ordered" and "categories" arguments
             dtype = values.dtype._from_categorical_dtype(values.dtype,
                                                          categories, ordered)
         else:
+            # If dtype=None and values is not categorical, create a new dtype
             dtype = CategoricalDtype(categories, ordered)
 
         # At this point, dtype is always a CategoricalDtype
@@ -294,28 +297,12 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
 
         # sanitize input
         if is_categorical_dtype(values):
+            if dtype.categories is None:
+                dtype = CategoricalDtype(values.categories, dtype.ordered)
 
-            # we are either a Series or a CategoricalIndex
-            if isinstance(values, (ABCSeries, ABCCategoricalIndex)):
-                values = values._values
-
-            if ordered is None:
-                ordered = values.ordered
-            if categories is None:
-                categories = values.categories
-            values = values.get_values()
-
-        elif isinstance(values, (ABCIndexClass, ABCSeries)):
-            # we'll do inference later
-            pass
-
-        else:
-
-            # on numpy < 1.6 datetimelike get inferred to all i8 by
-            # _sanitize_array which is fine, but since factorize does this
-            # correctly no need here this is an issue because _sanitize_array
-            # also coerces np.nan to a string under certain versions of numpy
-            # as well
+        elif not isinstance(values, (ABCIndexClass, ABCSeries)):
+            # _sanitize_array coerces np.nan to a string under certain versions
+            # of numpy
             values = maybe_infer_to_datetimelike(values, convert_dates=True)
             if not isinstance(values, np.ndarray):
                 values = _convert_to_list_like(values)
@@ -335,7 +322,7 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
                 codes, categories = factorize(values, sort=True)
             except TypeError:
                 codes, categories = factorize(values, sort=False)
-                if ordered:
+                if dtype.ordered:
                     # raise, as we don't have a sortable data structure and so
                     # the user should give us one by specifying categories
                     raise TypeError("'values' is not ordered, please "
@@ -347,34 +334,18 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
                 raise NotImplementedError("> 1 ndim Categorical are not "
                                           "supported at this time")
 
-            if dtype.categories is None:
-                # we're inferring from values
-                dtype = CategoricalDtype(categories, ordered)
+            # we're inferring from values
+            dtype = CategoricalDtype(categories, dtype.ordered)
 
-        else:
-            # there were two ways if categories are present
-            # - the old one, where each value is a int pointer to the levels
-            #   array -> not anymore possible, but code outside of pandas could
-            #   call us like that, so make some checks
-            # - the new one, where each value is also in the categories array
-            #   (or np.nan)
+        elif is_categorical_dtype(values):
+            old_codes = (values.cat.codes if isinstance(values, ABCSeries)
+                         else values.codes)
+            codes = _recode_for_categories(old_codes, values.dtype.categories,
+                                           dtype.categories)
 
+        else:
             codes = _get_codes_for_values(values, dtype.categories)
 
-            # TODO: check for old style usage. These warnings should be removes
-            # after 0.18/ in 2016
-            if (is_integer_dtype(values) and
-                    not is_integer_dtype(dtype.categories)):
-                warn("Values and categories have different dtypes. Did you "
-                     "mean to use\n'Categorical.from_codes(codes, "
-                     "categories)'?", RuntimeWarning, stacklevel=2)
-
-            if (len(values) and is_integer_dtype(values) and
-                    (codes == -1).all()):
-                warn("None of the categories were found in values. Did you "
-                     "mean to use\n'Categorical.from_codes(codes, "
-                     "categories)'?", RuntimeWarning, stacklevel=2)
-
         if null_mask.any():
             # Reinsert -1 placeholders for previously removed missing values
             full_codes = - np.ones(null_mask.shape, dtype=codes.dtype)

pandas/core/frame.py

@@ -4637,7 +4637,7 @@ def unstack(self, level=-1, fill_value=None):
                    other='melt'))
     def melt(self, id_vars=None, value_vars=None, var_name=None,
              value_name='value', col_level=None):
-        from pandas.core.reshape.reshape import melt
+        from pandas.core.reshape.melt import melt
         return melt(self, id_vars=id_vars, value_vars=value_vars,
                     var_name=var_name, value_name=value_name,
                     col_level=col_level)

pandas/core/indexes/multi.py

@@ -446,6 +446,17 @@ def _shallow_copy_with_infer(self, values=None, **kwargs):
                               **kwargs)
         return self._shallow_copy(values, **kwargs)
 
+    @Appender(_index_shared_docs['__contains__'] % _index_doc_kwargs)
+    def __contains__(self, key):
+        hash(key)
+        try:
+            self.get_loc(key)
+            return True
+        except (LookupError, TypeError):
+            return False
+
+    contains = __contains__
+
     @Appender(_index_shared_docs['_shallow_copy'])
     def _shallow_copy(self, values=None, **kwargs):
         if values is not None:
@@ -1370,17 +1381,6 @@ def nlevels(self):
     def levshape(self):
         return tuple(len(x) for x in self.levels)
 
-    @Appender(_index_shared_docs['__contains__'] % _index_doc_kwargs)
-    def __contains__(self, key):
-        hash(key)
-        try:
-            self.get_loc(key)
-            return True
-        except LookupError:
-            return False
-
-    contains = __contains__
-
     def __reduce__(self):
         """Necessary for making this object picklable"""
         d = dict(levels=[lev for lev in self.levels],

pandas/core/reshape/api.py

@@ -1,7 +1,7 @@
 # flake8: noqa
 
 from pandas.core.reshape.concat import concat
-from pandas.core.reshape.reshape import melt
+from pandas.core.reshape.melt import melt
 from pandas.core.reshape.merge import (
     merge, ordered_merge, merge_ordered, merge_asof)
 from pandas.core.reshape.pivot import pivot_table, crosstab