Merge remote-tracking branch 'upstream/master' into categorical-csv-2

Author: TomAugspurger

asv_bench/benchmarks/sparse.py

@@ -1,4 +1,4 @@
-from itertools import repeat
+import itertools
 
 from .pandas_vb_common import *
 import scipy.sparse
@@ -33,7 +33,7 @@ def time_sparse_from_scipy(self):
         SparseDataFrame(scipy.sparse.rand(1000, 1000, 0.005))
 
     def time_sparse_from_dict(self):
-        SparseDataFrame(dict(zip(range(1000), repeat([0]))))
+        SparseDataFrame(dict(zip(range(1000), itertools.repeat([0]))))
 
 
 class sparse_series_from_coo(object):

ci/requirements-3.6_NUMPY_DEV.build

@@ -1,3 +1,2 @@
 python=3.6*
 pytz
-cython

ci/requirements-3.6_NUMPY_DEV.build.sh

@@ -14,4 +14,7 @@ pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS numpy scipy
 # install dateutil from master
 pip install -U git+git://github.com/dateutil/dateutil.git
 
+# cython via pip
+pip install cython
+
 true

doc/source/10min.rst

@@ -11,7 +11,7 @@
    np.random.seed(123456)
    np.set_printoptions(precision=4, suppress=True)
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    pd.options.display.max_rows = 15
 
    #### portions of this were borrowed from the
@@ -95,17 +95,7 @@ will be completed:
    df2.append             df2.combine_first
    df2.apply              df2.compound
    df2.applymap           df2.consolidate
-   df2.as_blocks          df2.convert_objects
-   df2.asfreq             df2.copy
-   df2.as_matrix          df2.corr
-   df2.astype             df2.corrwith
-   df2.at                 df2.count
-   df2.at_time            df2.cov
-   df2.axes               df2.cummax
-   df2.B                  df2.cummin
-   df2.between_time       df2.cumprod
-   df2.bfill              df2.cumsum
-   df2.blocks             df2.D
+   df2.D
 
 As you can see, the columns ``A``, ``B``, ``C``, and ``D`` are automatically
 tab completed. ``E`` is there as well; the rest of the attributes have been

doc/source/computation.rst

@@ -8,7 +8,7 @@
    np.set_printoptions(precision=4, suppress=True)
    import pandas as pd
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
    plt.close('all')
    pd.options.display.max_rows=15

doc/source/cookbook.rst

@@ -20,7 +20,7 @@
    pd.options.display.max_rows=15
 
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
 
    np.set_printoptions(precision=4, suppress=True)
 

doc/source/dsintro.rst

@@ -10,7 +10,7 @@
    pd.options.display.max_rows = 15
 
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
    plt.close('all')
 

doc/source/gotchas.rst

@@ -14,7 +14,7 @@ Frequently Asked Questions (FAQ)
    import pandas as pd
    pd.options.display.max_rows = 15
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
    plt.close('all')
 

doc/source/groupby.rst

@@ -10,7 +10,7 @@
    import pandas as pd
    pd.options.display.max_rows = 15
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
    plt.close('all')
    from collections import OrderedDict

doc/source/missing_data.rst

@@ -7,7 +7,7 @@
    import pandas as pd
    pd.options.display.max_rows=15
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
 
 .. _missing_data:

doc/source/options.rst

@@ -474,10 +474,10 @@ Unicode Formatting
    Enabling this option will affect the performance for printing of DataFrame and Series (about 2 times slower).
    Use only when it is actually required.
 
-Some East Asian countries use Unicode characters its width is corresponding to 2 alphabets.
-If DataFrame or Series contains these characters, default output cannot be aligned properly.
+Some East Asian countries use Unicode characters whose width corresponds to two Latin characters.
+If a DataFrame or Series contains these characters, the default output mode may not align them properly.
 
-.. note:: Screen captures are attached for each outputs to show the actual results.
+.. note:: Screen captures are attached for each output to show the actual results.
 
 .. ipython:: python
 
@@ -486,8 +486,9 @@ If DataFrame or Series contains these characters, default output cannot be align
 
 .. image:: _static/option_unicode01.png
 
-Enable ``display.unicode.east_asian_width`` allows pandas to check each character's "East Asian Width" property.
-These characters can be aligned properly by checking this property, but it takes longer time than standard ``len`` function.
+Enabling ``display.unicode.east_asian_width`` allows pandas to check each character's "East Asian Width" property.
+These characters can be aligned properly by setting this option to ``True``. However, this will result in longer render
+times than the standard ``len`` function.
 
 .. ipython:: python
 
@@ -496,9 +497,10 @@ These characters can be aligned properly by checking this property, but it takes
 
 .. image:: _static/option_unicode02.png
 
-In addition, Unicode contains characters which width is "Ambiguous". These character's width should be either 1 or 2 depending on terminal setting or encoding. Because this cannot be distinguished from Python, ``display.unicode.ambiguous_as_wide`` option is added to handle this.
+In addition, Unicode characters whose width is "Ambiguous" can either be 1 or 2 characters wide depending on the
+terminal setting or encoding. The option ``display.unicode.ambiguous_as_wide`` can be used to handle the ambiguity.
 
-By default, "Ambiguous" character's width, "¡" (inverted exclamation) in below example, is regarded as 1.
+By default, an "Ambiguous" character's width, such as "¡" (inverted exclamation) in the example below, is taken to be 1.
 
 .. ipython:: python
 
@@ -507,7 +509,10 @@ By default, "Ambiguous" character's width, "¡" (inverted exclamation) in below
 
 .. image:: _static/option_unicode03.png
 
-Enabling ``display.unicode.ambiguous_as_wide`` lets pandas to figure these character's width as 2. Note that this option will be effective only when ``display.unicode.east_asian_width`` is enabled. Confirm starting position has been changed, but is not aligned properly because the setting is mismatched with this environment.
+Enabling ``display.unicode.ambiguous_as_wide`` makes pandas interpret these characters' widths to be 2.
+(Note that this option will only be effective when ``display.unicode.east_asian_width`` is enabled.)
+
+However, setting this option incorrectly for your terminal will cause these characters to be aligned incorrectly:
 
 .. ipython:: python
 

doc/source/visualization.rst

@@ -10,7 +10,7 @@
    np.set_printoptions(precision=4, suppress=True)
    pd.options.display.max_rows = 15
    import matplotlib
-   matplotlib.style.use('ggplot')
+   # matplotlib.style.use('default')
    import matplotlib.pyplot as plt
    plt.close('all')
 
@@ -24,13 +24,6 @@ We use the standard convention for referencing the matplotlib API:
 
    import matplotlib.pyplot as plt
 
-The plots in this document are made using matplotlib's ``ggplot`` style (new in version 1.4):
-
-.. code-block:: python
-
-   import matplotlib
-   matplotlib.style.use('ggplot')
-
 We provide the basics in pandas to easily create decent looking plots.
 See the :ref:`ecosystem <ecosystem.visualization>` section for visualization
 libraries that go beyond the basics documented here.
@@ -134,7 +127,7 @@ For example, a bar plot can be created the following way:
    plt.figure();
 
    @savefig bar_plot_ex.png
-   df.iloc[5].plot(kind='bar'); plt.axhline(0, color='k')
+   df.iloc[5].plot(kind='bar');
 
 .. versionadded:: 0.17.0
 
@@ -154,7 +147,7 @@ and :ref:`DataFrame.boxplot() <visualization.box>` methods, which use a separate
 
 Finally, there are several :ref:`plotting functions <visualization.tools>` in ``pandas.plotting``
 that take a :class:`Series` or :class:`DataFrame` as an argument. These
-include
+include:
 
 * :ref:`Scatter Matrix <visualization.scatter_matrix>`
 * :ref:`Andrews Curves <visualization.andrews_curves>`
@@ -1049,6 +1042,21 @@ be colored differently.
 Plot Formatting
 ---------------
 
+Setting the plot style
+~~~~~~~~~~~~~~~~~~~~~~
+
+From version 1.5 and up, matplotlib offers a range of preconfigured plotting styles. Setting the
+style can be used to easily give plots the general look that you want.
+Setting the style is as easy as calling ``matplotlib.style.use(my_plot_style)`` before
+creating your plot. For example you could do ``matplotlib.style.use('ggplot')`` for ggplot-style
+plots.
+
+You can see the various available style names at ``matplotlib.style.available`` and it's very
+easy to try them out.
+
+General plot style arguments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Most plotting methods have a set of keyword arguments that control the
 layout and formatting of the returned plot:
 

doc/source/whatsnew/v0.21.0.txt

@@ -510,17 +510,16 @@ Other API Changes
 - :class:`Period` is now immutable, and will now raise an ``AttributeError`` when a user tries to assign a new value to the ``ordinal`` or ``freq`` attributes (:issue:`17116`).
 - :func:`to_datetime` when passed a tz-aware ``origin=`` kwarg will now raise a more informative ``ValueError`` rather than a ``TypeError`` (:issue:`16842`)
 - Renamed non-functional ``index`` to ``index_col`` in :func:`read_stata` to improve API consistency (:issue:`16342`)
-
+- Bug in :func:`DataFrame.drop` caused boolean labels ``False`` and ``True`` to be treated as labels 0 and 1 respectively when dropping indices from a numeric index. This will now raise a ValueError (:issue:`16877`)
 
 .. _whatsnew_0210.deprecations:
 
 Deprecations
 ~~~~~~~~~~~~
 - :func:`read_excel()` has deprecated ``sheetname`` in favor of ``sheet_name`` for consistency with ``.to_excel()`` (:issue:`10559`).
-
 - ``pd.options.html.border`` has been deprecated in favor of ``pd.options.display.html.border`` (:issue:`15793`).
-
 - :func:`SeriesGroupBy.nth` has deprecated ``True`` in favor of ``'all'`` for its kwarg ``dropna`` (:issue:`11038`).
+- :func:`DataFrame.as_blocks` is deprecated, as this is exposing the internal implementation (:issue:`17302`)
 
 .. _whatsnew_0210.prior_deprecations:
 
@@ -676,4 +675,5 @@ PyPy
 
 Other
 ^^^^^
+- Bug where some inplace operators were not being wrapped and produced a copy when invoked (:issue:`12962`)
 - Bug in :func:`eval` where the ``inplace`` parameter was being incorrectly handled (:issue:`16732`)

pandas/_libs/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # flake8: noqa
 
 from .tslib import iNaT, NaT, Timestamp, Timedelta, OutOfBoundsDatetime

pandas/_libs/src/datetime.pxd

@@ -94,6 +94,7 @@ cdef extern from "datetime/np_datetime.h":
                                            PANDAS_DATETIMEUNIT fr,
                                            pandas_datetimestruct *result) nogil
     int days_per_month_table[2][12]
+    pandas_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS
 
     int dayofweek(int y, int m, int d) nogil
     int is_leapyear(int64_t year) nogil
@@ -161,3 +162,17 @@ cdef inline int64_t _date_to_datetime64(object val,
     dts.hour = dts.min = dts.sec = dts.us = 0
     dts.ps = dts.as = 0
     return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts)
+
+
+cdef inline bint check_dts_bounds(pandas_datetimestruct *dts):
+    """Returns True if an error needs to be raised"""
+    cdef:
+        bint error = False
+
+    if (dts.year <= 1677 and
+            cmp_pandas_datetimestruct(dts, &_NS_MIN_DTS) == -1):
+        error = True
+    elif (dts.year >= 2262 and
+          cmp_pandas_datetimestruct(dts, &_NS_MAX_DTS) == 1):
+        error = True
+    return error

pandas/_libs/src/datetime/np_datetime.c

@@ -40,6 +40,12 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask
 #endif
 
+const pandas_datetimestruct _NS_MIN_DTS = {
+    1677, 9, 21, 0, 12, 43, 145225, 0, 0};
+const pandas_datetimestruct _NS_MAX_DTS = {
+    2262, 4, 11, 23, 47, 16, 854775, 807000, 0};
+
+
 const int days_per_month_table[2][12] = {
     {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
     {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}};

pandas/_libs/src/datetime/np_datetime.h

@@ -54,6 +54,9 @@ typedef struct {
     int num;
 } pandas_datetime_metadata;
 
+extern const pandas_datetimestruct _NS_MIN_DTS;
+extern const pandas_datetimestruct _NS_MAX_DTS;
+
 // stuff pandas needs
 // ----------------------------------------------------------------------------