Merge pull request #235 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

doc/source/user_guide/boolean.rst

@@ -12,6 +12,11 @@
 Nullable Boolean data type
 **************************
 
+.. note::
+
+   BooleanArray is currently experimental. Its API or implementation may
+   change without warning.
+
 .. versionadded:: 1.0.0
 
 

doc/source/whatsnew/v1.3.1.rst

@@ -25,6 +25,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.isin` and :meth:`Series.isin` raising ``TypeError`` with nullable data containing at least one missing value (:issue:`42405`)
 - Regression in :func:`concat` between objects with bool dtype and integer dtype casting to object instead of to integer (:issue:`42092`)
 - Bug in :class:`Series` constructor not accepting a ``dask.Array`` (:issue:`38645`)
+- Fixed regression for ``SettingWithCopyWarning`` displaying incorrect stacklevel (:issue:`42570`)
 - Fixed regression in :func:`to_datetime` returning pd.NaT for inputs that produce duplicated values, when ``cache=True`` (:issue:`42259`)
 
 
@@ -36,7 +37,7 @@ Bug fixes
 ~~~~~~~~~
 - Fixed bug in :meth:`DataFrame.transpose` dropping values when the DataFrame had an Extension Array dtype and a duplicate index (:issue:`42380`)
 - Fixed bug in :meth:`DataFrame.to_xml` raising ``KeyError`` when called with ``index=False`` and an offset index (:issue:`42458`)
--
+- Fixed bug in :meth:`DataFrame.copy` failing to consolidate blocks in the result (:issue:`42579`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/frame.py

@@ -3745,7 +3745,7 @@ def _set_item_mgr(self, key, value: ArrayLike) -> None:
         # try to set first as we want an invalid
         # value exception to occur first
         if len(self):
-            self._check_setitem_copy()
+            self._check_setitem_copy(stacklevel=5)
 
     def _iset_item(self, loc: int, value) -> None:
         arraylike = self._sanitize_column(value)

pandas/core/internals/managers.py

@@ -598,6 +598,9 @@ def copy_func(ax):
 
         res = self.apply("copy", deep=deep)
         res.axes = new_axes
+
+        if deep:
+            res._consolidate_inplace()
         return res
 
     def consolidate(self: T) -> T:

pandas/core/reshape/reshape.py

@@ -402,7 +402,8 @@ def _unstack_multiple(data, clocs, fill_value=None):
 
             return result
 
-        dummy = data.copy()
+        # GH#42579 deep=False to avoid consolidating
+        dummy = data.copy(deep=False)
         dummy.index = dummy_index
 
         unstacked = dummy.unstack("__placeholder__", fill_value=fill_value)

pandas/tests/frame/methods/test_copy.py

@@ -1,5 +1,8 @@
+import numpy as np
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import DataFrame
 import pandas._testing as tm
 
@@ -41,3 +44,20 @@ def test_copy(self, float_frame, float_string_frame):
         # copy objects
         copy = float_string_frame.copy()
         assert copy._mgr is not float_string_frame._mgr
+
+    @td.skip_array_manager_invalid_test
+    def test_copy_consolidates(self):
+        # GH#42477
+        df = DataFrame(
+            {
+                "a": np.random.randint(0, 100, size=55),
+                "b": np.random.randint(0, 100, size=55),
+            }
+        )
+
+        for i in range(0, 10):
+            df.loc[:, f"n_{i}"] = np.random.randint(0, 100, size=55)
+
+        assert len(df._mgr.blocks) == 11
+        result = df.copy()
+        assert len(result._mgr.blocks) == 1

pandas/tests/indexing/test_chaining_and_caching.py

@@ -435,6 +435,16 @@ def test_detect_chained_assignment_warnings_filter_and_dupe_cols(self):
             )
             tm.assert_frame_equal(df, expected)
 
+    @pytest.mark.parametrize("rhs", [3, DataFrame({0: [1, 2, 3, 4]})])
+    def test_detect_chained_assignment_warning_stacklevel(self, rhs):
+        # GH#42570
+        df = DataFrame(np.arange(25).reshape(5, 5))
+        chained = df.loc[:3]
+        with option_context("chained_assignment", "warn"):
+            with tm.assert_produces_warning(com.SettingWithCopyWarning) as t:
+                chained[2] = rhs
+                assert t[0].filename == __file__
+
     # TODO(ArrayManager) fast_xs with array-like scalars is not yet working
     @td.skip_array_manager_not_yet_implemented
     def test_chained_getitem_with_lists(self):

pandas/tests/internals/test_internals.py

@@ -461,6 +461,9 @@ def test_copy(self, mgr):
                 # DatetimeTZBlock has DatetimeIndex values
                 assert cp_blk.values._data.base is blk.values._data.base
 
+        # copy(deep=True) consolidates, so the block-wise assertions will
+        #  fail is mgr is not consolidated
+        mgr._consolidate_inplace()
         cp = mgr.copy(deep=True)
         for blk, cp_blk in zip(mgr.blocks, cp.blocks):
 

pandas/tests/reshape/test_pivot.py

@@ -2148,6 +2148,23 @@ def test_pivot_table_sort_false(self):
         )
         tm.assert_frame_equal(result, expected)
 
+    def test_pivot_table_with_margins_and_numeric_columns(self):
+        # GH 26568
+        df = DataFrame([["a", "x", 1], ["a", "y", 2], ["b", "y", 3], ["b", "z", 4]])
+        df.columns = [10, 20, 30]
+
+        result = df.pivot_table(
+            index=10, columns=20, values=30, aggfunc="sum", fill_value=0, margins=True
+        )
+
+        expected = DataFrame([[1, 2, 0, 3], [0, 3, 4, 7], [1, 5, 4, 10]])
+        expected.columns = ["x", "y", "z", "All"]
+        expected.index = ["a", "b", "All"]
+        expected.columns.name = 20
+        expected.index.name = 10
+
+        tm.assert_frame_equal(result, expected)
+
 
 class TestPivot:
     def test_pivot(self):