Merge pull request #5462 from jreback/perf_isnull

PERF: performance improvements on isnull/notnull for large pandas objects

Author: jreback

doc/source/release.rst

@@ -803,6 +803,7 @@ Bug Fixes
   - Make tests create temp files in temp directory by default. (:issue:`5419`)
   - ``pd.to_timedelta`` of a scalar returns a scalar (:issue:`5410`)
   - ``pd.to_timedelta`` accepts ``NaN`` and ``NaT``, returning ``NaT`` instead of raising (:issue:`5437`)
+  - performance improvements in ``isnull`` on larger size pandas objects
 
 pandas 0.12.0
 -------------

pandas/core/common.py

@@ -128,7 +128,7 @@ def _isnull_new(obj):
     elif isinstance(obj, (ABCSeries, np.ndarray)):
         return _isnull_ndarraylike(obj)
     elif isinstance(obj, ABCGeneric):
-        return obj.apply(isnull)
+        return obj._constructor(obj._data.apply(lambda x: isnull(x.values)))
     elif isinstance(obj, list) or hasattr(obj, '__array__'):
         return _isnull_ndarraylike(np.asarray(obj))
     else:
@@ -155,7 +155,7 @@ def _isnull_old(obj):
     elif isinstance(obj, (ABCSeries, np.ndarray)):
         return _isnull_ndarraylike_old(obj)
     elif isinstance(obj, ABCGeneric):
-        return obj.apply(_isnull_old)
+        return obj._constructor(obj._data.apply(lambda x: _isnull_old(x.values)))
     elif isinstance(obj, list) or hasattr(obj, '__array__'):
         return _isnull_ndarraylike_old(np.asarray(obj))
     else:

pandas/core/generic.py

@@ -2208,13 +2208,13 @@ def isnull(self):
         """
         Return a boolean same-sized object indicating if the values are null
         """
-        return self.__class__(isnull(self),**self._construct_axes_dict()).__finalize__(self)
+        return isnull(self).__finalize__(self)
 
     def notnull(self):
         """
         Return a boolean same-sized object indicating if the values are not null
         """
-        return self.__class__(notnull(self),**self._construct_axes_dict()).__finalize__(self)
+        return notnull(self).__finalize__(self)
 
     def clip(self, lower=None, upper=None, out=None):
         """

pandas/core/internals.py

@@ -2153,6 +2153,13 @@ def apply(self, f, *args, **kwargs):
                     continue
             if callable(f):
                 applied = f(blk, *args, **kwargs)
+
+                # if we are no a block, try to coerce
+                if not isinstance(applied, Block):
+                    applied = make_block(applied,
+                                         blk.items,
+                                         blk.ref_items)
+
             else:
                 applied = getattr(blk, f)(*args, **kwargs)
 

pandas/tests/test_common.py

@@ -75,17 +75,28 @@ def test_isnull():
     assert not isnull(np.inf)
     assert not isnull(-np.inf)
 
+    # series
     for s in [tm.makeFloatSeries(),tm.makeStringSeries(),
               tm.makeObjectSeries(),tm.makeTimeSeries(),tm.makePeriodSeries()]:
             assert(isinstance(isnull(s), Series))
 
-    # call on DataFrame
-    df = DataFrame(np.random.randn(10, 5))
-    df['foo'] = 'bar'
-    result = isnull(df)
-    expected = result.apply(isnull)
-    tm.assert_frame_equal(result, expected)
-
+    # frame
+    for df in [tm.makeTimeDataFrame(),tm.makePeriodFrame(),tm.makeMixedDataFrame()]:
+        result = isnull(df)
+        expected = df.apply(isnull)
+        tm.assert_frame_equal(result, expected)
+
+    # panel
+    for p in [ tm.makePanel(), tm.makePeriodPanel(), tm.add_nans(tm.makePanel()) ]:
+        result = isnull(p)
+        expected = p.apply(isnull)
+        tm.assert_panel_equal(result, expected)
+
+    # panel 4d
+    for p in [ tm.makePanel4D(), tm.add_nans_panel4d(tm.makePanel4D()) ]:
+        result = isnull(p)
+        expected = p.apply(isnull)
+        tm.assert_panel4d_equal(result, expected)
 
 def test_isnull_lists():
     result = isnull([[False]])

pandas/util/testing.py

@@ -531,6 +531,9 @@ def assert_copy(iter1, iter2, **eql_kwargs):
 def getCols(k):
     return string.ascii_uppercase[:k]
 
+def getArangeMat():
+    return np.arange(N * K).reshape((N, K))
+
 
 # make index
 def makeStringIndex(k=10):
@@ -601,24 +604,20 @@ def getTimeSeriesData(nper=None):
     return dict((c, makeTimeSeries(nper)) for c in getCols(K))
 
 
+def getPeriodData(nper=None):
+    return dict((c, makePeriodSeries(nper)) for c in getCols(K))
+
+# make frame
 def makeTimeDataFrame(nper=None):
     data = getTimeSeriesData(nper)
     return DataFrame(data)
 
 
-def getPeriodData(nper=None):
-    return dict((c, makePeriodSeries(nper)) for c in getCols(K))
-
-# make frame
 def makeDataFrame():
     data = getSeriesData()
     return DataFrame(data)
 
 
-def getArangeMat():
-    return np.arange(N * K).reshape((N, K))
-
-
 def getMixedTypeDict():
     index = Index(['a', 'b', 'c', 'd', 'e'])
 
@@ -631,6 +630,8 @@ def getMixedTypeDict():
 
     return index, data
 
+def makeMixedDataFrame():
+    return DataFrame(getMixedTypeDict()[1])
 
 def makePeriodFrame(nper=None):
     data = getPeriodData(nper)
@@ -827,13 +828,13 @@ def add_nans(panel):
         dm = panel[item]
         for j, col in enumerate(dm.columns):
             dm[col][:i + j] = np.NaN
-
+    return panel
 
 def add_nans_panel4d(panel4d):
     for l, label in enumerate(panel4d.labels):
         panel = panel4d[label]
         add_nans(panel)
-
+    return panel4d
 
 class TestSubDict(dict):
 

vb_suite/frame_methods.py

@@ -221,6 +221,9 @@ def f(K=100):
 
 frame_xs_col = Benchmark('df.xs(50000,axis = 1)', setup)
 
+#----------------------------------------------------------------------
+# nulls/masking
+
 ## masking
 setup = common_setup + """
 data = np.random.randn(1000, 500)
@@ -230,8 +233,17 @@ def f(K=100):
 mask = isnull(df)
 """
 
-mask_bools = Benchmark('bools.mask(mask)', setup,
-                         start_date=datetime(2013,1,1))
+frame_mask_bools = Benchmark('bools.mask(mask)', setup,
+                             start_date=datetime(2013,1,1))
+
+frame_mask_floats  = Benchmark('bools.astype(float).mask(mask)', setup,
+                             start_date=datetime(2013,1,1))
+
+## isnull
+setup = common_setup + """
+data = np.random.randn(1000, 1000)
+df = DataFrame(data)
+"""
+frame_isnull  = Benchmark('isnull(df)', setup,
+                           start_date=datetime(2012,1,1))
 
-mask_floats  = Benchmark('bools.astype(float).mask(mask)', setup,
-                         start_date=datetime(2013,1,1))