ENH: enhance apply() on Panel for arbitrary functions (rather than just ufuncs) (GH1148)

ENH: add property dtypes on Panel objects

Author: jreback

doc/source/release.rst

@@ -73,6 +73,8 @@ Improvements to existing features
   - df.info() view now display dtype info per column (:issue: `5682`)
   - perf improvements in DataFrame ``count/dropna`` for ``axis=1``
   - Series.str.contains now has a `regex=False` keyword which can be faster for plain (non-regex) string patterns. (:issue: `5879`)
+  - support ``dtypes`` on ``Panel``
+  - extend ``Panel.apply`` to allow arbitrary functions (rather than only ufuncs) (:issue:`1148`)
 
 .. _release.bug_fixes-0.13.1:
 

pandas/core/panel.py

@@ -17,8 +17,10 @@
 from pandas.core.internals import (BlockManager,
                                    create_block_manager_from_arrays,
                                    create_block_manager_from_blocks)
+from pandas.core.series import Series
 from pandas.core.frame import DataFrame
 from pandas.core.generic import NDFrame, _shared_docs
+from pandas.tools.util import cartesian_product
 from pandas import compat
 from pandas.util.decorators import deprecate, Appender, Substitution
 import pandas.core.common as com
@@ -333,26 +335,34 @@ def axis_pretty(a):
             [class_name, dims] + [axis_pretty(a) for a in self._AXIS_ORDERS])
         return output
 
-    def _get_plane_axes(self, axis):
+    def _get_plane_axes_index(self, axis):
         """
-        Get my plane axes: these are already
+        Get my plane axes indexes: these are already
         (as compared with higher level planes),
-        as we are returning a DataFrame axes
+        as we are returning a DataFrame axes indexes
         """
-        axis = self._get_axis_name(axis)
+        axis_name = self._get_axis_name(axis)
 
-        if axis == 'major_axis':
-            index = self.minor_axis
-            columns = self.items
-        if axis == 'minor_axis':
-            index = self.major_axis
-            columns = self.items
-        elif axis == 'items':
-            index = self.major_axis
-            columns = self.minor_axis
+        if axis_name == 'major_axis':
+            index = 'minor_axis'
+            columns = 'items'
+        if axis_name == 'minor_axis':
+            index = 'major_axis'
+            columns = 'items'
+        elif axis_name == 'items':
+            index = 'major_axis'
+            columns = 'minor_axis'
 
         return index, columns
 
+    def _get_plane_axes(self, axis):
+        """
+        Get my plane axes indexes: these are already
+        (as compared with higher level planes),
+        as we are returning a DataFrame axes
+        """
+        return [ self._get_axis(axi) for axi in self._get_plane_axes_index(axis) ]
+
     fromDict = from_dict
 
     def to_sparse(self, fill_value=None, kind='block'):
@@ -431,6 +441,10 @@ def as_matrix(self):
         self._consolidate_inplace()
         return self._data.as_matrix()
 
+    @property
+    def dtypes(self):
+        return self.apply(lambda x: x.dtype, axis='items')
+
     #----------------------------------------------------------------------
     # Getting and setting elements
 
@@ -827,25 +841,104 @@ def to_frame(self, filter_observations=True):
     to_long = deprecate('to_long', to_frame)
     toLong = deprecate('toLong', to_frame)
 
-    def apply(self, func, axis='major'):
+    def apply(self, func, axis='major', args=(), **kwargs):
         """
-        Apply
+        Applies function along input axis of the Panel
 
         Parameters
         ----------
-        func : numpy function
-            Signature should match numpy.{sum, mean, var, std} etc.
+        func : function
+            Function to apply to each combination of 'other' axes
+            e.g. if axis = 'items', then the combination of major_axis/minor_axis
+            will be passed a Series
         axis : {'major', 'minor', 'items'}
-        fill_value : boolean, default True
-            Replace NaN values with specified first
+        args : tuple
+            Positional arguments to pass to function in addition to the
+            array/series
+        Additional keyword arguments will be passed as keywords to the function
+
+        Examples
+        --------
+        >>> p.apply(numpy.sqrt) # returns a Panel
+        >>> p.apply(lambda x: x.sum(), axis=0) # equiv to p.sum(0)
+        >>> p.apply(lambda x: x.sum(), axis=1) # equiv to p.sum(1)
+        >>> p.apply(lambda x: x.sum(), axis=2) # equiv to p.sum(2)
 
         Returns
         -------
-        result : DataFrame or Panel
+        result : Pandas Object
         """
-        i = self._get_axis_number(axis)
-        result = np.apply_along_axis(func, i, self.values)
-        return self._wrap_result(result, axis=axis)
+        axis = self._get_axis_number(axis)
+        axis_name = self._get_axis_name(axis)
+        ax = self._get_axis(axis)
+        values = self.values
+        ndim = self.ndim
+
+        if args or kwargs and not isinstance(func, np.ufunc):
+            f = lambda x: func(x, *args, **kwargs)
+        else:
+            f = func
+
+        # try ufunc like
+        if isinstance(f, np.ufunc):
+            try:
+                result = np.apply_along_axis(func, axis, values)
+                return self._wrap_result(result, axis=axis)
+            except (AttributeError):
+                pass
+
+        # iter thru the axes
+        slice_axis = self._get_axis(axis)
+        slice_indexer = [0]*(ndim-1)
+        indexer = np.zeros(ndim, 'O')
+        indlist = list(range(ndim))
+        indlist.remove(axis)
+        indexer[axis] = slice(None, None)
+        indexer.put(indlist, slice_indexer)
+        planes = [ self._get_axis(axi) for axi in indlist ]
+        shape = np.array(self.shape).take(indlist)
+
+        # all the iteration points
+        points = cartesian_product(planes)
+
+        results = []
+        for i in xrange(np.prod(shape)):
+
+            # construct the object
+            pts = tuple([ p[i] for p in points ])
+            indexer.put(indlist, slice_indexer)
+
+            obj = Series(values[tuple(indexer)],index=slice_axis,name=pts)
+            result = func(obj, *args, **kwargs)
+
+            results.append(result)
+
+            # increment the indexer
+            slice_indexer[-1] += 1
+            n = -1
+            while (slice_indexer[n] >= shape[n]) and (n > (1-ndim)):
+                slice_indexer[n-1] += 1
+                slice_indexer[n] = 0
+                n -= 1
+
+        # empty object
+        if not len(results):
+            return self._constructor(**self._construct_axes_dict())
+
+        # same ndim as current
+        if isinstance(results[0],Series):
+            arr = np.vstack([ r.values for r in results ])
+            arr = arr.T.reshape(tuple([len(slice_axis)] + list(shape)))
+            tranp = np.array([axis]+indlist).argsort()
+            arr = arr.transpose(tuple(list(tranp)))
+            return self._constructor(arr,**self._construct_axes_dict())
+
+        # ndim-1 shape
+        results = np.array(results).reshape(shape)
+        if results.ndim == 2 and axis_name != self._info_axis_name:
+            results = results.T
+            planes = planes[::-1]
+        return self._construct_return_type(results,planes)
 
     def _reduce(self, op, axis=0, skipna=True, numeric_only=None,
                 filter_type=None, **kwds):
@@ -863,21 +956,41 @@ def _reduce(self, op, axis=0, skipna=True, numeric_only=None,
 
     def _construct_return_type(self, result, axes=None, **kwargs):
         """ return the type for the ndim of the result """
-        ndim = result.ndim
-        if self.ndim == ndim:
+        ndim = getattr(result,'ndim',None)
+
+        # need to assume they are the same
+        if ndim is None:
+            if isinstance(result,dict):
+                ndim = getattr(result.values()[0],'ndim',None)
+
+                # a saclar result
+                if ndim is None:
+                    ndim = 0
+
+                # have a dict, so top-level is +1 dim
+                else:
+                    ndim += 1
+
+        # scalar
+        if ndim == 0:
+            return Series(result)
+
+        # same as self
+        elif self.ndim == ndim:
             """ return the construction dictionary for these axes """
             if axes is None:
                 return self._constructor(result)
             return self._constructor(result, **self._construct_axes_dict())
 
+        # sliced
         elif self.ndim == ndim + 1:
             if axes is None:
                 return self._constructor_sliced(result)
             return self._constructor_sliced(
                 result, **self._extract_axes_for_slice(self, axes))
 
         raise PandasError('invalid _construct_return_type [self->%s] '
-                          '[result->%s]' % (self.ndim, result.ndim))
+                          '[result->%s]' % (self, result))
 
     def _wrap_result(self, result, axis):
         axis = self._get_axis_name(axis)

pandas/core/panelnd.py

@@ -56,9 +56,10 @@ def __init__(self, *args, **kwargs):
         self._init_data(*args, **kwargs)
     klass.__init__ = __init__
 
-    def _get_plane_axes(self, axis):
+    def _get_plane_axes_index(self, axis):
+        """ return the sliced index for this object """
 
-        axis = self._get_axis_name(axis)
+        axis_name = self._get_axis_name(axis)
         index = self._AXIS_ORDERS.index(axis)
 
         planes = []
@@ -67,8 +68,8 @@ def _get_plane_axes(self, axis):
         if index != self._AXIS_LEN:
             planes.extend(self._AXIS_ORDERS[index + 1:])
 
-        return [getattr(self, p) for p in planes]
-    klass._get_plane_axes = _get_plane_axes
+        return planes
+    klass._get_plane_axes_index = _get_plane_axes_index
 
     def _combine(self, other, func, axis=0):
         if isinstance(other, klass):

pandas/tests/test_panel.py

@@ -1061,6 +1061,64 @@ def test_convert_objects(self):
         result = p.convert_objects(convert_numeric='force')
         assert_panel_equal(result, expected)
 
+    def test_dtypes(self):
+
+        result = self.panel.dtypes
+        expected = DataFrame(np.dtype('float64'),index=self.panel.major_axis,columns=self.panel.minor_axis)
+        assert_frame_equal(result, expected)
+
+    def test_apply(self):
+        # GH1148
+
+        from pandas import Series,DataFrame
+
+        # ufunc
+        applied = self.panel.apply(np.sqrt)
+        self.assert_(assert_almost_equal(applied.values,
+                                         np.sqrt(self.panel.values)))
+
+        # ufunc same shape
+        result = self.panel.apply(lambda x: x*2, axis='items')
+        expected = self.panel*2
+        assert_panel_equal(result, expected)
+        result = self.panel.apply(lambda x: x*2, axis='major_axis')
+        expected = self.panel*2
+        assert_panel_equal(result, expected)
+        result = self.panel.apply(lambda x: x*2, axis='minor_axis')
+        expected = self.panel*2
+        assert_panel_equal(result, expected)
+
+        # reduction to DataFrame
+        result = self.panel.apply(lambda x: x.dtype, axis='items')
+        expected = DataFrame(np.dtype('float64'),index=self.panel.major_axis,columns=self.panel.minor_axis)
+        assert_frame_equal(result,expected)
+        result = self.panel.apply(lambda x: x.dtype, axis='major_axis')
+        expected = DataFrame(np.dtype('float64'),index=self.panel.minor_axis,columns=self.panel.items)
+        assert_frame_equal(result,expected)
+        result = self.panel.apply(lambda x: x.dtype, axis='minor_axis')
+        expected = DataFrame(np.dtype('float64'),index=self.panel.major_axis,columns=self.panel.items)
+        assert_frame_equal(result,expected)
+
+        # reductions via other dims
+        expected = self.panel.sum(0)
+        result = self.panel.apply(lambda x: x.sum(), axis='items')
+        assert_frame_equal(result,expected)
+        expected = self.panel.sum(1)
+        result = self.panel.apply(lambda x: x.sum(), axis='major_axis')
+        assert_frame_equal(result,expected)
+        expected = self.panel.sum(2)
+        result = self.panel.apply(lambda x: x.sum(), axis='minor_axis')
+        assert_frame_equal(result,expected)
+
+        # pass args
+        result = self.panel.apply(lambda x, y: x.sum() + y, axis='items', args=[5])
+        expected = self.panel.sum(0) + 5
+        assert_frame_equal(result,expected)
+
+        result = self.panel.apply(lambda x, y: x.sum() + y, axis='items', y=5)
+        expected = self.panel.sum(0) + 5
+        assert_frame_equal(result,expected)
+
     def test_reindex(self):
         ref = self.panel['ItemB']
 
@@ -1989,12 +2047,6 @@ def test_get_dummies(self):
         dummies = get_dummies(self.panel['Label'])
         self.assert_(np.array_equal(dummies.values, minor_dummies.values))
 
-    def test_apply(self):
-        # ufunc
-        applied = self.panel.apply(np.sqrt)
-        self.assert_(assert_almost_equal(applied.values,
-                                         np.sqrt(self.panel.values)))
-
     def test_mean(self):
         means = self.panel.mean(level='minor')