use more memoryviews

Author: jbrockmendel

pandas/_libs/algos.pyx

@@ -77,7 +77,7 @@ class NegInfinity(object):
     __ge__ = lambda self, other: isinstance(other, NegInfinity)
 
 
-cpdef ndarray[int64_t, ndim=1] unique_deltas(ndarray[int64_t] arr):
+cpdef ndarray[int64_t, ndim=1] unique_deltas(int64_t[:] arr):
     """
     Efficiently find the unique first-differences of the given array.
 
@@ -236,7 +236,7 @@ def nancorr(ndarray[float64_t, ndim=2] mat, bint cov=0, minp=None):
         Py_ssize_t i, j, xi, yi, N, K
         bint minpv
         ndarray[float64_t, ndim=2] result
-        ndarray[uint8_t, ndim=2] mask
+        uint8_t[:, :] mask
         int64_t nobs = 0
         float64_t vx, vy, sumx, sumy, sumxx, sumyy, meanx, meany, divisor
 
@@ -301,7 +301,7 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1):
         ndarray[float64_t, ndim=2] result
         ndarray[float64_t, ndim=1] maskedx
         ndarray[float64_t, ndim=1] maskedy
-        ndarray[uint8_t, ndim=2] mask
+        uint8_t[:, :] mask
         int64_t nobs = 0
         float64_t vx, vy, sumx, sumxx, sumyy, mean, divisor
 
@@ -475,10 +475,7 @@ pad_bool = pad["uint8_t"]
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def pad_inplace(algos_t[:] values,
-                ndarray[uint8_t, cast=True] mask,
-                # TODO: What does the cast=True mean?  If unneeded, use bint[:]?
-                limit=None):
+def pad_inplace(algos_t[:] values, uint8_t[:] mask, limit=None):
     cdef:
         Py_ssize_t i, N
         algos_t val
@@ -522,9 +519,7 @@ pad_inplace_bool = pad_inplace["uint8_t"]
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def pad_2d_inplace(ndarray[algos_t, ndim=2] values,
-                   ndarray[uint8_t, ndim=2] mask,
-                   limit=None):
+def pad_2d_inplace(algos_t[:, :] values, uint8_t[:, :] mask, limit=None):
     cdef:
         Py_ssize_t i, j, N, K
         algos_t val
@@ -675,9 +670,7 @@ backfill_bool = backfill["uint8_t"]
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def backfill_inplace(algos_t[:] values,
-                     ndarray[uint8_t, cast=True] mask,
-                     limit=None):
+def backfill_inplace(algos_t[:] values, uint8_t[:] mask, limit=None):
     cdef:
         Py_ssize_t i, N
         algos_t val
@@ -721,9 +714,7 @@ backfill_inplace_bool = backfill_inplace["uint8_t"]
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def backfill_2d_inplace(ndarray[algos_t, ndim=2] values,
-                        ndarray[uint8_t, ndim=2] mask,
-                        limit=None):
+def backfill_2d_inplace(algos_t[:, :] values, uint8_t[:, :] mask, limit=None):
     cdef:
         Py_ssize_t i, j, N, K
         algos_t val

pandas/_libs/algos_common_helper.pxi.in

@@ -1,16 +1,6 @@
 """
 Template for each `dtype` helper function using 1-d template
 
-# 1-d template
-- pad
-- pad_1d
-- pad_2d
-- backfill
-- backfill_1d
-- backfill_2d
-- is_monotonic
-- arrmap
-
 WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 """
 
@@ -39,7 +29,7 @@ def get_dispatch(dtypes):
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def diff_2d_{{name}}(ndarray[{{c_type}}, ndim=2] arr,
-                     ndarray[{{dest_type}}, ndim=2] out,
+                     {{dest_type}}[:, :] out,
                      Py_ssize_t periods, int axis):
     cdef:
         Py_ssize_t i, j, sx, sy
@@ -83,7 +73,7 @@ def diff_2d_{{name}}(ndarray[{{c_type}}, ndim=2] arr,
 
 def put2d_{{name}}_{{dest_name}}(ndarray[{{c_type}}, ndim=2, cast=True] values,
                                  ndarray[int64_t] indexer, Py_ssize_t loc,
-                                 ndarray[{{dest_type}}] out):
+                                 {{dest_type}}[:, :] out):
     cdef:
         Py_ssize_t i, j, k
 

pandas/_libs/groupby.pyx

@@ -99,16 +99,16 @@ cdef inline float64_t kth_smallest_c(float64_t* a,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_median_float64(ndarray[float64_t, ndim=2] out,
-                         ndarray[int64_t] counts,
+                         int64_t[:] counts,
                          ndarray[float64_t, ndim=2] values,
-                         ndarray[int64_t] labels,
+                         int64_t[:] labels,
                          Py_ssize_t min_count=-1):
     """
     Only aggregates on axis=0
     """
     cdef:
         Py_ssize_t i, j, N, K, ngroups, size
-        ndarray[int64_t] _counts
+        int64_t[:] _counts
         ndarray data
         float64_t* ptr
 
@@ -217,7 +217,7 @@ def group_cumsum(numeric[:, :] out,
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def group_shift_indexer(ndarray[int64_t] out, ndarray[int64_t] labels,
+def group_shift_indexer(int64_t[:] out, int64_t[:] labels,
                         int ngroups, int periods):
     cdef:
         Py_ssize_t N, i, j, ii
@@ -269,8 +269,8 @@ def group_shift_indexer(ndarray[int64_t] out, ndarray[int64_t] labels,
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def group_fillna_indexer(ndarray[int64_t] out, ndarray[int64_t] labels,
-                         ndarray[uint8_t] mask, object direction,
+def group_fillna_indexer(int64_t[:] out, ndarray[int64_t] labels,
+                         uint8_t[:] mask, object direction,
                          int64_t limit):
     """Indexes how to fill values forwards or backwards within a group
 
@@ -328,9 +328,9 @@ def group_fillna_indexer(ndarray[int64_t] out, ndarray[int64_t] labels,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_any_all(ndarray[uint8_t] out,
-                  ndarray[int64_t] labels,
-                  ndarray[uint8_t] values,
-                  ndarray[uint8_t] mask,
+                  int64_t[:] labels,
+                  uint8_t[:] values,
+                  uint8_t[:] mask,
                   object val_test,
                   bint skipna):
     """Aggregated boolean values to show truthfulness of group elements
@@ -353,7 +353,7 @@ def group_any_all(ndarray[uint8_t] out,
     The returned values will either be 0 or 1 (False or True, respectively).
     """
     cdef:
-        Py_ssize_t i, N=len(labels)
+        Py_ssize_t i, N = len(labels)
         int64_t lab
         uint8_t flag_val
 

pandas/_libs/groupby_helper.pxi.in

@@ -30,17 +30,17 @@ def get_dispatch(dtypes):
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_add_{{name}}(ndarray[{{c_type}}, ndim=2] out,
-                       ndarray[int64_t] counts,
+                       int64_t[:] counts,
                        ndarray[{{c_type}}, ndim=2] values,
-                       ndarray[int64_t] labels,
+                       int64_t[:] labels,
                        Py_ssize_t min_count=0):
     """
     Only aggregates on axis=0
     """
     cdef:
         Py_ssize_t i, j, N, K, lab, ncounts = len(counts)
         {{c_type}} val, count
-        ndarray[{{c_type}}, ndim=2] sumx, nobs
+        {{c_type}}[:, :] sumx, nobs
 
     if not len(values) == len(labels):
         raise AssertionError("len(index) != len(labels)")
@@ -77,17 +77,17 @@ def group_add_{{name}}(ndarray[{{c_type}}, ndim=2] out,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_prod_{{name}}(ndarray[{{c_type}}, ndim=2] out,
-                        ndarray[int64_t] counts,
+                        int64_t[:] counts,
                         ndarray[{{c_type}}, ndim=2] values,
-                        ndarray[int64_t] labels,
+                        int64_t[:] labels,
                         Py_ssize_t min_count=0):
     """
     Only aggregates on axis=0
     """
     cdef:
         Py_ssize_t i, j, N, K, lab, ncounts = len(counts)
         {{c_type}} val, count
-        ndarray[{{c_type}}, ndim=2] prodx, nobs
+        {{c_type}}[:, :] prodx, nobs
 
     if not len(values) == len(labels):
         raise AssertionError("len(index) != len(labels)")
@@ -124,14 +124,14 @@ def group_prod_{{name}}(ndarray[{{c_type}}, ndim=2] out,
 @cython.boundscheck(False)
 @cython.cdivision(True)
 def group_var_{{name}}(ndarray[{{c_type}}, ndim=2] out,
-                       ndarray[int64_t] counts,
+                       int64_t[:] counts,
                        ndarray[{{c_type}}, ndim=2] values,
-                       ndarray[int64_t] labels,
+                       int64_t[:] labels,
                        Py_ssize_t min_count=-1):
     cdef:
         Py_ssize_t i, j, N, K, lab, ncounts = len(counts)
         {{c_type}} val, ct, oldmean
-        ndarray[{{c_type}}, ndim=2] nobs, mean
+        {{c_type}}[:, :] nobs, mean
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
@@ -176,14 +176,14 @@ def group_var_{{name}}(ndarray[{{c_type}}, ndim=2] out,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_mean_{{name}}(ndarray[{{c_type}}, ndim=2] out,
-                        ndarray[int64_t] counts,
+                        int64_t[:] counts,
                         ndarray[{{c_type}}, ndim=2] values,
                         ndarray[int64_t] labels,
                         Py_ssize_t min_count=-1):
     cdef:
         Py_ssize_t i, j, N, K, lab, ncounts = len(counts)
         {{c_type}} val, count
-        ndarray[{{c_type}}, ndim=2] sumx, nobs
+        {{c_type}}[:, :] sumx, nobs
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
@@ -221,9 +221,9 @@ def group_mean_{{name}}(ndarray[{{c_type}}, ndim=2] out,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_ohlc_{{name}}(ndarray[{{c_type}}, ndim=2] out,
-                  ndarray[int64_t] counts,
+                  int64_t[:] counts,
                   ndarray[{{c_type}}, ndim=2] values,
-                  ndarray[int64_t] labels,
+                  int64_t[:] labels,
                   Py_ssize_t min_count=-1):
     """
     Only aggregates on axis=0