Adapt DataCopyPad for get row f16 and f32

Author: leo-pony

ggml/src/ggml-cann/kernels/get_row_f16.cpp

@@ -60,32 +60,64 @@ class GET_ROW_F16 {
 
     __aicore__ inline void copy_in(uint32_t offset, size_t len) {
         LocalTensor<half> input_local = input_queue.AllocTensor<half>();
-        size_t tail = len % 32;
-        len = len & ~31;
+        const size_t elem_per_block = 32 / sizeof(half);
+        size_t tail = len % elem_per_block;
+        len = len & ~elem_per_block;
         DataCopy(input_local, input_gm[offset], len);
         if(tail != 0) {
+            DataCopy(input_local[len], input_gm[offset + len], elem_per_block);
+            // clean
+            for (int i = tail; i < elem_per_block; i++) {
+                input_local[len].SetValue(i, 0);
+            }
+#if 0
+            const half padVal = 0;
+            uint64_t mask0 = ((uint64_t)1ul << 16) - ((uint64_t)1ul << tail);
+            uint64_t mask[2] = {mask0, 0};
+            Duplicate<half>(input_local[len], padVal, mask, 1 /*no repeat*/, 1/*no gap in block*/, 8/*no gap between ∂repeats*/);
+#endif
+
+#if 0
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(half);
             DataCopyPadExtParams<half> padParams;
             DataCopyPad(input_local[len], input_gm[offset + len],
                         dataCopyParams, padParams);
+
+            uint16_t rightPadNum = 32 / sizeof(half) - tail;
+            PadParams padParas{0, rightPadNum, 0};            
+            Pad(input_local[len], input_gm[offset + len], padParas, tilingData.padTilingData);
+#endif
         }
         input_queue.EnQue(input_local);
     }
 
     __aicore__ inline void copy_out(uint32_t offset, size_t len) {
         LocalTensor<float> output_local = output_queue.DeQue<float>();
-        size_t tail = len % 32;
-        len = len & ~31;
-        DataCopy(output_gm[offset], output_local, len);
+        const size_t elem_per_block = 32 / sizeof(float);
+        size_t tail = len % elem_per_block;
+        len = len & ~elem_per_block;
+        // DataCopy(output_gm[offset], output_local, len);
         if(tail != 0) {
+            len += elem_per_block;
+#if 0
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPad(output_gm[offset + len], output_local[len],
                         dataCopyParams);
+#endif
         }
+        DataCopy(output_gm[offset], output_local, len);
+        // clean
+        if (tail != 0) {
+            for (int i = tail; i < elem_per_block; i++) {
+                output_gm[offset + len - elem_per_block].SetValue(i, 0);
+            }
+            DataCacheCleanAndInvalid<float, CacheLine::SINGLE_CACHE_LINE>(output_gm[offset + len - elem_per_block]);
+        }
+
         output_queue.FreeTensor(output_local);
     }
 

ggml/src/ggml-cann/kernels/get_row_f32.cpp

@@ -55,32 +55,59 @@ class GET_ROW_F32 {
 
     __aicore__ inline void copy_in(uint32_t offset, size_t len) {
         LocalTensor<float> input_local = input_queue.AllocTensor<float>();
-        size_t tail = len % 32;
-        len = len & ~31;
+        const size_t elem_per_block = 32 / sizeof(float);
+        size_t tail = len % elem_per_block;
+        len = len & ~elem_per_block;
         DataCopy(input_local, input_gm[offset], len);
         if(tail != 0) {
+            DataCopy(input_local[len], input_gm[offset + len], elem_per_block);
+            // clean
+            for (int i = tail; i < elem_per_block; i++) {
+                input_local[len].SetValue(i, 0);
+            }
+#if 0
+            const float padVal = 0;
+            uint64_t mask0 = ((uint64_t)1ul << 8) - ((uint64_t)1ul << tail);
+            uint64_t mask[2] = {mask0, 0};
+            Duplicate<float>(input_local[len], padVal, mask, 1 /*no repeat*/, 1/*no gap in block*/, 8/*no gap between repeats*/);
+#endif
+#if 0
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPadExtParams<float> padParams;
             DataCopyPad(input_local[len], input_gm[offset + len],
                         dataCopyParams, padParams);
+#endif
         }
         input_queue.EnQue(input_local);
     }
 
     __aicore__ inline void copy_out(uint32_t offset, size_t len) {
         LocalTensor<float> output_local = output_queue.DeQue<float>();
-        size_t tail = len % 32;
-        len = len & ~31;
-        DataCopy(output_gm[offset], output_local, len);
+        const size_t elem_per_block = 32 / sizeof(float);
+        size_t tail = len % elem_per_block;
+        len = len & ~elem_per_block;
+        //DataCopy(output_gm[offset], output_local, len);
         if(tail != 0) {
+            len += elem_per_block;
+#if 0
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPad(output_gm[offset + len], output_local[len],
                         dataCopyParams);
+#endif
+        }
+        DataCopy(output_gm[offset], output_local, len);
+
+        if (tail != 0) {         // clean
+            for (int i = tail; i < elem_per_block; i++) {
+                output_gm[offset + len - elem_per_block].SetValue(i, 0);
+            }
+            DataCacheCleanAndInvalid<float, CacheLine::SINGLE_CACHE_LINE>(output_gm[offset + len - elem_per_block]);
         }
+    
         output_queue.FreeTensor(output_local);
     }