f16_get_row debug OK

Author: leo-pony

ggml/src/ggml-cann/aclnn_ops.cpp

@@ -2329,6 +2329,13 @@ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
             }
             break;
         case GGML_TYPE_F16:
+        {
+            if ((src0->ne[0] % 16) != 0) {
+                size_t dst_len = src1->ne[0] * src1->ne[1] * src1->ne[2] * src0->ne[0] * ggml_type_size(GGML_TYPE_F32); // out is also f32, even input is f16
+/*                      printf("\n\nggml_cann_get_rows: row elements:%d, src1->ne[0]:%d, src1->ne[1]:%d, src1->ne[2]:%d, src0->ne[0]:%d, ggml_type_size(GGML_TYPE_F32):%d, dst_len:%d.\n", src0->ne[0], 
+                src1->ne[0], src1->ne[1], src1->ne[2], src0->ne[0], ggml_type_size(GGML_TYPE_F32), dst_len); */
+                ACL_CHECK(aclrtMemset((char*)dst->data, dst_len, 0, dst_len));
+            }
             aclrtlaunch_ascendc_get_row_f16(
                 24, ctx.stream(), src0->data, src1->data, dst->data,
                 ((ggml_tensor*)src0->extra)->ne,
@@ -2337,6 +2344,7 @@ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
                 ((ggml_tensor*)src1->extra)->nb, ((ggml_tensor*)dst->extra)->ne,
                 ((ggml_tensor*)dst->extra)->nb);
             break;
+        }
         case GGML_TYPE_Q4_0:
             aclrtlaunch_ascendc_get_row_q4_0(
                 24, ctx.stream(), src0->data, src1->data, dst->data,

ggml/src/ggml-cann/kernels/get_row_f16.cpp

@@ -14,7 +14,7 @@ class GET_ROW_F16 {
                                 int64_t *output_ne_ub, size_t *output_nb_ub) {
         // TODO, use template for F16/f32
         int64_t op_block_num = GetBlockNum();
-        int64_t op_block_idx = GetBlockIdx();
+        op_block_idx = GetBlockIdx();
 
         for (int i = 0; i < 4; i++) {
             input_ne[i] = input_ne_ub[i];
@@ -59,65 +59,61 @@ class GET_ROW_F16 {
     }
 
     __aicore__ inline void copy_in(uint32_t offset, size_t len) {
+        size_t origin_len = len;
         LocalTensor<half> input_local = input_queue.AllocTensor<half>();
         const size_t elem_per_block = 32 / sizeof(half);
         size_t tail = len % elem_per_block;
-        len = len & ~elem_per_block;
-        DataCopy(input_local, input_gm[offset], len);
+        len = len & ~(elem_per_block - 1);
         if(tail != 0) {
-            DataCopy(input_local[len], input_gm[offset + len], elem_per_block);
-            // clean
-            for (int i = tail; i < elem_per_block; i++) {
-                input_local[len].SetValue(i, 0);
-            }
-#if 0
-            const half padVal = 0;
-            uint64_t mask0 = ((uint64_t)1ul << 16) - ((uint64_t)1ul << tail);
-            uint64_t mask[2] = {mask0, 0};
-            Duplicate<half>(input_local[len], padVal, mask, 1 /*no repeat*/, 1/*no gap in block*/, 8/*no gap between ∂repeats*/);
-#endif
-
+            //printf("f16 get_row: copy_in: offset:%d, len:%d, origin_len:%d, tail:%d, elem_per_block:%d.\n", offset, len, origin_len, tail, elem_per_block);
+            //DumpTensor(input_local, 5, elem_per_block);
+            len += elem_per_block;
+            // DataCopy(input_local[len], input_gm[offset + len], elem_per_block);
 #if 0
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(half);
             DataCopyPadExtParams<half> padParams;
             DataCopyPad(input_local[len], input_gm[offset + len],
                         dataCopyParams, padParams);
-
-            uint16_t rightPadNum = 32 / sizeof(half) - tail;
-            PadParams padParas{0, rightPadNum, 0};            
-            Pad(input_local[len], input_gm[offset + len], padParas, tilingData.padTilingData);
 #endif
         }
+        DataCopy(input_local, input_gm[offset], len);
         input_queue.EnQue(input_local);
     }
 
     __aicore__ inline void copy_out(uint32_t offset, size_t len) {
         LocalTensor<float> output_local = output_queue.DeQue<float>();
         const size_t elem_per_block = 32 / sizeof(float);
         size_t tail = len % elem_per_block;
-        len = len & ~elem_per_block;
-        // DataCopy(output_gm[offset], output_local, len);
+        len = len & ~(elem_per_block - 1);
+        if (len > 0) {
+            DataCopy(output_gm[offset], output_local, len);
+        }
+#if 1
         if(tail != 0) {
-            len += elem_per_block;
+/*             printf("\nf16 BLOCK_IDX:%d get_row: Copy_Out AtomicAdd: offset:%d, len:%d, tail:%d, elem_per_block:%d.\n", op_block_idx, offset, len, tail, elem_per_block);
+            DumpTensor(output_gm, 5, elem_per_block); */
+            for (size_t i = tail; i < elem_per_block; i++) {
+                output_local[len + i].SetValue(0, 0);
+            }
+            // DumpTensor(output_local[len], 5, elem_per_block);
+            SetAtomicAdd<float>();
+            DataCopy(output_gm[offset + len], output_local[len], elem_per_block);
+            SetAtomicNone();
+            // DumpTensor(output_gm, 5, elem_per_block);
+        }
+#endif
+
 #if 0
+        if(tail != 0) {
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPad(output_gm[offset + len], output_local[len],
                         dataCopyParams);
-#endif
-        }
-        DataCopy(output_gm[offset], output_local, len);
-        // clean
-        if (tail != 0) {
-            for (int i = tail; i < elem_per_block; i++) {
-                output_gm[offset + len - elem_per_block].SetValue(i, 0);
-            }
-            DataCacheCleanAndInvalid<float, CacheLine::SINGLE_CACHE_LINE>(output_gm[offset + len - elem_per_block]);
         }
-
+#endif
         output_queue.FreeTensor(output_local);
     }
 
@@ -182,6 +178,7 @@ class GET_ROW_F16 {
     GlobalTensor<float> output_gm;
     TQue<QuePosition::VECIN, BUFFER_NUM> input_queue;
     TQue<QuePosition::VECOUT, BUFFER_NUM> output_queue;
+    int64_t op_block_idx;
 };
 
 template <typename T>

ggml/src/ggml-cann/kernels/get_row_f32.cpp

@@ -107,6 +107,9 @@ class GET_ROW_F32 {
 
 #if 1
         if(tail != 0) {
+            for (size_t i = tail; i < elem_per_block; i++) {
+                output_local[len + i].SetValue(0, 0);
+            }
             //printf("\nf32 BLOCK_IDX:%d get_row: Copy_Out AtomicAdd: offset:%d, len:%d, tail:%d, elem_per_block:%d.\n", op_block_idx, offset, len, tail, elem_per_block);
 /*             DumpTensor(output_gm[offset + len], 5, elem_per_block);
             DumpTensor(output_local[len], 5, elem_per_block); */