|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#include <executorch/backends/cadence/hifi/kernels/kernels.h> |
| 10 | +#include <executorch/kernels/portable/cpu/scalar_utils.h> |
| 11 | +#include <executorch/kernels/portable/cpu/util/broadcast_util.h> |
| 12 | +#include <executorch/kernels/portable/cpu/util/functional_util.h> |
| 13 | +#include <executorch/kernels/portable/cpu/util/kernel_ops_util.h> |
| 14 | +#include <executorch/runtime/kernel/kernel_includes.h> |
| 15 | +#include <executorch/runtime/platform/assert.h> |
| 16 | + |
| 17 | +using exec_aten::Scalar; |
| 18 | +using exec_aten::ScalarType; |
| 19 | +using exec_aten::Tensor; |
| 20 | +using executorch::runtime::can_cast; |
| 21 | +using executorch::runtime::CppTypeToScalarType; |
| 22 | +using executorch::runtime::KernelRuntimeContext; |
| 23 | +using torch::executor::Error; |
| 24 | + |
| 25 | +namespace cadence { |
| 26 | +namespace impl { |
| 27 | +namespace HiFi { |
| 28 | +namespace native { |
| 29 | + |
| 30 | +namespace { |
| 31 | +template < |
| 32 | + bool can_cast, |
| 33 | + typename CTYPE_A, |
| 34 | + typename CTYPE_B, |
| 35 | + typename CTYPE_IN, |
| 36 | + typename CTYPE_OUT> |
| 37 | +struct AddInner; |
| 38 | + |
| 39 | +template < |
| 40 | + typename CTYPE_A, |
| 41 | + typename CTYPE_B, |
| 42 | + typename CTYPE_IN, |
| 43 | + typename CTYPE_OUT> |
| 44 | +struct AddInner<true, CTYPE_A, CTYPE_B, CTYPE_IN, CTYPE_OUT> { |
| 45 | + static void |
| 46 | + run(const Tensor& a, const Tensor& b, CTYPE_IN alpha_val, Tensor& out) { |
| 47 | + torch::executor::apply_binary_elementwise_fn<CTYPE_A, CTYPE_B, CTYPE_OUT>( |
| 48 | + // NOLINTNEXTLINE(facebook-hte-ConstantArgumentPassByValue) |
| 49 | + [alpha_val](const CTYPE_A val_a, const CTYPE_B val_b) { |
| 50 | + CTYPE_IN a_casted = static_cast<CTYPE_IN>(val_a); |
| 51 | + CTYPE_IN b_casted = static_cast<CTYPE_IN>(val_b); |
| 52 | + CTYPE_IN value = a_casted + alpha_val * b_casted; |
| 53 | + |
| 54 | + return static_cast<CTYPE_OUT>(value); |
| 55 | + }, |
| 56 | + a, |
| 57 | + b, |
| 58 | + out); |
| 59 | + } |
| 60 | +}; |
| 61 | + |
| 62 | +template <typename CTYPE_IN> |
| 63 | +struct ReportCanCastBug { |
| 64 | + static void run(const Tensor&, const Tensor&, CTYPE_IN, Tensor&) { |
| 65 | + ET_DCHECK_MSG(false, "BUG: canCast should have been checked above"); |
| 66 | + } |
| 67 | +}; |
| 68 | + |
| 69 | +template < |
| 70 | + typename CTYPE_A, |
| 71 | + typename CTYPE_B, |
| 72 | + typename CTYPE_IN, |
| 73 | + typename CTYPE_OUT> |
| 74 | +struct AddInner<false, CTYPE_A, CTYPE_B, CTYPE_IN, CTYPE_OUT> |
| 75 | + : public ReportCanCastBug<CTYPE_IN> {}; |
| 76 | + |
| 77 | +} // namespace |
| 78 | + |
| 79 | +Tensor& add_out( |
| 80 | + KernelRuntimeContext& ctx, |
| 81 | + const Tensor& a, |
| 82 | + const Tensor& b, |
| 83 | + const Scalar& alpha, |
| 84 | + Tensor& out) { |
| 85 | + ET_KERNEL_CHECK( |
| 86 | + ctx, |
| 87 | + torch::executor::resize_to_broadcast_target_size(a, b, out) == Error::Ok, |
| 88 | + InvalidArgument, |
| 89 | + out); |
| 90 | + |
| 91 | + ET_KERNEL_CHECK( |
| 92 | + ctx, |
| 93 | + executorch::runtime::tensor_is_realhbbf16_type(out), |
| 94 | + InvalidArgument, |
| 95 | + out); |
| 96 | + ET_KERNEL_CHECK( |
| 97 | + ctx, |
| 98 | + executorch::runtime::tensors_have_same_dim_order(a, b, out), |
| 99 | + InvalidArgument, |
| 100 | + out); |
| 101 | + |
| 102 | + ScalarType a_type = a.scalar_type(); |
| 103 | + ScalarType b_type = b.scalar_type(); |
| 104 | + ScalarType alpha_type = |
| 105 | + torch::executor::native::utils::get_scalar_dtype(alpha); |
| 106 | + ScalarType common_type = |
| 107 | + executorch::runtime::promoteTypes(a_type, b_type, /*half_to_float*/ true); |
| 108 | + ScalarType out_type = out.scalar_type(); |
| 109 | + |
| 110 | + ET_KERNEL_CHECK( |
| 111 | + ctx, |
| 112 | + executorch::runtime::canCast(common_type, out_type), |
| 113 | + InvalidArgument, |
| 114 | + out); |
| 115 | + ET_KERNEL_CHECK( |
| 116 | + ctx, |
| 117 | + torch::executor::check_alpha_type(alpha_type, common_type), |
| 118 | + InvalidArgument, |
| 119 | + out); |
| 120 | + |
| 121 | + float alpha_val; |
| 122 | + torch::executor::native::utils::extract_scalar(alpha, &alpha_val); |
| 123 | + |
| 124 | + constexpr auto name = "add.out"; |
| 125 | + constexpr int kNnlibMaxDim = 4; /*fallback if broadcast and dim > 4 */ |
| 126 | + |
| 127 | + int a_dim = a.dim(), b_dim = b.dim(), out_dim = out.dim(); |
| 128 | + bool optimized = 1; |
| 129 | + /*find broadcast*/ |
| 130 | + const bool a_is_broadcasted = !out.sizes().equals(a.sizes()); |
| 131 | + const bool b_is_broadcasted = !out.sizes().equals(b.sizes()); |
| 132 | + const bool broadcast = (a_is_broadcasted || b_is_broadcasted); |
| 133 | + int max_dim = a.dim() > b.dim() ? a.dim() : b.dim(); |
| 134 | + max_dim = out.dim() > max_dim ? out.dim() : max_dim; |
| 135 | + |
| 136 | + if ((out_type != ScalarType::Float) || (alpha_val != 1.0)) |
| 137 | + optimized = 0; |
| 138 | + |
| 139 | + if ((a_dim == 0) || (b_dim == 0)) |
| 140 | + optimized = 0; |
| 141 | + |
| 142 | + if ((broadcast == 1) && (max_dim > kNnlibMaxDim)) |
| 143 | + optimized = 0; |
| 144 | + |
| 145 | + if (optimized) { |
| 146 | + const float* const a_data = a.const_data_ptr<float>(); |
| 147 | + const float* const b_data = b.const_data_ptr<float>(); |
| 148 | + float* const out_data = out.mutable_data_ptr<float>(); |
| 149 | + |
| 150 | + if (broadcast == 1) { |
| 151 | + int out_shape[kNnlibMaxDim]; |
| 152 | + int inp1_shape[kNnlibMaxDim]; |
| 153 | + int inp2_shape[kNnlibMaxDim]; |
| 154 | + |
| 155 | + for (int i = 0; i < kNnlibMaxDim; i++) { |
| 156 | + out_shape[i] = 1; |
| 157 | + inp1_shape[i] = 1; |
| 158 | + inp2_shape[i] = 1; |
| 159 | + } |
| 160 | + |
| 161 | + int off_o = kNnlibMaxDim - out.dim(); |
| 162 | + int off_a = kNnlibMaxDim - a.dim(); |
| 163 | + int off_b = kNnlibMaxDim - b.dim(); |
| 164 | + |
| 165 | + for (int i = 0; i < out.dim(); i++) |
| 166 | + out_shape[i + off_o] = out.size(i); |
| 167 | + for (int i = 0; i < a.dim(); i++) |
| 168 | + inp1_shape[i + off_a] = a.size(i); |
| 169 | + for (int i = 0; i < b.dim(); i++) |
| 170 | + inp2_shape[i + off_b] = b.size(i); |
| 171 | + |
| 172 | + xa_nn_elm_add_broadcast_4D_f32xf32_f32( |
| 173 | + out_data, out_shape, a_data, inp1_shape, b_data, inp2_shape); |
| 174 | + } else { |
| 175 | + xa_nn_elm_add_f32xf32_f32(out_data, a_data, b_data, out.numel()); |
| 176 | + } |
| 177 | + |
| 178 | + return out; |
| 179 | + } |
| 180 | + |
| 181 | + ET_SWITCH_REALHBBF16_TYPES(a_type, ctx, name, CTYPE_A, [&]() { |
| 182 | + ET_SWITCH_REALHBBF16_TYPES(b_type, ctx, name, CTYPE_B, [&]() { |
| 183 | + using CTYPE_IN = typename torch::executor:: |
| 184 | + promote_types<CTYPE_A, CTYPE_B, /*half_to_float*/ true>::type; |
| 185 | + ET_DCHECK(CppTypeToScalarType<CTYPE_IN>::value == common_type); |
| 186 | + CTYPE_IN alpha_val; |
| 187 | + torch::executor::native::utils::extract_scalar(alpha, &alpha_val); |
| 188 | + |
| 189 | + ET_SWITCH_REALHBBF16_TYPES(out_type, ctx, name, CTYPE_OUT, [&]() { |
| 190 | + AddInner< |
| 191 | + can_cast<CTYPE_IN, CTYPE_OUT>::value, |
| 192 | + CTYPE_A, |
| 193 | + CTYPE_B, |
| 194 | + CTYPE_IN, |
| 195 | + CTYPE_OUT>::run(a, b, alpha_val, out); |
| 196 | + }); |
| 197 | + }); |
| 198 | + }); |
| 199 | + |
| 200 | + return out; |
| 201 | +} |
| 202 | + |
| 203 | +} // namespace native |
| 204 | +} // namespace HiFi |
| 205 | +} // namespace impl |
| 206 | +} // namespace cadence |
0 commit comments