Skip to content

Commit 1c6a50e

Browse files
momchil-velikovmomchil-velikov
committed
[AArch64] Add FP8 Neon intrinsics for dot-product
THis patch adds the following intrinsics: float16x4_t vdot_f16_mf8_fpm(float16x4_t vd, mfloat8x8_t vn, mfloat8x8_t vm, fpm_t fpm) float16x8_t vdotq_f16_mf8_fpm(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, fpm_t fpm) float16x4_t vdot_lane_f16_mf8_fpm(float16x4_t vd, mfloat8x8_t vn, mfloat8x8_t vm, __builtin_constant_p(lane), fpm_t fpm) float16x4_t vdot_laneq_f16_mf8_fpm(float16x4_t vd, mfloat8x8_t vn, mfloat8x16_t vm, __builtin_constant_p(lane), fpm_t fpm) float16x8_t vdotq_lane_f16_mf8_fpm(float16x8_t vd, mfloat8x16_t vn, mfloat8x8_t vm, __builtin_constant_p(lane), fpm_t fpm) float16x8_t vdotq_laneq_f16_mf8_fpm(float16x8_t vd, mfloat8x16_t vn, mfloat8x16_t vm, __builtin_constant_p(lane), fpm_t fpm) [fixup] Remove not needed argument (NFC) [fixup] Update intrinsics declarations [fixup] Add C++ runs to tests, remove some opt passes
1 parent 2e88b01 commit 1c6a50e

File tree

10 files changed

+529
-40
lines changed

10 files changed

+529
-40
lines changed

‎clang/include/clang/Basic/arm_neon.td

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2147,6 +2147,26 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
21472147
def VCVTN_F8_F16 : VInst<"vcvt_mf8_f16_fpm", ".(>F)(>F)V", "mQm">;
21482148
}
21492149

2150+
let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8dot2,neon" in {
2151+
def VDOT_F16_MF8 : VInst<"vdot_f16_mf8_fpm", "(>F)(>F)..V", "mQm">;
2152+
2153+
def VDOT_LANE_F16_MF8 : VInst<"vdot_lane_f16_mf8_fpm", "(>F)(>F)..IV", "m", [ImmCheck<3, ImmCheck0_3, 0>]>;
2154+
def VDOT_LANEQ_F16_MF8 : VInst<"vdot_laneq_f16_mf8_fpm", "(>F)(>F).QIV", "m", [ImmCheck<3, ImmCheck0_7, 0>]>;
2155+
2156+
def VDOTQ_LANE_F16_MF8 : VInst<"vdot_lane_f16_mf8_fpm", "(>F)(>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_3, 0>]>;
2157+
def VDOTQ_LANEQ_F16_MF8 : VInst<"vdot_laneq_f16_mf8_fpm", "(>F)(>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_7, 0>]>;
2158+
}
2159+
2160+
let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8dot4,neon" in {
2161+
def VDOT_F32_MF8 : VInst<"vdot_f32_mf8_fpm", "(>>F)(>>F)..V", "mQm">;
2162+
2163+
def VDOT_LANE_F32_MF8 : VInst<"vdot_lane_f32_mf8_fpm", "(>>F)(>>F)..IV", "m", [ImmCheck<3, ImmCheck0_1, 0>]>;
2164+
def VDOT_LANEQ_F32_MF8 : VInst<"vdot_laneq_f32_mf8_fpm", "(>>F)(>>F).QIV", "m", [ImmCheck<3, ImmCheck0_3, 0>]>;
2165+
2166+
def VDOTQ_LANE_F32_MF8 : VInst<"vdot_lane_f32_mf8_fpm", "(>>F)(>>F).qIV", "Qm", [ImmCheck<3, ImmCheck0_1, 0>]>;
2167+
def VDOTQ_LANEQ_F32_MF8 : VInst<"vdot_laneq_f32_mf8_fpm", "(>>F)(>>F)..IV", "Qm", [ImmCheck<3, ImmCheck0_3, 0>]>;
2168+
}
2169+
21502170
let ArchGuard = "defined(__aarch64__)", TargetGuard = "neon,faminmax" in {
21512171
def FAMIN : WInst<"vamin", "...", "fhQdQfQh">;
21522172
def FAMAX : WInst<"vamax", "...", "fhQdQfQh">;

‎clang/include/clang/Basic/arm_neon_incl.td

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ class Inst <string n, string p, string t, Operation o, list<ImmCheck> ch = []>{
302302
class SInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
303303
class IInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
304304
class WInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
305-
class VInst<string n, string p, string t> : Inst<n, p, t, OP_NONE> {}
305+
class VInst<string n, string p, string t, list<ImmCheck> ch = []> : Inst<n, p, t, OP_NONE, ch> {}
306306

307307
// The following instruction classes are implemented via operators
308308
// instead of builtins. As such these declarations are only used for

‎clang/lib/CodeGen/CGBuiltin.cpp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6906,6 +6906,24 @@ Value *CodeGenFunction::EmitFP8NeonCall(Function *F,
69066906
return EmitNeonCall(F, Ops, name);
69076907
}
69086908

6909+
llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
6910+
unsigned IID, bool ExtendLane, llvm::Type *RetTy,
6911+
SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
6912+
6913+
const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
6914+
RetTy->getPrimitiveSizeInBits();
6915+
llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
6916+
Ops[1]->getType()};
6917+
if (ExtendLane) {
6918+
auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
6919+
Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
6920+
Builder.getInt64(0));
6921+
}
6922+
llvm::Value *FPM =
6923+
EmitScalarOrConstFoldImmArg(/* ICEArguments */ 0, E->getNumArgs() - 1, E);
6924+
return EmitFP8NeonCall(CGM.getIntrinsic(IID, Tys), Ops, FPM, name);
6925+
}
6926+
69096927
Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
69106928
bool neg) {
69116929
int SV = cast<ConstantInt>(V)->getSExtValue();
@@ -12866,6 +12884,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
1286612884

1286712885
unsigned Int;
1286812886
bool ExtractLow = false;
12887+
bool ExtendLane = false;
1286912888
switch (BuiltinID) {
1287012889
default: return nullptr;
1287112890
case NEON::BI__builtin_neon_vbsl_v:
@@ -14133,6 +14152,31 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
1413314152
return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn2, Ty,
1413414153
Ops[1]->getType(), false, Ops, E, "vfcvtn2");
1413514154
}
14155+
14156+
case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
14157+
case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
14158+
return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2, false, HalfTy,
14159+
Ops, E, "fdot2");
14160+
case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
14161+
case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
14162+
ExtendLane = true;
14163+
LLVM_FALLTHROUGH;
14164+
case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
14165+
case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
14166+
return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2_lane,
14167+
ExtendLane, HalfTy, Ops, E, "fdot2_lane");
14168+
case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
14169+
case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
14170+
return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4, false,
14171+
FloatTy, Ops, E, "fdot4");
14172+
case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
14173+
case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
14174+
ExtendLane = true;
14175+
LLVM_FALLTHROUGH;
14176+
case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
14177+
case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
14178+
return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4_lane,
14179+
ExtendLane, FloatTy, Ops, E, "fdot4_lane");
1413614180
case NEON::BI__builtin_neon_vamin_f16:
1413714181
case NEON::BI__builtin_neon_vaminq_f16:
1413814182
case NEON::BI__builtin_neon_vamin_f32:

‎clang/lib/CodeGen/CodeGenFunction.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4698,6 +4698,10 @@ class CodeGenFunction : public CodeGenTypeCache {
46984698
llvm::Type *Ty1, bool Extract,
46994699
SmallVectorImpl<llvm::Value *> &Ops,
47004700
const CallExpr *E, const char *name);
4701+
llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLane,
4702+
llvm::Type *RetTy,
4703+
SmallVectorImpl<llvm::Value *> &Ops,
4704+
const CallExpr *E, const char *name);
47014705
llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
47024706
const llvm::ElementCount &Count);
47034707
llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);

0 commit comments

Comments
 (0)