Skip to content

Commit 7b766a6

Browse files
XinWang10XinWang10
authored
[X86] Support APX CMOV/CFCMOV instructions (#82592)
This patch support ND CMOV instructions and CFCMOV instructions. RFC: https://discourse.llvm.org/t/rfc-design-for-apx-feature-egpr-and-ndd-support/73031/4
1 parent bfb8682 commit 7b766a6

35 files changed

+4356
-176
lines changed

‎llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4068,7 +4068,7 @@ unsigned X86AsmParser::checkTargetMatchPredicate(MCInst &Inst) {
40684068

40694069
if (UseApxExtendedReg && !X86II::canUseApxExtendedReg(MCID))
40704070
return Match_Unsupported;
4071-
if (ForcedNoFlag != !!(MCID.TSFlags & X86II::EVEX_NF))
4071+
if (ForcedNoFlag == !(MCID.TSFlags & X86II::EVEX_NF) && !X86::isCFCMOVCC(Opc))
40724072
return Match_Unsupported;
40734073

40744074
if (ForcedVEXEncoding == VEXEncoding_EVEX &&

‎llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,14 @@ enum : uint64_t {
545545
/// PrefixByte - This form is used for instructions that represent a prefix
546546
/// byte like data16 or rep.
547547
PrefixByte = 10,
548+
/// MRMDestRegCC - This form is used for the cfcmov instructions, which use
549+
/// the Mod/RM byte to specify the operands reg(r/m) and reg(reg) and also
550+
/// encodes a condition code.
551+
MRMDestRegCC = 18,
552+
/// MRMDestMemCC - This form is used for the cfcmov instructions, which use
553+
/// the Mod/RM byte to specify the operands mem(r/m) and reg(reg) and also
554+
/// encodes a condition code.
555+
MRMDestMemCC = 19,
548556
/// MRMDestMem4VOp3CC - This form is used for instructions that use the Mod/RM
549557
/// byte to specify a destination which in this case is memory and operand 3
550558
/// with VEX.VVVV, and also encodes a condition code.
@@ -1032,6 +1040,7 @@ inline int getMemoryOperandNo(uint64_t TSFlags) {
10321040
return -1;
10331041
case X86II::MRMDestMem:
10341042
case X86II::MRMDestMemFSIB:
1043+
case X86II::MRMDestMemCC:
10351044
return hasNewDataDest(TSFlags);
10361045
case X86II::MRMSrcMem:
10371046
case X86II::MRMSrcMemFSIB:
@@ -1045,11 +1054,13 @@ inline int getMemoryOperandNo(uint64_t TSFlags) {
10451054
// Skip registers encoded in reg, VEX_VVVV, and I8IMM.
10461055
return 3;
10471056
case X86II::MRMSrcMemCC:
1057+
return 1 + hasNewDataDest(TSFlags);
10481058
case X86II::MRMDestMem4VOp3CC:
10491059
// Start from 1, skip any registers encoded in VEX_VVVV or I8IMM, or a
10501060
// mask register.
10511061
return 1;
10521062
case X86II::MRMDestReg:
1063+
case X86II::MRMDestRegCC:
10531064
case X86II::MRMSrcReg:
10541065
case X86II::MRMSrcReg4VOp3:
10551066
case X86II::MRMSrcRegOp4:

‎llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ void X86InstPrinterCommon::printInstFlags(const MCInst *MI, raw_ostream &O,
394394
else if (Flags & X86::IP_HAS_REPEAT)
395395
O << "\trep\t";
396396

397-
if (TSFlags & X86II::EVEX_NF)
397+
if (TSFlags & X86II::EVEX_NF && !X86::isCFCMOVCC(MI->getOpcode()))
398398
O << "\t{nf}";
399399

400400
// These all require a pseudo prefix

‎llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,6 +1071,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
10711071
case X86II::MRM_C0:
10721072
case X86II::RawFrm:
10731073
break;
1074+
case X86II::MRMDestMemCC:
10741075
case X86II::MRMDestMemFSIB:
10751076
case X86II::MRMDestMem: {
10761077
// MRMDestMem instructions forms:
@@ -1102,6 +1103,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
11021103
}
11031104
break;
11041105
}
1106+
case X86II::MRMSrcMemCC:
11051107
case X86II::MRMSrcMemFSIB:
11061108
case X86II::MRMSrcMem: {
11071109
// MRMSrcMem instructions forms:
@@ -1180,6 +1182,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
11801182
}
11811183
break;
11821184
}
1185+
case X86II::MRMSrcRegCC:
11831186
case X86II::MRMSrcReg: {
11841187
// MRMSrcReg instructions forms:
11851188
// dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
@@ -1242,6 +1245,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
12421245
++CurOp;
12431246
break;
12441247
}
1248+
case X86II::MRMDestRegCC:
12451249
case X86II::MRMDestReg: {
12461250
// MRMDestReg instructions forms:
12471251
// dst(ModR/M), src(ModR/M)
@@ -1638,6 +1642,15 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
16381642
CurOp = SrcRegNum + 1;
16391643
break;
16401644
}
1645+
case X86II::MRMDestRegCC: {
1646+
unsigned FirstOp = CurOp++;
1647+
unsigned SecondOp = CurOp++;
1648+
unsigned CC = MI.getOperand(CurOp++).getImm();
1649+
emitByte(BaseOpcode + CC, CB);
1650+
emitRegModRMByte(MI.getOperand(FirstOp),
1651+
getX86RegNum(MI.getOperand(SecondOp)), CB);
1652+
break;
1653+
}
16411654
case X86II::MRMDestMem4VOp3CC: {
16421655
unsigned CC = MI.getOperand(8).getImm();
16431656
emitByte(BaseOpcode + CC, CB);
@@ -1667,6 +1680,16 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
16671680
CurOp = SrcRegNum + 1;
16681681
break;
16691682
}
1683+
case X86II::MRMDestMemCC: {
1684+
unsigned MemOp = CurOp;
1685+
CurOp = MemOp + X86::AddrNumOperands;
1686+
unsigned RegOp = CurOp++;
1687+
unsigned CC = MI.getOperand(CurOp++).getImm();
1688+
emitByte(BaseOpcode + CC, CB);
1689+
emitMemModRMByte(MI, MemOp, getX86RegNum(MI.getOperand(RegOp)), TSFlags,
1690+
Kind, StartByte, CB, Fixups, STI);
1691+
break;
1692+
}
16701693
case X86II::MRMSrcReg: {
16711694
emitByte(BaseOpcode, CB);
16721695
unsigned SrcRegNum = CurOp + 1;
@@ -1717,6 +1740,8 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
17171740
break;
17181741
}
17191742
case X86II::MRMSrcRegCC: {
1743+
if (IsND) // Skip new data destination
1744+
++CurOp;
17201745
unsigned FirstOp = CurOp++;
17211746
unsigned SecondOp = CurOp++;
17221747

@@ -1778,6 +1803,8 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
17781803
break;
17791804
}
17801805
case X86II::MRMSrcMemCC: {
1806+
if (IsND) // Skip new data destination
1807+
++CurOp;
17811808
unsigned RegOp = CurOp++;
17821809
unsigned FirstMemOp = CurOp;
17831810
CurOp = FirstMemOp + X86::AddrNumOperands;

‎llvm/lib/Target/X86/X86FastISel.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2133,7 +2133,8 @@ bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
21332133
return false;
21342134

21352135
const TargetRegisterInfo &TRI = *Subtarget->getRegisterInfo();
2136-
unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(*RC)/8);
2136+
unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(*RC) / 8, false,
2137+
Subtarget->hasNDD());
21372138
Register ResultReg = fastEmitInst_rri(Opc, RC, RHSReg, LHSReg, CC);
21382139
updateValueMap(I, ResultReg);
21392140
return true;

‎llvm/lib/Target/X86/X86FlagsCopyLowering.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,8 @@ bool X86FlagsCopyLoweringPass::runOnMachineFunction(MachineFunction &MF) {
604604
}
605605

606606
// Otherwise we can just rewrite in-place.
607-
if (X86::getCondFromCMov(MI) != X86::COND_INVALID) {
607+
if (X86::getCondFromCMov(MI) != X86::COND_INVALID ||
608+
X86::getCondFromCFCMov(MI) != X86::COND_INVALID) {
608609
rewriteCMov(*TestMBB, TestPos, TestLoc, MI, *FlagUse, CondRegs);
609610
} else if (getCondFromFCMOV(MI.getOpcode()) != X86::COND_INVALID) {
610611
rewriteFCMov(*TestMBB, TestPos, TestLoc, MI, *FlagUse, CondRegs);
@@ -829,7 +830,9 @@ void X86FlagsCopyLoweringPass::rewriteCMov(MachineBasicBlock &TestMBB,
829830
MachineOperand &FlagUse,
830831
CondRegArray &CondRegs) {
831832
// First get the register containing this specific condition.
832-
X86::CondCode Cond = X86::getCondFromCMov(CMovI);
833+
X86::CondCode Cond = X86::getCondFromCMov(CMovI) == X86::COND_INVALID
834+
? X86::getCondFromCFCMov(CMovI)
835+
: X86::getCondFromCMov(CMovI);
833836
unsigned CondReg;
834837
bool Inverted;
835838
std::tie(CondReg, Inverted) =

‎llvm/lib/Target/X86/X86InstrAsmAlias.td

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,12 @@ defm : IntegerCondCodeMnemonicAlias<"cmov", "q", "att">;
402402
// No size suffix for intel-style asm.
403403
defm : IntegerCondCodeMnemonicAlias<"cmov", "", "intel">;
404404

405+
// Aliases for cfcmov<CC>{w,l,q}
406+
defm : IntegerCondCodeMnemonicAlias<"cfcmov", "w", "att">;
407+
defm : IntegerCondCodeMnemonicAlias<"cfcmov", "l", "att">;
408+
defm : IntegerCondCodeMnemonicAlias<"cfcmov", "q", "att">;
409+
// No size suffix for intel-style asm.
410+
defm : IntegerCondCodeMnemonicAlias<"cfcmov", "", "intel">;
405411
//===----------------------------------------------------------------------===//
406412
// Assembler Instruction Aliases
407413
//===----------------------------------------------------------------------===//
@@ -768,6 +774,20 @@ multiclass CMOV_SETCC_Aliases<string Cond, int CC> {
768774
(CMOV64rr GR64:$dst, GR64:$src, CC), 0>;
769775
def : InstAlias<"cmov"#Cond#"{q}\t{$src, $dst|$dst, $src}",
770776
(CMOV64rm GR64:$dst, i64mem:$src, CC), 0>;
777+
let Predicates = [In64BitMode] in {
778+
def : InstAlias<"cmov"#Cond#"{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
779+
(CMOV16rr_ND GR16:$dst, GR16:$src1, GR16:$src2, CC), 0>;
780+
def : InstAlias<"cmov"#Cond#"{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
781+
(CMOV16rm_ND GR16:$dst, GR16:$src1, i16mem:$src2, CC), 0>;
782+
def : InstAlias<"cmov"#Cond#"{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
783+
(CMOV32rr_ND GR32:$dst, GR32:$src1, GR32:$src2, CC), 0>;
784+
def : InstAlias<"cmov"#Cond#"{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
785+
(CMOV32rm_ND GR32:$dst, GR32:$src1, i32mem:$src2, CC), 0>;
786+
def : InstAlias<"cmov"#Cond#"{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
787+
(CMOV64rr_ND GR64:$dst, GR64:$src1, GR64:$src2, CC), 0>;
788+
def : InstAlias<"cmov"#Cond#"{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
789+
(CMOV64rm_ND GR64:$dst, GR64:$src1, i64mem:$src2, CC), 0>;
790+
}
771791

772792
def : InstAlias<"set"#Cond#"\t$dst", (SETCCr GR8:$dst, CC), 0>;
773793
def : InstAlias<"set"#Cond#"\t$dst", (SETCCm i8mem:$dst, CC), 0>;
@@ -790,6 +810,57 @@ defm : CMOV_SETCC_Aliases<"ge", 13>;
790810
defm : CMOV_SETCC_Aliases<"le", 14>;
791811
defm : CMOV_SETCC_Aliases<"g" , 15>;
792812

813+
multiclass CFCMOV_Aliases<string Cond, int CC> {
814+
let Predicates = [In64BitMode] in {
815+
def : InstAlias<"cfcmov"#Cond#"{w}\t{$src, $dst|$dst, $src}",
816+
(CFCMOV16rr GR16:$dst, GR16:$src, CC), 0>;
817+
def : InstAlias<"cfcmov"#Cond#"{l}\t{$src, $dst|$dst, $src}",
818+
(CFCMOV32rr GR32:$dst, GR32:$src, CC), 0>;
819+
def : InstAlias<"cfcmov"#Cond#"{q}\t{$src, $dst|$dst, $src}",
820+
(CFCMOV64rr GR64:$dst, GR64:$src, CC), 0>;
821+
def : InstAlias<"cfcmov"#Cond#"{w}\t{$src, $dst|$dst, $src}",
822+
(CFCMOV16rm GR16:$dst, i16mem:$src, CC), 0>;
823+
def : InstAlias<"cfcmov"#Cond#"{l}\t{$src, $dst|$dst, $src}",
824+
(CFCMOV32rm GR32:$dst, i32mem:$src, CC), 0>;
825+
def : InstAlias<"cfcmov"#Cond#"{q}\t{$src, $dst|$dst, $src}",
826+
(CFCMOV64rm GR64:$dst, i64mem:$src, CC), 0>;
827+
def : InstAlias<"cfcmov"#Cond#"{w}\t{$src, $dst|$dst, $src}",
828+
(CFCMOV16mr i16mem:$dst, GR16:$src, CC), 0>;
829+
def : InstAlias<"cfcmov"#Cond#"{l}\t{$src, $dst|$dst, $src}",
830+
(CFCMOV32mr i32mem:$dst, GR32:$src, CC), 0>;
831+
def : InstAlias<"cfcmov"#Cond#"{q}\t{$src, $dst|$dst, $src}",
832+
(CFCMOV64mr i64mem:$dst, GR64:$src, CC), 0>;
833+
def : InstAlias<"cfcmov"#Cond#"{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
834+
(CFCMOV16rr_ND GR16:$dst, GR16:$src1, GR16:$src2, CC), 0>;
835+
def : InstAlias<"cfcmov"#Cond#"{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
836+
(CFCMOV32rr_ND GR32:$dst, GR32:$src1, GR32:$src2, CC), 0>;
837+
def : InstAlias<"cfcmov"#Cond#"{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
838+
(CFCMOV64rr_ND GR64:$dst, GR64:$src1, GR64:$src2, CC), 0>;
839+
def : InstAlias<"cfcmov"#Cond#"{w}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
840+
(CFCMOV16rm_ND GR16:$dst, GR16:$src1, i16mem:$src2, CC), 0>;
841+
def : InstAlias<"cfcmov"#Cond#"{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
842+
(CFCMOV32rm_ND GR32:$dst, GR32:$src1, i32mem:$src2, CC), 0>;
843+
def : InstAlias<"cfcmov"#Cond#"{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",
844+
(CFCMOV64rm_ND GR64:$dst, GR64:$src1, i64mem:$src2, CC), 0>;
845+
}
846+
}
847+
defm : CFCMOV_Aliases<"o" , 0>;
848+
defm : CFCMOV_Aliases<"no", 1>;
849+
defm : CFCMOV_Aliases<"b" , 2>;
850+
defm : CFCMOV_Aliases<"ae", 3>;
851+
defm : CFCMOV_Aliases<"e" , 4>;
852+
defm : CFCMOV_Aliases<"ne", 5>;
853+
defm : CFCMOV_Aliases<"be", 6>;
854+
defm : CFCMOV_Aliases<"a" , 7>;
855+
defm : CFCMOV_Aliases<"s" , 8>;
856+
defm : CFCMOV_Aliases<"ns", 9>;
857+
defm : CFCMOV_Aliases<"p" , 10>;
858+
defm : CFCMOV_Aliases<"np", 11>;
859+
defm : CFCMOV_Aliases<"l" , 12>;
860+
defm : CFCMOV_Aliases<"ge", 13>;
861+
defm : CFCMOV_Aliases<"le", 14>;
862+
defm : CFCMOV_Aliases<"g" , 15>;
863+
793864
// Condition dump instructions Alias
794865
def : InstAlias<"jo\t$dst", (JCC_1 brtarget8:$dst, 0), 0>;
795866
def : InstAlias<"jno\t$dst", (JCC_1 brtarget8:$dst, 1), 0>;

0 commit comments

Comments
 (0)