[RISCV] Don't fold vmerge.vvm or vmv.v.v into vredsum.vs if AVL changed #99006
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
When folding, we currently check if the pseudo's result is not lanewise (e.g. vredsum.vs or viota.m) and bail if we're changing the mask. However we also need to check for the AVL too. This patch bails if the AVL changed for these pseudos, and also renames the pseudo table property to be more explicit.
|
@llvm/pr-subscribers-backend-risc-v Author: Luke Lau (lukel97) ChangesWhen folding, we currently check if the pseudo's result is not lanewise (e.g. vredsum.vs or viota.m) and bail if we're changing the mask. This patch bails if the AVL changed for these pseudos, and also renames the pseudo table property to be more explicit. Full diff: https://github.com/llvm/llvm-project/pull/99006.diff 5 Files Affected:
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index adde745a5a91b..eef6ae677ac85 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -3753,11 +3753,6 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
if (!Info)
return false;
- // When Mask is not a true mask, this transformation is illegal for some
- // operations whose results are affected by mask, like viota.m.
- if (Info->MaskAffectsResult && Mask && !usesAllOnesMask(Mask, Glue))
- return false;
-
// If True has a merge operand then it needs to be the same as vmerge's False,
// since False will be used for the result's merge operand.
if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
@@ -3835,6 +3830,16 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
if (!VL)
return false;
+ // Some operations produce different elementwise results depending on the
+ // active elements, like viota.m or vredsum. This transformation is illegal
+ // for these if we change the active elements (i.e. mask or VL).
+ if (Info->ActiveElementsAffectResult) {
+ if (Mask && !usesAllOnesMask(Mask, Glue))
+ return false;
+ if (TrueVL != VL)
+ return false;
+ }
+
// If we end up changing the VL or mask of True, then we need to make sure it
// doesn't raise any observable fp exceptions, since changing the active
// elements will affect how fflags is set.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index f0c0953a3e56a..025cc36d19eb7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -381,7 +381,7 @@ struct RISCVMaskedPseudoInfo {
uint16_t MaskedPseudo;
uint16_t UnmaskedPseudo;
uint8_t MaskOpIdx;
- uint8_t MaskAffectsResult : 1;
+ uint8_t ActiveElementsAffectResult : 1;
};
#define GET_RISCVMaskedPseudosTable_DECL
#include "RISCVGenSearchableTables.inc"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index d72390b7c14b5..b860273d639ee 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -561,17 +561,17 @@ def RISCVVIntrinsicsTable : GenericTable {
// unmasked variant. For all but compares, both the masked and
// unmasked variant have a passthru and policy operand. For compares,
// neither has a policy op, and only the masked version has a passthru.
-class RISCVMaskedPseudo<bits<4> MaskIdx, bit MaskAffectsRes=false> {
+class RISCVMaskedPseudo<bits<4> MaskIdx, bit ActiveAffectsRes=false> {
Pseudo MaskedPseudo = !cast<Pseudo>(NAME);
Pseudo UnmaskedPseudo = !cast<Pseudo>(!subst("_MASK", "", NAME));
bits<4> MaskOpIdx = MaskIdx;
- bit MaskAffectsResult = MaskAffectsRes;
+ bit ActiveElementsAffectResult = ActiveAffectsRes;
}
def RISCVMaskedPseudosTable : GenericTable {
let FilterClass = "RISCVMaskedPseudo";
let CppTypeName = "RISCVMaskedPseudoInfo";
- let Fields = ["MaskedPseudo", "UnmaskedPseudo", "MaskOpIdx", "MaskAffectsResult"];
+ let Fields = ["MaskedPseudo", "UnmaskedPseudo", "MaskOpIdx", "ActiveElementsAffectResult"];
let PrimaryKey = ["MaskedPseudo"];
let PrimaryKeyName = "getMaskedPseudoInfo";
}
@@ -2065,7 +2065,7 @@ multiclass VPseudoVIOTA_M {
SchedUnary<"WriteVIotaV", "ReadVIotaV", mx,
forceMergeOpRead=true>;
def "_" # mx # "_MASK" : VPseudoUnaryMask<m.vrclass, VR, constraint>,
- RISCVMaskedPseudo<MaskIdx=2, MaskAffectsRes=true>,
+ RISCVMaskedPseudo<MaskIdx=2, ActiveAffectsRes=true>,
SchedUnary<"WriteVIotaV", "ReadVIotaV", mx,
forceMergeOpRead=true>;
}
@@ -3162,7 +3162,7 @@ multiclass VPseudoTernaryWithTailPolicy<VReg RetClass,
defvar mx = MInfo.MX;
def "_" # mx # "_E" # sew : VPseudoTernaryNoMaskWithPolicy<RetClass, Op1Class, Op2Class>;
def "_" # mx # "_E" # sew # "_MASK" : VPseudoTernaryMaskPolicy<RetClass, Op1Class, Op2Class>,
- RISCVMaskedPseudo<MaskIdx=3, MaskAffectsRes=true>;
+ RISCVMaskedPseudo<MaskIdx=3, ActiveAffectsRes=true>;
}
}
@@ -3179,7 +3179,7 @@ multiclass VPseudoTernaryWithTailPolicyRoundingMode<VReg RetClass,
def "_" # mx # "_E" # sew # "_MASK"
: VPseudoTernaryMaskPolicyRoundingMode<RetClass, Op1Class,
Op2Class>,
- RISCVMaskedPseudo<MaskIdx=3, MaskAffectsRes=true>;
+ RISCVMaskedPseudo<MaskIdx=3, ActiveAffectsRes=true>;
}
}
diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll b/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
index 61acf1afa94de..ec03f773c7108 100644
--- a/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
@@ -19,6 +19,17 @@ define <vscale x 4 x i32> @vadd(<vscale x 4 x i32> %passthru, <vscale x 4 x i32>
ret <vscale x 4 x i32> %w
}
+define <vscale x 4 x i32> @vadd_mask(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m, iXLen %vl) {
+; CHECK-LABEL: vadd_mask:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m2, tu, mu
+; CHECK-NEXT: vadd.vv v8, v10, v12, v0.t
+; CHECK-NEXT: ret
+ %v = call <vscale x 4 x i32> @llvm.riscv.vadd.mask.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m, iXLen %vl, iXLen 3)
+ %w = call <vscale x 4 x i32> @llvm.riscv.vmv.v.v.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, iXLen %vl)
+ ret <vscale x 4 x i32> %w
+}
+
define <vscale x 4 x i32> @vadd_undef(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
; CHECK-LABEL: vadd_undef:
; CHECK: # %bb.0:
@@ -106,8 +117,8 @@ declare <vscale x 4 x float> @llvm.riscv.vmv.v.v.nxv4f32(<vscale x 4 x float>, <
declare <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, iXLen, iXLen)
-define <vscale x 4 x float> @vfadd(<vscale x 4 x float> %passthru, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl1, iXLen %vl2) {
-; CHECK-LABEL: vfadd:
+define <vscale x 4 x float> @unfoldable_vfadd(<vscale x 4 x float> %passthru, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: unfoldable_vfadd:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli zero, a0, e32, m2, ta, ma
; CHECK-NEXT: vfadd.vv v10, v10, v12
@@ -118,3 +129,42 @@ define <vscale x 4 x float> @vfadd(<vscale x 4 x float> %passthru, <vscale x 4 x
%w = call <vscale x 4 x float> @llvm.riscv.vmv.v.v.nxv4f32(<vscale x 4 x float> %passthru, <vscale x 4 x float> %v, iXLen %vl2)
ret <vscale x 4 x float> %w
}
+
+define <vscale x 4 x float> @foldable_vfadd(<vscale x 4 x float> %passthru, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
+; CHECK-LABEL: foldable_vfadd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT: vfadd.vv v8, v10, v12
+; CHECK-NEXT: ret
+ %v = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen 7, iXLen %vl)
+ %w = call <vscale x 4 x float> @llvm.riscv.vmv.v.v.nxv4f32(<vscale x 4 x float> %passthru, <vscale x 4 x float> %v, iXLen %vl)
+ ret <vscale x 4 x float> %w
+}
+
+; This shouldn't be folded because we need to preserve exceptions with
+; "fpexcept.strict" exception behaviour, and changing the VL may hide them.
+define <vscale x 4 x float> @unfoldable_constrained_fadd(<vscale x 4 x float> %passthru, <vscale x 4 x float> %x, <vscale x 4 x float> %y, iXLen %vl) strictfp {
+; CHECK-LABEL: unfoldable_constrained_fadd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT: vfadd.vv v10, v10, v12
+; CHECK-NEXT: vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT: vmv.v.v v8, v10
+; CHECK-NEXT: ret
+ %a = call <vscale x 4 x float> @llvm.experimental.constrained.fadd(<vscale x 4 x float> %x, <vscale x 4 x float> %y, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
+ %b = call <vscale x 4 x float> @llvm.riscv.vmv.v.v.nxv4f32(<vscale x 4 x float> %passthru, <vscale x 4 x float> %a, iXLen %vl) strictfp
+ ret <vscale x 4 x float> %b
+}
+
+define <vscale x 2 x i32> @unfoldable_vredsum(<vscale x 2 x i32> %passthru, <vscale x 4 x i32> %x, <vscale x 2 x i32> %y) {
+; CHECK-LABEL: unfoldable_vredsum:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT: vredsum.vs v9, v10, v9
+; CHECK-NEXT: vsetivli zero, 1, e32, m1, tu, ma
+; CHECK-NEXT: vmv.v.v v8, v9
+; CHECK-NEXT: ret
+ %a = call <vscale x 2 x i32> @llvm.riscv.vredsum.nxv2i32.nxv4i32(<vscale x 2 x i32> poison, <vscale x 4 x i32> %x, <vscale x 2 x i32> %y, iXLen -1)
+ %b = call <vscale x 2 x i32> @llvm.riscv.vmv.v.v.nxv2i32(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %a, iXLen 1)
+ ret <vscale x 2 x i32> %b
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index b6921abf8fdf4..a08bcae074b9b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -1014,6 +1014,24 @@ define <vscale x 2 x float> @vfredusum_allones_mask(<vscale x 2 x float> %passth
ret <vscale x 2 x float> %b
}
+define <vscale x 2 x i32> @unfoldable_vredsum_allones_mask_diff_vl(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %x, <vscale x 2 x i32> %y) {
+; CHECK-LABEL: unfoldable_vredsum_allones_mask_diff_vl:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmv1r.v v11, v8
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, tu, ma
+; CHECK-NEXT: vredsum.vs v11, v9, v10
+; CHECK-NEXT: vsetivli zero, 1, e32, m1, tu, ma
+; CHECK-NEXT: vmv.v.v v8, v11
+; CHECK-NEXT: ret
+ %a = call <vscale x 2 x i32> @llvm.riscv.vredsum.nxv2i32.nxv2i32(
+ <vscale x 2 x i32> %passthru,
+ <vscale x 2 x i32> %x,
+ <vscale x 2 x i32> %y,
+ i64 -1)
+ %b = call <vscale x 2 x i32> @llvm.riscv.vmerge.nxv2i32.nxv2i32(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %passthru, <vscale x 2 x i32> %a, <vscale x 2 x i1> splat (i1 -1), i64 1)
+ ret <vscale x 2 x i32> %b
+}
+
declare <vscale x 32 x i16> @llvm.riscv.vle.nxv32i16.i64(<vscale x 32 x i16>, ptr nocapture, i64)
declare <vscale x 32 x i8> @llvm.riscv.vssubu.mask.nxv32i8.i8.i64(<vscale x 32 x i8>, <vscale x 32 x i8>, i8, <vscale x 32 x i1>, i64, i64 immarg)
declare <vscale x 32 x i1> @llvm.riscv.vmseq.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>, <vscale x 32 x i8>, i64)
|
preames
approved these changes
Jul 16, 2024
yuxuanchen1997
pushed a commit
that referenced
this pull request
Jul 25, 2024
…ed (#99006) Summary: When folding, we currently check if the pseudo's result is not lanewise (e.g. vredsum.vs or viota.m) and bail if we're changing the mask. However we also need to check for the AVL too. This patch bails if the AVL changed for these pseudos, and also renames the pseudo table property to be more explicit. Test Plan: Reviewers: Subscribers: Tasks: Tags: Differential Revision: https://phabricator.intern.facebook.com/D60250957
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
When folding, we currently check if the pseudo's result is not lanewise (e.g. vredsum.vs or viota.m) and bail if we're changing the mask.
However we also need to check for the AVL too.
This patch bails if the AVL changed for these pseudos, and also renames the pseudo table property to be more explicit.