Skip to content

[AArch64][SME] Allow inlining when streaming-mode attributes dont match up. #68415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 30, 2023
Merged

[AArch64][SME] Allow inlining when streaming-mode attributes dont match up. #68415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ec46c7
[AArch64][SME] Allow inlining when streaming-mode attributes dont mat…
sdesmalen-arm Sep 7, 2023
1ff85fa
Clang-format, update test names
sdesmalen-arm Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,16 +190,49 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode(
static cl::opt<bool> EnableScalableAutovecInStreamingMode(
"enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden);

static bool isSMEABIRoutineCall(const CallInst &CI) {
const auto *F = CI.getCalledFunction();
return F && StringSwitch<bool>(F->getName())
.Case("__arm_sme_state", true)
.Case("__arm_tpidr2_save", true)
.Case("__arm_tpidr2_restore", true)
.Case("__arm_za_disable", true)
.Default(false);
}

/// Returns true if the function has explicit operations that can only be
/// lowered using incompatible instructions for the selected mode. This also
/// returns true if the function F may use or modify ZA state.
static bool hasPossibleIncompatibleOps(const Function *F) {
for (const BasicBlock &BB : *F) {
for (const Instruction &I : BB) {
// Be conservative for now and assume that any call to inline asm or to
// intrinsics could could result in non-streaming ops (e.g. calls to
// @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that
// all native LLVM instructions can be lowered to compatible instructions.
if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() &&
(cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) ||
isSMEABIRoutineCall(cast<CallInst>(I))))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be a great place for a remark that explains why we can't inline.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes that would be useful. We could update the TargetTransformInfo::areInlineCompatible interface to return an optional message, so that InlineCost can pass that into the InlineResult::failure("conflicting attributes[: some specific reason here]"). I'll look into that.

return true;
}
}
return false;
}

bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
const Function *Callee) const {
SMEAttrs CallerAttrs(*Caller);
SMEAttrs CalleeAttrs(*Callee);
if (CallerAttrs.requiresSMChange(CalleeAttrs,
/*BodyOverridesInterface=*/true) ||
CallerAttrs.requiresLazySave(CalleeAttrs) ||
CalleeAttrs.hasNewZABody())
if (CalleeAttrs.hasNewZABody())
return false;

if (CallerAttrs.requiresLazySave(CalleeAttrs) ||
CallerAttrs.requiresSMChange(CalleeAttrs,
/*BodyOverridesInterface=*/true)) {
if (hasPossibleIncompatibleOps(Callee))
return false;
}

const TargetMachine &TM = getTLI()->getTargetMachine();

const FeatureBitset &CallerBits =
Expand Down
138 changes: 108 additions & 30 deletions llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,11 @@ entry:
; [ ] N -> SC
; [ ] N -> N + B
; [ ] N -> SC + B
define void @normal_caller_streaming_callee_dont_inline() {
; CHECK-LABEL: define void @normal_caller_streaming_callee_dont_inline
define void @normal_caller_streaming_callee_inline() {
; CHECK-LABEL: define void @normal_caller_streaming_callee_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -136,11 +136,11 @@ entry:
; [ ] N -> SC
; [x] N -> N + B
; [ ] N -> SC + B
define void @normal_caller_locally_streaming_callee_dont_inline() {
; CHECK-LABEL: define void @normal_caller_locally_streaming_callee_dont_inline
define void @normal_caller_locally_streaming_callee_inline() {
; CHECK-LABEL: define void @normal_caller_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @locally_streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -153,11 +153,11 @@ entry:
; [ ] N -> SC
; [ ] N -> N + B
; [x] N -> SC + B
define void @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() {
; CHECK-LABEL: define void @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline
define void @normal_caller_streaming_compatible_locally_streaming_callee_inline() {
; CHECK-LABEL: define void @normal_caller_streaming_compatible_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @streaming_compatible_locally_streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -170,11 +170,11 @@ entry:
; [ ] S -> SC
; [ ] S -> N + B
; [ ] S -> SC + B
define void @streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_normal_callee_dont_inline
define void @streaming_caller_normal_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_normal_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @normal_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -255,11 +255,11 @@ entry:
; [ ] N + B -> SC
; [ ] N + B -> N + B
; [ ] N + B -> SC + B
define void @locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define void @locally_streaming_caller_normal_callee_dont_inline
define void @locally_streaming_caller_normal_callee_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define void @locally_streaming_caller_normal_callee_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @normal_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -340,11 +340,11 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [ ] SC -> SC + B
define void @streaming_compatible_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_normal_callee_dont_inline
define void @streaming_compatible_caller_normal_callee_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_normal_callee_inline
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @normal_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -357,11 +357,11 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [ ] SC -> SC + B
define void @streaming_compatible_caller_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_streaming_callee_dont_inline
define void @streaming_compatible_caller_streaming_callee_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_streaming_callee_inline
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -391,11 +391,11 @@ entry:
; [ ] SC -> SC
; [x] SC -> N + B
; [ ] SC -> SC + B
define void @streaming_compatible_caller_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_locally_streaming_callee_dont_inline
define void @streaming_compatible_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @locally_streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -408,11 +408,11 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [x] SC -> SC + B
define void @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline
define void @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define void @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @streaming_compatible_locally_streaming_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -424,11 +424,11 @@ entry:
; [ ] SC + B -> SC
; [ ] SC + B -> N + B
; [ ] SC + B -> SC + B
define void @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define void @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline
define void @streaming_compatible_locally_streaming_caller_normal_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define void @streaming_compatible_locally_streaming_caller_normal_callee_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @normal_callee()
; CHECK-NEXT: call void @inlined_body()
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -503,3 +503,81 @@ entry:
call void @streaming_compatible_locally_streaming_callee()
ret void
}

define void @normal_callee_with_inlineasm() {
; CHECK-LABEL: define void @normal_callee_with_inlineasm
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void asm sideeffect "
; CHECK-NEXT: ret void
;
entry:
call void asm sideeffect "; inlineasm", ""()
ret void
}

define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_normal_callee_with_inlineasm_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @normal_callee_with_inlineasm()
; CHECK-NEXT: ret void
;
entry:
call void @normal_callee_with_inlineasm()
ret void
}

define i64 @normal_callee_with_intrinsic_call() {
; CHECK-LABEL: define i64 @normal_callee_with_intrinsic_call
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.aarch64.sve.cntb(i32 4)
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
%res = call i64 @llvm.aarch64.sve.cntb(i32 4)
ret i64 %res
}

define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[RES:%.*]] = call i64 @normal_callee_with_intrinsic_call()
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
%res = call i64 @normal_callee_with_intrinsic_call()
ret i64 %res
}

declare i64 @llvm.aarch64.sve.cntb(i32)

define i64 @normal_callee_call_sme_state() {
; CHECK-LABEL: define i64 @normal_callee_call_sme_state
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[RES:%.*]] = call { i64, i64 } @__arm_sme_state()
; CHECK-NEXT: [[RES_0:%.*]] = extractvalue { i64, i64 } [[RES]], 0
; CHECK-NEXT: ret i64 [[RES_0]]
;
entry:
%res = call {i64, i64} @__arm_sme_state()
%res.0 = extractvalue {i64, i64} %res, 0
ret i64 %res.0
}

declare {i64, i64} @__arm_sme_state()

define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[RES:%.*]] = call i64 @normal_callee_call_sme_state()
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
%res = call i64 @normal_callee_call_sme_state()
ret i64 %res
}
Loading