[JITLink][LoongArch] Support R_LARCH_ALIGN relaxation #122259
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
Alignment relaxation must be implemented due to the reason shown in the closed pr #121097. |
|
@llvm/pr-subscribers-backend-loongarch Author: ZhaoQi (zhaoqi5) ChangesLinker relaxation is not implemented for jitlink now. But if This commit adapts lld's algorithm to jitlink. Currently, only Without this, interpreting C++ code using clang-repl or running Similar to 310473c but only implement align. Full diff: https://github.com/llvm/llvm-project/pull/122259.diff 5 Files Affected:
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
index 7e5d0f1f918521..a8655dc6f14e37 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h
@@ -33,6 +33,10 @@ Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_loongarch(
void link_ELF_loongarch(std::unique_ptr<LinkGraph> G,
std::unique_ptr<JITLinkContext> Ctx);
+/// Returns a pass that performs linker relaxation. Should be added to
+/// PostAllocationPasses.
+LinkGraphPassFunction createRelaxationPass_ELF_loongarch();
+
} // end namespace jitlink
} // end namespace llvm
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h b/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
index 1db4b822181094..d6025edf7d110d 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h
@@ -225,6 +225,13 @@ enum EdgeKind_loongarch : Edge::Kind {
/// out-of-range error will be returned.
///
Call36PCRel,
+
+ /// Alignment requirement used by linker relaxation.
+ ///
+ /// Linker relaxation will use this to ensure all code sequences are properly
+ /// aligned and then remove these edges from the graph.
+ ///
+ AlignRelaxable,
};
/// Returns a string name for the given loongarch edge. For debugging purposes
@@ -362,6 +369,9 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
*(little32_t *)(FixupPtr + 4) = Jirl | Lo16;
break;
}
+ case AlignRelaxable:
+ // Ignore when the relaxation pass did not run
+ break;
default:
return make_error<JITLinkError>(
"In graph " + G.getName() + ", section " + B.getSection().getName() +
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
index a12e9f33e80a6b..3f0a7b645e83fc 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp
@@ -45,6 +45,238 @@ class ELFJITLinker_loongarch : public JITLinker<ELFJITLinker_loongarch> {
}
};
+namespace {
+
+struct SymbolAnchor {
+ uint64_t Offset;
+ Symbol *Sym;
+ bool End; // true for the anchor of getOffset() + getSize()
+};
+
+struct BlockRelaxAux {
+ // This records symbol start and end offsets which will be adjusted according
+ // to the nearest RelocDeltas element.
+ SmallVector<SymbolAnchor, 0> Anchors;
+ // All edges that either 1) are R_LARCH_ALIGN or 2) have a R_LARCH_RELAX edge
+ // at the same offset.
+ SmallVector<Edge *, 0> RelaxEdges;
+ // For RelaxEdges[I], the actual offset is RelaxEdges[I]->getOffset() - (I ?
+ // RelocDeltas[I - 1] : 0).
+ SmallVector<uint32_t, 0> RelocDeltas;
+ // For RelaxEdges[I], the actual type is EdgeKinds[I].
+ SmallVector<Edge::Kind, 0> EdgeKinds;
+ // List of rewritten instructions. Contains one raw encoded instruction per
+ // element in EdgeKinds that isn't Invalid or R_LARCH_ALIGN.
+ SmallVector<uint32_t, 0> Writes;
+};
+
+struct RelaxAux {
+ DenseMap<Block *, BlockRelaxAux> Blocks;
+};
+
+} // namespace
+
+static bool shouldRelax(const Section &S) {
+ return (S.getMemProt() & orc::MemProt::Exec) != orc::MemProt::None;
+}
+
+static bool isRelaxable(const Edge &E) {
+ switch (E.getKind()) {
+ default:
+ return false;
+ case AlignRelaxable:
+ return true;
+ }
+}
+
+static RelaxAux initRelaxAux(LinkGraph &G) {
+ RelaxAux Aux;
+ for (auto &S : G.sections()) {
+ if (!shouldRelax(S))
+ continue;
+ for (auto *B : S.blocks()) {
+ auto BlockEmplaceResult = Aux.Blocks.try_emplace(B);
+ assert(BlockEmplaceResult.second && "Block encountered twice");
+ auto &BlockAux = BlockEmplaceResult.first->second;
+
+ for (auto &E : B->edges())
+ if (isRelaxable(E))
+ BlockAux.RelaxEdges.push_back(&E);
+
+ if (BlockAux.RelaxEdges.empty()) {
+ Aux.Blocks.erase(BlockEmplaceResult.first);
+ continue;
+ }
+
+ const auto NumEdges = BlockAux.RelaxEdges.size();
+ BlockAux.RelocDeltas.resize(NumEdges, 0);
+ BlockAux.EdgeKinds.resize_for_overwrite(NumEdges);
+
+ // Store anchors (offset and offset+size) for symbols.
+ for (auto *Sym : S.symbols()) {
+ if (!Sym->isDefined() || &Sym->getBlock() != B)
+ continue;
+
+ BlockAux.Anchors.push_back({Sym->getOffset(), Sym, false});
+ BlockAux.Anchors.push_back(
+ {Sym->getOffset() + Sym->getSize(), Sym, true});
+ }
+ }
+ }
+
+ // Sort anchors by offset so that we can find the closest relocation
+ // efficiently. For a zero size symbol, ensure that its start anchor precedes
+ // its end anchor. For two symbols with anchors at the same offset, their
+ // order does not matter.
+ for (auto &BlockAuxIter : Aux.Blocks) {
+ llvm::sort(BlockAuxIter.second.Anchors, [](auto &A, auto &B) {
+ return std::make_pair(A.Offset, A.End) < std::make_pair(B.Offset, B.End);
+ });
+ }
+
+ return Aux;
+}
+
+static void relaxAlign(orc::ExecutorAddr Loc, const Edge &E, uint32_t &Remove,
+ Edge::Kind &NewEdgeKind) {
+ const uint64_t Addend =
+ !E.getTarget().isDefined() ? Log2_64(E.getAddend()) + 1 : E.getAddend();
+ const uint64_t AllBytes = (1ULL << (Addend & 0xff)) - 4;
+ const uint64_t Align = 1ULL << (Addend & 0xff);
+ const uint64_t MaxBytes = Addend >> 8;
+ const uint64_t Off = Loc.getValue() & (Align - 1);
+ const uint64_t CurBytes = Off == 0 ? 0 : Align - Off;
+ // All bytes beyond the alignment boundary should be removed.
+ // If emit bytes more than max bytes to emit, remove all.
+ if (MaxBytes != 0 && CurBytes > MaxBytes)
+ Remove = AllBytes;
+ else
+ Remove = AllBytes - CurBytes;
+
+ assert(static_cast<int32_t>(Remove) >= 0 &&
+ "R_LARCH_ALIGN needs expanding the content");
+ NewEdgeKind = AlignRelaxable;
+}
+
+static bool relaxBlock(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) {
+ const auto BlockAddr = Block.getAddress();
+ bool Changed = false;
+ ArrayRef<SymbolAnchor> SA = ArrayRef(Aux.Anchors);
+ uint32_t Delta = 0;
+
+ Aux.EdgeKinds.assign(Aux.EdgeKinds.size(), Edge::Invalid);
+ Aux.Writes.clear();
+
+ for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) {
+ const auto Loc = BlockAddr + E->getOffset() - Delta;
+ auto &Cur = Aux.RelocDeltas[I];
+ uint32_t Remove = 0;
+ switch (E->getKind()) {
+ case AlignRelaxable:
+ relaxAlign(Loc, *E, Remove, Aux.EdgeKinds[I]);
+ break;
+ default:
+ llvm_unreachable("Unexpected relaxable edge kind");
+ }
+
+ // For all anchors whose offsets are <= E->getOffset(), they are preceded by
+ // the previous relocation whose RelocDeltas value equals Delta.
+ // Decrease their offset and update their size.
+ for (; SA.size() && SA[0].Offset <= E->getOffset(); SA = SA.slice(1)) {
+ if (SA[0].End)
+ SA[0].Sym->setSize(SA[0].Offset - Delta - SA[0].Sym->getOffset());
+ else
+ SA[0].Sym->setOffset(SA[0].Offset - Delta);
+ }
+
+ Delta += Remove;
+ if (Delta != Cur) {
+ Cur = Delta;
+ Changed = true;
+ }
+ }
+
+ for (const SymbolAnchor &A : SA) {
+ if (A.End)
+ A.Sym->setSize(A.Offset - Delta - A.Sym->getOffset());
+ else
+ A.Sym->setOffset(A.Offset - Delta);
+ }
+
+ return Changed;
+}
+
+static bool relaxOnce(LinkGraph &G, RelaxAux &Aux) {
+ bool Changed = false;
+
+ for (auto &[B, BlockAux] : Aux.Blocks)
+ Changed |= relaxBlock(G, *B, BlockAux);
+
+ return Changed;
+}
+
+static void finalizeBlockRelax(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) {
+ auto Contents = Block.getAlreadyMutableContent();
+ auto *Dest = Contents.data();
+ uint32_t Offset = 0;
+ uint32_t Delta = 0;
+
+ // Update section content: remove NOPs for R_LARCH_ALIGN and rewrite
+ // instructions for relaxed relocations.
+ for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) {
+ uint32_t Remove = Aux.RelocDeltas[I] - Delta;
+ Delta = Aux.RelocDeltas[I];
+ if (Remove == 0 && Aux.EdgeKinds[I] == Edge::Invalid)
+ continue;
+
+ // Copy from last location to the current relocated location.
+ const auto Size = E->getOffset() - Offset;
+ std::memmove(Dest, Contents.data() + Offset, Size);
+ Dest += Size;
+ Offset = E->getOffset() + Remove;
+ }
+
+ std::memmove(Dest, Contents.data() + Offset, Contents.size() - Offset);
+
+ // Fixup edge offsets and kinds.
+ Delta = 0;
+ size_t I = 0;
+ for (auto &E : Block.edges()) {
+ E.setOffset(E.getOffset() - Delta);
+
+ if (I < Aux.RelaxEdges.size() && Aux.RelaxEdges[I] == &E) {
+ if (Aux.EdgeKinds[I] != Edge::Invalid)
+ E.setKind(Aux.EdgeKinds[I]);
+
+ Delta = Aux.RelocDeltas[I];
+ ++I;
+ }
+ }
+
+ // Remove AlignRelaxable edges: all other relaxable edges got modified and
+ // will be used later while linking. Alignment is entirely handled here so we
+ // don't need these edges anymore.
+ for (auto IE = Block.edges().begin(); IE != Block.edges().end();) {
+ if (IE->getKind() == AlignRelaxable)
+ IE = Block.removeEdge(IE);
+ else
+ ++IE;
+ }
+}
+
+static void finalizeRelax(LinkGraph &G, RelaxAux &Aux) {
+ for (auto &[B, BlockAux] : Aux.Blocks)
+ finalizeBlockRelax(G, *B, BlockAux);
+}
+
+static Error relax(LinkGraph &G) {
+ auto Aux = initRelaxAux(G);
+ while (relaxOnce(G, Aux)) {
+ }
+ finalizeRelax(G, Aux);
+ return Error::success();
+}
+
template <typename ELFT>
class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
private:
@@ -74,6 +306,8 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
return RequestGOTAndTransformToPageOffset12;
case ELF::R_LARCH_CALL36:
return Call36PCRel;
+ case ELF::R_LARCH_ALIGN:
+ return AlignRelaxable;
}
return make_error<JITLinkError>(
@@ -81,6 +315,11 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
object::getELFRelocationTypeName(ELF::EM_LOONGARCH, Type));
}
+ EdgeKind_loongarch getRelaxableRelocationKind(EdgeKind_loongarch Kind) {
+ // TODO: Implement more. Just ignore all relaxations now.
+ return Kind;
+ }
+
Error addRelocations() override {
LLVM_DEBUG(dbgs() << "Processing relocations:\n");
@@ -99,6 +338,25 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
Block &BlockToFix) {
using Base = ELFLinkGraphBuilder<ELFT>;
+ uint32_t Type = Rel.getType(false);
+ int64_t Addend = Rel.r_addend;
+
+ if (Type == ELF::R_LARCH_RELAX) {
+ if (BlockToFix.edges_empty())
+ return make_error<StringError>(
+ "R_LARCH_RELAX without preceding relocation",
+ inconvertibleErrorCode());
+
+ auto &PrevEdge = *std::prev(BlockToFix.edges().end());
+ auto Kind = static_cast<EdgeKind_loongarch>(PrevEdge.getKind());
+ PrevEdge.setKind(getRelaxableRelocationKind(Kind));
+ return Error::success();
+ }
+
+ Expected<loongarch::EdgeKind_loongarch> Kind = getRelocationKind(Type);
+ if (!Kind)
+ return Kind.takeError();
+
uint32_t SymbolIndex = Rel.getSymbol(false);
auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
if (!ObjSymbol)
@@ -113,12 +371,6 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {
Base::GraphSymbols.size()),
inconvertibleErrorCode());
- uint32_t Type = Rel.getType(false);
- Expected<loongarch::EdgeKind_loongarch> Kind = getRelocationKind(Type);
- if (!Kind)
- return Kind.takeError();
-
- int64_t Addend = Rel.r_addend;
auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
Edge GE(*Kind, Offset, *GraphSymbol, Addend);
@@ -209,6 +461,9 @@ void link_ELF_loongarch(std::unique_ptr<LinkGraph> G,
// Add an in-place GOT/PLTStubs build pass.
Config.PostPrunePasses.push_back(buildTables_ELF_loongarch);
+
+ // Add a linker relaxation pass.
+ Config.PostAllocationPasses.push_back(relax);
}
if (auto Err = Ctx->modifyPassConfig(*G, Config))
@@ -217,5 +472,7 @@ void link_ELF_loongarch(std::unique_ptr<LinkGraph> G,
ELFJITLinker_loongarch::link(std::move(Ctx), std::move(G), std::move(Config));
}
+LinkGraphPassFunction createRelaxationPass_ELF_loongarch() { return relax; }
+
} // namespace jitlink
} // namespace llvm
diff --git a/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp
index cdb3da04354eee..a5579b074de7cf 100644
--- a/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp
@@ -52,6 +52,7 @@ const char *getEdgeKindName(Edge::Kind K) {
KIND_NAME_CASE(RequestGOTAndTransformToPage20)
KIND_NAME_CASE(RequestGOTAndTransformToPageOffset12)
KIND_NAME_CASE(Call36PCRel)
+ KIND_NAME_CASE(AlignRelaxable)
default:
return getGenericEdgeKindName(K);
}
diff --git a/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s b/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s
new file mode 100644
index 00000000000000..ec1aceb25b6147
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s
@@ -0,0 +1,95 @@
+# RUN: rm -rf %t && mkdir %t && cd %t
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 -mattr=+relax %s -o %t.la32
+# RUN: llvm-jitlink --noexec \
+# RUN: -slab-allocate 100Kb -slab-address 0x0 -slab-page-size 16384 \
+# RUN: --check %s %t.la32
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t.la64
+# RUN: llvm-jitlink --noexec \
+# RUN: -slab-allocate 100Kb -slab-address 0x0 -slab-page-size 16384 \
+# RUN: --check %s %t.la64
+
+## Test that we can handle R_LARCH_ALIGN.
+
+ .text
+
+ .globl main,align4,align8,align16,align32,alignmax12,alignmax8
+ .type main,@function
+main:
+ bl f
+ .align 2
+align4:
+ bl f
+ .size align4, .-align4
+ .align 3
+align8:
+ bl f
+ .size align8, .-align8
+ .align 4
+align16:
+ bl f
+ .size align16, .-align16
+ .align 5
+align32:
+ bl f
+ .size align32, .-align32
+ .align 4,,12
+alignmax12:
+ bl f
+ .size alignmax12, .-alignmax12
+ .align 4,,8
+alignmax8:
+ bl f
+ .size alignmax8, .-alignmax8
+ .size main, .-main
+
+ .globl f
+f:
+ ret
+ .size f, .-f
+
+# jitlink-check: main = 0x0
+# jitlink-check: align4 = 0x4
+# jitlink-check: align8 = 0x8
+# jitlink-check: align16 = 0x10
+# jitlink-check: align32 = 0x20
+# jitlink-check: alignmax12 = 0x30
+## 3 nops (12 bytes) should be inserted to satisfy alignment.
+## But the max bytes we can insert is 8. So alignment is ignored.
+# jitlink-check: alignmax8 = 0x34
+
+## main: bl f
+# jitlink-check: (*{4}(main))[31:26] = 0x15
+# jitlink-check: decode_operand(main, 0)[27:0] = (f - main)[27:0]
+
+## align 4: bl f
+# jitlink-check: (*{4}(align4))[31:26] = 0x15
+# jitlink-check: decode_operand(align4, 0)[27:0] = (f - align4)[27:0]
+
+## align8: bl f; nop
+# jitlink-check: (*{4}(align8))[31:26] = 0x15
+# jitlink-check: decode_operand(align8, 0)[27:0] = (f - align8)[27:0]
+# jitlink-check: (*{4}(align8+4)) = 0x3400000
+
+## align16: bl f; nop; nop; nop
+# jitlink-check: (*{4}(align16))[31:26] = 0x15
+# jitlink-check: decode_operand(align16, 0)[27:0] = (f - align16)[27:0]
+# jitlink-check: (*{4}(align16+4)) = 0x3400000
+# jitlink-check: (*{4}(align16+8)) = 0x3400000
+# jitlink-check: (*{4}(align16+12)) = 0x3400000
+
+## align32: bl f; nop; nop; nop
+# jitlink-check: (*{4}(align32))[31:26] = 0x15
+# jitlink-check: decode_operand(align32, 0)[27:0] = (f - align32)[27:0]
+# jitlink-check: (*{4}(align32+4)) = 0x3400000
+# jitlink-check: (*{4}(align32+8)) = 0x3400000
+# jitlink-check: (*{4}(align32+12)) = 0x3400000
+
+## alignmax12: bl f
+# jitlink-check: (*{4}(alignmax12))[31:26] = 0x15
+# jitlink-check: decode_operand(alignmax12, 0)[27:0] = (f - alignmax12)[27:0]
+
+## alignmax8: bl f
+# jitlink-check: (*{4}(alignmax8))[31:26] = 0x15
+# jitlink-check: decode_operand(alignmax8, 0)[27:0] = (f - alignmax8)[27:0]
|
Linker relaxation is not implemented for jitlink now (maybe implement in the future). But if relaxation is enabled by clang, R_LARCH_RELAX and R_LARCH_ALIGN relocations will be emitted. So we just ignore linker relaxation and check the alignment now. If not, interpreting C++ code using clang-repl when relaxation is enabled will occur error: `JIT session error: Unsupported loongarch relocation:102: R_LARCH_ALIGN`. Similar to: f5b5398.
Linker relaxation is not implemented for jitlink now. But if relaxation is enabled by clang, R_LARCH_RELAX and R_LARCH_ALIGN relocations will be emitted. This commit adapts lld's algorithm to jitlink. Currently, only relaxing R_LARCH_ALIGN is implemented. Other relaxable relocs can be implemented in the future. Without this, interpreting C++ code using clang-repl or running ir using lli when relaxation is enabled will occur error: `JIT session error: Unsupported loongarch relocation:102: R_LARCH_ALIGN`. Similar to 310473c but only implement align.
zhaoqi5
force-pushed
the
users/zhaoqi5/jitlink-relax-align
branch
from
41de630 to
91aea9b
Compare
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Linker relaxation is not implemented for jitlink now. But if
relaxation is enabled by clang, R_LARCH_RELAX and R_LARCH_ALIGN
relocations will be emitted.
This commit adapts lld's algorithm to jitlink. Currently, only
relaxing R_LARCH_ALIGN is implemented. Other relaxable relocs
can be implemented in the future.
Without this, interpreting C++ code using clang-repl or running
ir using lli when relaxation is enabled will occur error:
JIT session error: Unsupported loongarch relocation:102: R_LARCH_ALIGN.Similar to 310473c but only implement align.