|
| 1 | +//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// |
| 2 | +// |
| 3 | +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +// See https://llvm.org/LICENSE.txt for license information. |
| 5 | +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +// |
| 7 | +//===----------------------------------------------------------------------===// |
| 8 | +// |
| 9 | +// Populates the VFABI attribute with the scalar-to-vector mappings |
| 10 | +// from the TargetLibraryInfo. |
| 11 | +// |
| 12 | +//===----------------------------------------------------------------------===// |
| 13 | + |
| 14 | +#include "llvm/Transforms/Utils/InjectTLIMappings.h" |
| 15 | +#include "llvm/ADT/Statistic.h" |
| 16 | +#include "llvm/Analysis/VectorUtils.h" |
| 17 | +#include "llvm/IR/InstIterator.h" |
| 18 | +#include "llvm/Transforms/Utils.h" |
| 19 | +#include "llvm/Transforms/Utils/ModuleUtils.h" |
| 20 | + |
| 21 | +using namespace llvm; |
| 22 | + |
| 23 | +#define DEBUG_TYPE "inject-tli-mappings" |
| 24 | + |
| 25 | +STATISTIC(NumCallInjected, |
| 26 | + "Number of calls in which the mappings have been injected."); |
| 27 | + |
| 28 | +STATISTIC(NumVFDeclAdded, |
| 29 | + "Number of function declarations that have been added."); |
| 30 | +STATISTIC(NumCompUsedAdded, |
| 31 | + "Number of `@llvm.compiler.used` operands that have been added."); |
| 32 | + |
| 33 | +/// Helper function to map the TLI name to a strings that holds |
| 34 | +/// scalar-to-vector mapping. |
| 35 | +/// |
| 36 | +/// _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>) |
| 37 | +/// |
| 38 | +/// where: |
| 39 | +/// |
| 40 | +/// <isa> = "_LLVM_" |
| 41 | +/// <mask> = "N". Note: TLI does not support masked interfaces. |
| 42 | +/// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor` |
| 43 | +/// field of the `VecDesc` struct. |
| 44 | +/// <vparams> = "v", as many as are the number of parameters of CI. |
| 45 | +/// <scalarname> = the name of the scalar function called by CI. |
| 46 | +/// <vectorname> = the name of the vector function mapped by the TLI. |
| 47 | +static std::string mangleTLIName(StringRef VectorName, const CallInst &CI, |
| 48 | + unsigned VF) { |
| 49 | + SmallString<256> Buffer; |
| 50 | + llvm::raw_svector_ostream Out(Buffer); |
| 51 | + Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF; |
| 52 | + for (unsigned I = 0; I < CI.getNumArgOperands(); ++I) |
| 53 | + Out << "v"; |
| 54 | + Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")"; |
| 55 | + return Out.str(); |
| 56 | +} |
| 57 | + |
| 58 | +/// A helper function for converting Scalar types to vector types. |
| 59 | +/// If the incoming type is void, we return void. If the VF is 1, we return |
| 60 | +/// the scalar type. |
| 61 | +static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) { |
| 62 | + if (Scalar->isVoidTy() || VF == 1) |
| 63 | + return Scalar; |
| 64 | + return VectorType::get(Scalar, {VF, isScalable}); |
| 65 | +} |
| 66 | + |
| 67 | +/// A helper function that adds the vector function declaration that |
| 68 | +/// vectorizes the CallInst CI with a vectorization factor of VF |
| 69 | +/// lanes. The TLI assumes that all parameters and the return type of |
| 70 | +/// CI (other than void) need to be widened to a VectorType of VF |
| 71 | +/// lanes. |
| 72 | +static void addVariantDeclaration(CallInst &CI, const unsigned VF, |
| 73 | + const StringRef VFName) { |
| 74 | + Module *M = CI.getModule(); |
| 75 | + |
| 76 | + // Add function declaration. |
| 77 | + Type *RetTy = ToVectorTy(CI.getType(), VF); |
| 78 | + SmallVector<Type *, 4> Tys; |
| 79 | + for (Value *ArgOperand : CI.arg_operands()) |
| 80 | + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); |
| 81 | + assert(!CI.getFunctionType()->isVarArg() && |
| 82 | + "VarArg functions are not supported."); |
| 83 | + FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); |
| 84 | + Function *VectorF = |
| 85 | + Function::Create(FTy, Function::ExternalLinkage, VFName, M); |
| 86 | + VectorF->copyAttributesFrom(CI.getCalledFunction()); |
| 87 | + ++NumVFDeclAdded; |
| 88 | + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName |
| 89 | + << "` of type " << *(VectorF->getType()) << "\n"); |
| 90 | + |
| 91 | + // Make function declaration (without a body) "sticky" in the IR by |
| 92 | + // listing it in the @llvm.compiler.used intrinsic. |
| 93 | + assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " |
| 94 | + "only on declarations."); |
| 95 | + appendToCompilerUsed(*M, {VectorF}); |
| 96 | + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName |
| 97 | + << "` to `@llvm.compiler.used`.\n"); |
| 98 | + ++NumCompUsedAdded; |
| 99 | +} |
| 100 | + |
| 101 | +static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { |
| 102 | + // This is needed to make sure we don't query the TLI for calls to |
| 103 | + // bitcast of function pointers, like `%call = call i32 (i32*, ...) |
| 104 | + // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, |
| 105 | + // as such calls make the `isFunctionVectorizable` raise an |
| 106 | + // exception. |
| 107 | + if (CI.isNoBuiltin() || !CI.getCalledFunction()) |
| 108 | + return; |
| 109 | + |
| 110 | + const std::string ScalarName = CI.getCalledFunction()->getName(); |
| 111 | + // Nothing to be done if the TLI thinks the function is not |
| 112 | + // vectorizable. |
| 113 | + if (!TLI.isFunctionVectorizable(ScalarName)) |
| 114 | + return; |
| 115 | + SmallVector<std::string, 8> Mappings; |
| 116 | + VFABI::getVectorVariantNames(CI, Mappings); |
| 117 | + Module *M = CI.getModule(); |
| 118 | + const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), |
| 119 | + Mappings.end()); |
| 120 | + // All VFs in the TLI are powers of 2. |
| 121 | + for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF; |
| 122 | + VF *= 2) { |
| 123 | + const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); |
| 124 | + if (!TLIName.empty()) { |
| 125 | + std::string MangledName = mangleTLIName(TLIName, CI, VF); |
| 126 | + if (!OriginalSetOfMappings.count(MangledName)) { |
| 127 | + Mappings.push_back(MangledName); |
| 128 | + ++NumCallInjected; |
| 129 | + } |
| 130 | + Function *VariantF = M->getFunction(TLIName); |
| 131 | + if (!VariantF) |
| 132 | + addVariantDeclaration(CI, VF, TLIName); |
| 133 | + } |
| 134 | + } |
| 135 | + |
| 136 | + VFABI::setVectorVariantNames(&CI, Mappings); |
| 137 | +} |
| 138 | + |
| 139 | +static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { |
| 140 | + for (auto &I : instructions(F)) |
| 141 | + if (auto CI = dyn_cast<CallInst>(&I)) |
| 142 | + addMappingsFromTLI(TLI, *CI); |
| 143 | + // Even if the pass adds IR attributes, the analyses are preserved. |
| 144 | + return false; |
| 145 | +} |
| 146 | + |
| 147 | +//////////////////////////////////////////////////////////////////////////////// |
| 148 | +// New pass manager implementation. |
| 149 | +//////////////////////////////////////////////////////////////////////////////// |
| 150 | +PreservedAnalyses InjectTLIMappings::run(Function &F, |
| 151 | + FunctionAnalysisManager &AM) { |
| 152 | + const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); |
| 153 | + runImpl(TLI, F); |
| 154 | + // Even if the pass adds IR attributes, the analyses are preserved. |
| 155 | + return PreservedAnalyses::all(); |
| 156 | +} |
| 157 | + |
| 158 | +//////////////////////////////////////////////////////////////////////////////// |
| 159 | +// Legacy PM Implementation. |
| 160 | +//////////////////////////////////////////////////////////////////////////////// |
| 161 | +bool InjectTLIMappingsLegacy::runOnFunction(Function &F) { |
| 162 | + const TargetLibraryInfo &TLI = |
| 163 | + getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); |
| 164 | + return runImpl(TLI, F); |
| 165 | +} |
| 166 | + |
| 167 | +void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { |
| 168 | + AU.setPreservesCFG(); |
| 169 | + AU.addRequired<TargetLibraryInfoWrapperPass>(); |
| 170 | + AU.addPreserved<TargetLibraryInfoWrapperPass>(); |
| 171 | +} |
| 172 | + |
| 173 | +//////////////////////////////////////////////////////////////////////////////// |
| 174 | +// Legacy Pass manager initialization |
| 175 | +//////////////////////////////////////////////////////////////////////////////// |
| 176 | +char InjectTLIMappingsLegacy::ID = 0; |
| 177 | + |
| 178 | +INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, |
| 179 | + "Inject TLI Mappings", false, false) |
| 180 | +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) |
| 181 | +INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", |
| 182 | + false, false) |
| 183 | + |
| 184 | +FunctionPass *llvm::createInjectTLIMappingsLegacyPass() { |
| 185 | + return new InjectTLIMappingsLegacy(); |
| 186 | +} |
0 commit comments