Prepare VCBE to handle standalone stack calls

State before:
The head of a function group is a kernel. Each function that is used in
different function groups (kernels and theirs call graphs) must be
copied. This rule must be applied even for stack calls. In other words,
stack calls are kernel-dependent functions. Such an approach was OK
until there was a need to compile standalone stack calls.

How it is now reorganized:
Each kernel is a head of a function group of type GROUP. Each stack call
(or indirect function) is a head of function group of type SUBGROUP
(legacy, we may consider removal of different types later). Stack calls
won't be copied. Function group passes are run on each function group
independently. This allows the creation of independent stack calls with
no need in a kernel. So, each SUBGROUP is independent function group and
compiles separately, but at the same time, its head may be referenced by
others function groups of type GROUP of SUBGROUP.

Author: Anton Sidorenko

IGC/VectorCompiler/igcdeps/src/TranslationInterface.cpp

@@ -193,6 +193,8 @@ static void adjustTransformationsAndOptimizations(vc::CompileOptions &Opts) {
     Opts.ForceDisableNonOverlappingRegionOpt = true;
   if (IGC_IS_FLAG_ENABLED(VCPassDebugToFinalizer))
     Opts.ForcePassDebugToFinalizer = true;
+  if (IGC_IS_FLAG_ENABLED(VCSaveStackCallLinkage))
+    Opts.SaveStackCallLinkage = true;
 }
 
 static void adjustDumpOptions(vc::CompileOptions &Opts) {

IGC/VectorCompiler/include/vc/Driver/Driver.h

@@ -97,6 +97,7 @@ struct CompileOptions {
 
   // from IGC_XXX env
   FunctionControl FCtrl = FunctionControl::Default;
+  bool SaveStackCallLinkage = false;
 };
 
 struct ExternalData {

IGC/VectorCompiler/include/vc/Support/BackendConfig.h

@@ -99,6 +99,11 @@ struct GenXBackendOptions {
   // max private stateless memory size per thread
   unsigned StatelessPrivateMemSize;
 
+  // Historically stack calls linkage is changed to internal in CMABI. This
+  // option allows saving the original linkage type for such functions. This is
+  // required for linking (e.g. invoke_simd).
+  bool SaveStackCallLinkage = false;
+
   GenXBackendOptions();
 };
 
@@ -217,6 +222,8 @@ class GenXBackendConfig : public ImmutablePass {
   }
 
   bool useBindlessBuffers() const { return Options.UseBindlessBuffers; }
+
+  bool saveStackCallLinkage() const { return Options.SaveStackCallLinkage; }
 };
 } // namespace llvm
 

IGC/VectorCompiler/lib/Driver/Driver.cpp

@@ -232,6 +232,8 @@ static GenXBackendOptions createBackendOptions(const vc::CompileOptions &Opts) {
   BackendOpts.WATable = Opts.WATable;
   BackendOpts.IsLargeGRFMode = Opts.IsLargeGRFMode;
   BackendOpts.UseBindlessBuffers = Opts.UseBindlessBuffers;
+  if (Opts.SaveStackCallLinkage)
+    BackendOpts.SaveStackCallLinkage = true;
   return BackendOpts;
 }
 

IGC/VectorCompiler/lib/GenXCodeGen/FunctionGroup.cpp

@@ -18,6 +18,7 @@ SPDX-License-Identifier: MIT
 //===----------------------------------------------------------------------===//
 
 #include "FunctionGroup.h"
+#include "vc/GenXOpts/Utils/KernelInfo.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/GenXIntrinsics/GenXMetadata.h"
 #include "llvm/IR/Dominators.h"
@@ -83,6 +84,14 @@ FunctionGroup *FunctionGroupAnalysis::getSubGroup(const Function *F) const {
   return getGroup(F, FGType::SUBGROUP);
 }
 
+FunctionGroup *FunctionGroupAnalysis::getAnyGroup(const Function *F) const {
+  auto *Group = getGroup(F, FGType::SUBGROUP);
+  if (!Group)
+    Group = getGroup(F, FGType::GROUP);
+  IGC_ASSERT_MESSAGE(Group, "Function isn't assigned to any function group");
+  return Group;
+}
+
 // getGroupForHead : get the FunctionGroup for which Function F is the
 // head, else 0
 FunctionGroup *FunctionGroupAnalysis::getGroupForHead(const Function *F) const {
@@ -98,7 +107,8 @@ FunctionGroup *FunctionGroupAnalysis::getGroupForHead(const Function *F) const {
 void FunctionGroupAnalysis::replaceFunction(Function *OldF, Function *NewF) {
   for (auto T : TypesToProcess) {
     auto OldFIt = GroupMap[T].find(OldF);
-    IGC_ASSERT(OldFIt != GroupMap[T].end());
+    if (OldFIt == GroupMap[T].end())
+      continue;
     FunctionGroup *FG = OldFIt->second;
     GroupMap[T].erase(OldFIt);
     GroupMap[T][NewF] = FG;
@@ -166,8 +176,12 @@ bool FunctionGroupAnalysis::buildGroup(CallGraph &Callees, Function *F,
   LLVM_DEBUG(dbgs() << "process function " << F->getName() << " from " << curGr
                     << ", type = " << Type << "\n");
   if (Visited.count(F) > 0) {
-    if (GroupMap[Type].count(F) > 0 && GroupMap[Type][F] != curGr &&
-        !F->hasFnAttribute(TypeToAttr(Type))) {
+    bool NeedCloning =
+        std::any_of(std::begin(TypesToProcess), std::end(TypesToProcess),
+                    [&GM = GroupMap, curGr, F](FGType CurType) {
+                      return GM[CurType].count(F) && GM[CurType][F] != curGr;
+                    });
+    if (NeedCloning && !F->hasFnAttribute(TypeToAttr(Type))) {
       ValueToValueMapTy VMap;
       Function *ClonedFunc = CloneFunction(F, VMap);
       LLVM_DEBUG(dbgs() << "Cloning: " << ClonedFunc->getName() << "\n");
@@ -181,16 +195,17 @@ bool FunctionGroupAnalysis::buildGroup(CallGraph &Callees, Function *F,
         if (GroupMap[Type][CI->getFunction()] == curGr)
           *u = ClonedFunc;
       }
-      for (auto T : TypesToProcess) {
-        if (T >= Type)
-          break;
-        addToFunctionGroup(getGroup(F, T), ClonedFunc, T);
-      }
       addToFunctionGroup(curGr, ClonedFunc, Type);
 
       for (auto &Callee : Callees[F]) {
         if (Callee == F)
           continue;
+        if (genx::requiresStackCall(Callee)) {
+          LLVM_DEBUG(dbgs()
+                     << "\tDo not process next callee " << Callee->getName()
+                     << " because it's a stack call\n");
+          continue;
+        }
         LLVM_DEBUG(dbgs() << "Next callee: " << Callee->getName() << "\n");
         result |= buildGroup(Callees, Callee, curGr, Type);
       }
@@ -209,6 +224,13 @@ bool FunctionGroupAnalysis::buildGroup(CallGraph &Callees, Function *F,
       addToFunctionGroup(curGr, F, Type);
     }
     for (auto &Callee : Callees[F]) {
+      if (genx::requiresStackCall(Callee)) {
+        LLVM_DEBUG(dbgs() << "\tDo not process next callee "
+                          << Callee->getName()
+                          << " because it's a stack call\n");
+        continue;
+      }
+
       LLVM_DEBUG(dbgs() << "Next callee: " << Callee->getName() << "\n");
       result |= buildGroup(Callees, Callee, curGr, Type);
     }
@@ -418,7 +440,7 @@ bool FGPassManager::runFGPassSequence(unsigned &Pass) {
   bool Changed = false;
 
   Changed |= doFGInitialization(BeginPass, Pass, FGA);
-  for (auto *FG : FGA)
+  for (auto *FG : FGA.AllGroups())
     Changed |= runPassesOnFunctionGroup(BeginPass, Pass, *FG);
   Changed |= doFGFinalization(BeginPass, Pass, FGA);
 

IGC/VectorCompiler/lib/GenXCodeGen/FunctionGroup.h

@@ -27,6 +27,8 @@ SPDX-License-Identifier: MIT
 #ifndef FUNCTIONGROUP_H
 #define FUNCTIONGROUP_H
 
+#include "vc/GenXOpts/Utils/KernelInfo.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ValueHandle.h"
@@ -41,6 +43,18 @@ class FunctionGroupAnalysis;
 class LLVMContext;
 class PMStack;
 
+namespace genx {
+namespace fg {
+inline bool isGroupHead(const Function &F) { return genx::isKernel(&F); }
+inline bool isSubGroupHead(const Function &F) {
+  return genx::isReferencedIndirectly(&F) || genx::requiresStackCall(&F);
+}
+inline bool isHead(const Function &F) {
+  return isGroupHead(F) || isSubGroupHead(F);
+}
+} // namespace fg
+} // namespace genx
+
 //----------------------------------------------------------------------
 // FunctionGroup : a group of Functions
 //
@@ -50,6 +64,9 @@ class FunctionGroup {
   // Elements are asserting value handles, so we spot when a Function
   // in the group gets destroyed too early.
   SmallVector<AssertingVH<Function>, 8> Functions;
+  // FunctionGroup can call the head function of another FunctionGroups with
+  // SUBGROUP type.
+  SetVector<const FunctionGroup *> Subgroups;
 
 public:
   FunctionGroup(FunctionGroupAnalysis *FGA) : FGA(FGA) {}
@@ -83,6 +100,25 @@ class FunctionGroup {
   StringRef getName() { return getHead()->getName(); }
   LLVMContext &getContext() { return getHead()->getContext(); }
   Module *getModule() { return getHead()->getParent(); }
+  void addSubgroup(FunctionGroup *FG) {
+    IGC_ASSERT(FG);
+    IGC_ASSERT(FG->getHead());
+    IGC_ASSERT_MESSAGE(genx::fg::isSubGroupHead(*FG->getHead()),
+                       "Provided function group has incorrect type");
+    Subgroups.insert(FG);
+  }
+  using subgroup_iterator = decltype(Subgroups)::iterator;
+  using const_subgroup_iterator = decltype(Subgroups)::const_iterator;
+  subgroup_iterator begin_subgroup() { return Subgroups.begin(); }
+  subgroup_iterator end_subgroup() { return Subgroups.end(); }
+  const_subgroup_iterator begin_subgroup() const { return Subgroups.begin(); }
+  const_subgroup_iterator end_subgroup() const { return Subgroups.end(); }
+  iterator_range<subgroup_iterator> subgroups() {
+    return make_range(begin_subgroup(), end_subgroup());
+  }
+  iterator_range<const_subgroup_iterator> subgroups() const {
+    return make_range(begin_subgroup(), end_subgroup());
+  }
 };
 
 //----------------------------------------------------------------------
@@ -153,6 +189,8 @@ class FunctionGroupAnalysis : public ModulePass {
   FunctionGroup *getGroup(const Function *F, FGType Type) const;
   FunctionGroup *getGroup(const Function *F) const;
   FunctionGroup *getSubGroup(const Function *F) const;
+  // get group or subgroup depending on where the function is.
+  FunctionGroup *getAnyGroup(const Function *F) const;
   // getGroupForHead : get the FunctionGroup for which Function F is the
   // head, else 0
   FunctionGroup *getGroupForHead(const Function *F) const;
@@ -161,10 +199,40 @@ class FunctionGroupAnalysis : public ModulePass {
   // iterator for FunctionGroups in the analysis
   typedef SmallVectorImpl<FunctionGroup *>::iterator iterator;
   typedef SmallVectorImpl<FunctionGroup *>::const_iterator const_iterator;
+  using all_iterator = concat_iterator<FunctionGroup *, iterator, iterator>;
+  using const_all_iterator =
+      concat_iterator<FunctionGroup *const, const_iterator, const_iterator>;
   iterator begin() { return iterator(Groups.begin()); }
   iterator end() { return iterator(Groups.end()); }
   const_iterator begin() const { return const_iterator(Groups.begin()); }
   const_iterator end() const { return const_iterator(Groups.end()); }
+
+  iterator subgroup_begin() { return iterator(NonMainGroups.begin()); }
+  iterator subgroup_end() { return iterator(NonMainGroups.end()); }
+  const_iterator subgroup_begin() const {
+    return const_iterator(NonMainGroups.begin());
+  }
+  const_iterator subgroup_end() const {
+    return const_iterator(NonMainGroups.end());
+  }
+  iterator_range<iterator> subgroups() {
+    return make_range(subgroup_begin(), subgroup_end());
+  }
+  iterator_range<const_iterator> subgroups() const {
+    return make_range(subgroup_begin(), subgroup_end());
+  }
+
+  iterator_range<all_iterator> AllGroups() {
+    return concat<FunctionGroup *>(
+        make_range(begin(), end()),
+        make_range(subgroup_begin(), subgroup_end()));
+  }
+  iterator_range<const_all_iterator> AllGroups() const {
+    return concat<FunctionGroup *const>(
+        make_range(begin(), end()),
+        make_range(subgroup_begin(), subgroup_end()));
+  }
+
   size_t size() const { return Groups.size(); }
   // addToFunctionGroup : add Function F to FunctionGroup FG
   // Using this (rather than calling push_back directly on the FunctionGroup)

IGC/VectorCompiler/lib/GenXCodeGen/GenXArgIndirection.cpp

@@ -1095,9 +1095,14 @@ void SubroutineArg::gatherBalesToModify(Alignment Align)
       auto User = cast<Instruction>(ui->getUser());
       if (auto CI = dyn_cast<CallInst>(User)) {
         Function *CF = CI->getCalledFunction();
-        if (!GenXIntrinsic::isAnyNonTrivialIntrinsic(CF)) {
+        if (!GenXIntrinsic::isAnyNonTrivialIntrinsic(CF) &&
+            !genx::requiresStackCall(CF)) {
           // Non-intrinsic call. Ignore. (A call site using an arg being
           // indirected gets handled differently.)
+          // Cannot indirect if there is a stack call. Do not ignore stack
+          // calls here and add them to BalesToModify. In checkIndirectBale
+          // report that such bale cannot be indirected. This method is
+          // confusing, must be improved.
           continue;
         }
       } else {
@@ -1165,6 +1170,13 @@ bool GenXArgIndirection::checkIndirectBale(Bale *B, LiveRange *ArgLR,
                           << "\n\tintrinsic with raw return value\n");
         return false;
       }
+    } else if (auto *CI = dyn_cast<CallInst>(MainInst->Inst)) {
+      auto *Callee = CI->getCalledFunction();
+      IGC_ASSERT_MESSAGE(genx::requiresStackCall(Callee),
+                         "Expect a stack call to stop indirection. See "
+                         "SubroutineArg::gatherBalesToModify");
+      LLVM_DEBUG(dbgs() << *CI << "\n\tcalls stack call function\n");
+      return false;
     }
   }
   // Check the rdregion(s) and wrregion.

IGC/VectorCompiler/lib/GenXCodeGen/GenXCategory.cpp

@@ -800,13 +800,14 @@ CategoryAndAlignment GenXCategory::getCategoryAndAlignmentForDef(Value *V) const
   if (V->getType()->getScalarType()->getPrimitiveSizeInBits() == 1)
     return RegCategory::PREDICATE;
   if (Argument *Arg = dyn_cast<Argument>(V)) {
+    auto *F = Arg->getParent();
     // This is a function Argument.
     if (!InFGHead) {
       // It is an arg in a subroutine.  Get the category from the corresponding
       // arg at some call site.  (We should not have disagreement among the
       // call sites and the function arg, since whichever one gets a category
       // first forces the category of all the others.)
-      return getCategoryForCallArg(Arg->getParent(), Arg->getArgNo());
+      return getCategoryForCallArg(F, Arg->getArgNo());
     }
     unsigned ArgNo = Arg->getArgNo();
     if (KM.getNumArgs() > ArgNo) {
@@ -819,7 +820,12 @@ CategoryAndAlignment GenXCategory::getCategoryAndAlignmentForDef(Value *V) const
     // determine the category. This is the fallback for compatibility with
     // hand coded LLVM IR from before this metadata was added. (If we only
     // had to cope with non-kernel functions, we could just return GENERAL.)
-    return RegCategory::NONE;
+    // FIXME: temporary fix for stack calls. We need to figure out how to
+    // determine arguments category if it cannot be deduced from the arg uses.
+    // * calls from another function groups might help (but we do not have
+    // liveness -> category for them). What about standalone stack calls?
+    IGC_ASSERT(genx::requiresStackCall(F));
+    return getCategoryForCallArg(F, Arg->getArgNo());
   }
   // The def is a phi-instruction.
   if (PHINode *Phi = dyn_cast<PHINode>(V)) {
@@ -1070,22 +1076,19 @@ unsigned GenXCategory::getCategoryForCallArg(Function *Callee, unsigned ArgNo) c
       return Cat;
   }
   // Then try the arg at each call site.
-  bool UseUndef = true;
   for (auto *U: Callee->users()) {
     if (auto *CI = checkFunctionCall(U, Callee)) {
       auto ArgV = CI->getArgOperand(ArgNo);
-      if (!isa<UndefValue>(ArgV)) {
-        UseUndef = false;
         if (auto LR = Liveness->getLiveRangeOrNull(ArgV)) {
           unsigned Cat = LR->getCategory();
           if (Cat != RegCategory::NONE)
             return Cat;
         }
-      }
     }
   }
-  // special case handling to break deadlock when all uses are undef,
-  // force the argument to be GENERAL
-  return(UseUndef ? RegCategory::GENERAL : RegCategory::NONE);
+  // special case handling to break deadlock when all uses are undef or stack
+  // call arg category cannot be deduced from the uses in the function, force
+  // the argument to be GENERAL
+  return EnforceCategoryPromotion ? RegCategory::GENERAL : RegCategory::NONE;
 }