Merge from 'main' to 'sycl-web' (5 commits)

  CONFLICT (content): Merge conflict in llvm/include/llvm/CodeGen/MachinePassRegistry.def

Author: mdfazlay

llvm/include/llvm/CodeGen/CodeGenPassBuilder.h

@@ -28,6 +28,7 @@
 #include "llvm/CodeGen/ExpandReductions.h"
 #include "llvm/CodeGen/FPBuiltinFnSelection.h"
 #include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/InterleavedAccess.h"
 #include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/JMCInstrumenter.h"

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h

@@ -242,6 +242,13 @@ enum {
   /// - OpIdx(ULEB128) - Operand index
   /// - Val(8) Expected integer
   GIM_CheckConstantInt,
+
+  /// Check the operand is a specific 8-bit signed integer
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Val(1) Expected integer
+  GIM_CheckConstantInt8,
+
   /// Check the operand is a specific literal integer (i.e. MO.isImm() or
   /// MO.isCImm() is true).
   /// - InsnID(ULEB128) - Instruction ID
@@ -399,6 +406,12 @@ enum {
   /// - TempRegFlags(2) - The register flags to set
   GIR_AddTempRegister,
 
+  /// Add a temporary register to the specified instruction without
+  /// setting any flags.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - TempRegID(ULEB128) - The temporary register ID to add
+  GIR_AddSimpleTempRegister,
+
   /// Add a temporary register to the specified instruction
   /// - InsnID(ULEB128) - Instruction ID to modify
   /// - TempRegID(ULEB128) - The temporary register ID to add
@@ -411,6 +424,11 @@ enum {
   /// - Imm(8) - The immediate to add
   GIR_AddImm,
 
+  /// Add signed 8 bit immediate to the specified instruction
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - Imm(1) - The immediate to add
+  GIR_AddImm8,
+
   /// Add an CImm to the specified instruction
   /// - InsnID(ULEB128) - Instruction ID to modify
   /// - Ty(1) - Type of the constant immediate.

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h

@@ -784,10 +784,13 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
-    case GIM_CheckConstantInt: {
+    case GIM_CheckConstantInt:
+    case GIM_CheckConstantInt8: {
+      const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
+
       uint64_t InsnID = readULEB();
       uint64_t OpIdx = readULEB();
-      uint64_t Value = readU64();
+      uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -1157,11 +1160,14 @@ bool GIMatchTableExecutor::executeMatchTable(
       MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
       break;
     }
+    case GIR_AddSimpleTempRegister:
     case GIR_AddTempRegister:
     case GIR_AddTempSubRegister: {
       uint64_t InsnID = readULEB();
       uint64_t TempRegID = readULEB();
-      uint16_t TempRegFlags = readU16();
+      uint16_t TempRegFlags = 0;
+      if (MatcherOpcode != GIR_AddSimpleTempRegister)
+        TempRegFlags = readU16();
       uint16_t SubReg = 0;
       if (MatcherOpcode == GIR_AddTempSubRegister)
         SubReg = readU16();
@@ -1179,9 +1185,11 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
+    case GIR_AddImm8:
     case GIR_AddImm: {
+      const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
       uint64_t InsnID = readULEB();
-      uint64_t Imm = readU64();
+      uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       OutMIs[InsnID].addImm(Imm);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),

llvm/include/llvm/CodeGen/IndirectBrExpand.h

@@ -0,0 +1,28 @@
+//===- llvm/CodeGen/IndirectBrExpand.h -------------------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_INDIRECTBREXPAND_H
+#define LLVM_CODEGEN_INDIRECTBREXPAND_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class TargetMachine;
+
+class IndirectBrExpandPass : public PassInfoMixin<IndirectBrExpandPass> {
+  const TargetMachine *TM;
+
+public:
+  IndirectBrExpandPass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_INDIRECTBREXPAND_H

llvm/include/llvm/CodeGen/MachinePassRegistry.def

@@ -49,6 +49,7 @@ FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ())
 FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
 FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ())
 FUNCTION_PASS("fpbuiltin-fn-selection", FPBuiltinFnSelectionPass, ())
+FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, (TM))
 FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM))
 FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass, (TM))
 FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ())
@@ -132,7 +133,6 @@ DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ())
 DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ())
 DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ())
 DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
-DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
 DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
 DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ())
 #undef DUMMY_FUNCTION_PASS

llvm/include/llvm/InitializePasses.h

@@ -131,7 +131,7 @@ void initializeIVUsersWrapperPassPass(PassRegistry&);
 void initializeIfConverterPass(PassRegistry&);
 void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry&);
 void initializeImplicitNullChecksPass(PassRegistry&);
-void initializeIndirectBrExpandPassPass(PassRegistry&);
+void initializeIndirectBrExpandLegacyPassPass(PassRegistry &);
 void initializeInferAddressSpacesPass(PassRegistry&);
 void initializeInstSimplifyLegacyPassPass(PassRegistry &);
 void initializeInstructionCombiningPassPass(PassRegistry&);

llvm/lib/CodeGen/CodeGen.cpp

@@ -54,7 +54,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeHardwareLoopsLegacyPass(Registry);
   initializeIfConverterPass(Registry);
   initializeImplicitNullChecksPass(Registry);
-  initializeIndirectBrExpandPassPass(Registry);
+  initializeIndirectBrExpandLegacyPassPass(Registry);
   initializeInterleavedLoadCombinePass(Registry);
   initializeInterleavedAccessPass(Registry);
   initializeJMCInstrumenterPass(Registry);

llvm/lib/CodeGen/IndirectBrExpandPass.cpp

@@ -29,6 +29,7 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
@@ -48,14 +49,12 @@ using namespace llvm;
 
 namespace {
 
-class IndirectBrExpandPass : public FunctionPass {
-  const TargetLowering *TLI = nullptr;
-
+class IndirectBrExpandLegacyPass : public FunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
 
-  IndirectBrExpandPass() : FunctionPass(ID) {
-    initializeIndirectBrExpandPassPass(*PassRegistry::getPassRegistry());
+  IndirectBrExpandLegacyPass() : FunctionPass(ID) {
+    initializeIndirectBrExpandLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -67,33 +66,41 @@ class IndirectBrExpandPass : public FunctionPass {
 
 } // end anonymous namespace
 
-char IndirectBrExpandPass::ID = 0;
+static bool runImpl(Function &F, const TargetLowering *TLI,
+                    DomTreeUpdater *DTU);
+
+PreservedAnalyses IndirectBrExpandPass::run(Function &F,
+                                            FunctionAnalysisManager &FAM) {
+  auto *STI = TM->getSubtargetImpl(F);
+  if (!STI->enableIndirectBrExpand())
+    return PreservedAnalyses::all();
+
+  auto *TLI = STI->getTargetLowering();
+  auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
+  bool Changed = runImpl(F, TLI, DT ? &DTU : nullptr);
+  if (!Changed)
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  return PA;
+}
+
+char IndirectBrExpandLegacyPass::ID = 0;
 
-INITIALIZE_PASS_BEGIN(IndirectBrExpandPass, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(IndirectBrExpandLegacyPass, DEBUG_TYPE,
                       "Expand indirectbr instructions", false, false)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_END(IndirectBrExpandPass, DEBUG_TYPE,
+INITIALIZE_PASS_END(IndirectBrExpandLegacyPass, DEBUG_TYPE,
                     "Expand indirectbr instructions", false, false)
 
 FunctionPass *llvm::createIndirectBrExpandPass() {
-  return new IndirectBrExpandPass();
+  return new IndirectBrExpandLegacyPass();
 }
 
-bool IndirectBrExpandPass::runOnFunction(Function &F) {
+bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU) {
   auto &DL = F.getParent()->getDataLayout();
-  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
-  if (!TPC)
-    return false;
-
-  auto &TM = TPC->getTM<TargetMachine>();
-  auto &STI = *TM.getSubtargetImpl(F);
-  if (!STI.enableIndirectBrExpand())
-    return false;
-  TLI = STI.getTargetLowering();
-
-  std::optional<DomTreeUpdater> DTU;
-  if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
-    DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy);
 
   SmallVector<IndirectBrInst *, 1> IndirectBrs;
 
@@ -268,3 +275,21 @@ bool IndirectBrExpandPass::runOnFunction(Function &F) {
 
   return true;
 }
+
+bool IndirectBrExpandLegacyPass::runOnFunction(Function &F) {
+  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
+  if (!TPC)
+    return false;
+
+  auto &TM = TPC->getTM<TargetMachine>();
+  auto &STI = *TM.getSubtargetImpl(F);
+  if (!STI.enableIndirectBrExpand())
+    return false;
+  auto *TLI = STI.getTargetLowering();
+
+  std::optional<DomTreeUpdater> DTU;
+  if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
+    DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy);
+
+  return runImpl(F, TLI, DTU ? &*DTU : nullptr);
+}

llvm/lib/CodeGen/RegisterCoalescer.cpp

@@ -1201,6 +1201,8 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
                       << printMBBReference(MBB) << '\t' << CopyMI);
   }
 
+  const bool IsUndefCopy = CopyMI.getOperand(1).isUndef();
+
   // Remove CopyMI.
   // Note: This is fine to remove the copy before updating the live-ranges.
   // While updating the live-ranges, we only look at slot indices and
@@ -1214,6 +1216,19 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
   LIS->pruneValue(*static_cast<LiveRange *>(&IntB), CopyIdx.getRegSlot(),
                   &EndPoints);
   BValNo->markUnused();
+
+  if (IsUndefCopy) {
+    // We're introducing an undef phi def, and need to set undef on any users of
+    // the previously local def to avoid artifically extending the lifetime
+    // through the block.
+    for (MachineOperand &MO : MRI->use_nodbg_operands(IntB.reg())) {
+      const MachineInstr &MI = *MO.getParent();
+      SlotIndex UseIdx = LIS->getInstructionIndex(MI);
+      if (!IntB.liveAt(UseIdx))
+        MO.setIsUndef(true);
+    }
+  }
+
   // Extend IntB to the EndPoints of its original live interval.
   LIS->extendToIndices(IntB, EndPoints);
 

llvm/lib/Passes/PassBuilder.cpp

@@ -78,6 +78,7 @@
 #include "llvm/CodeGen/ExpandLargeFpConvert.h"
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/HardwareLoops.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/InterleavedAccess.h"
 #include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/JMCInstrumenter.h"

llvm/lib/Passes/PassRegistry.def

@@ -330,6 +330,7 @@ FUNCTION_PASS("guard-widening", GuardWideningPass())
 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
 FUNCTION_PASS("gvn-sink", GVNSinkPass())
 FUNCTION_PASS("helloworld", HelloWorldPass())
+FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass(TM))
 FUNCTION_PASS("infer-address-spaces", InferAddressSpacesPass())
 FUNCTION_PASS("infer-alignment", InferAlignmentPass())
 FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings())

llvm/test/CodeGen/X86/coalescer-partial-redundancy-clear-dead-flag-undef-copy.mir

@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=x86_64-pc-linux-gnu -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+
+# Check for "Live range continues after dead def flag".
+
+# There are 2 copies of undef, but the registers also appear to be
+# live due to block live outs, and thus were not deleted as
+# eliminateUndefCopy only considered the live range, and not the undef
+# flag.
+#
+# removePartialRedundancy would move the COPY undef %0 in bb.1 to
+# bb.0.  The live range of %1 would then be extended to be live out of
+# %bb.1 for the backedge phi. This would then fail the verifier, since
+# the dead flag was no longer valid. This was fixed by directly
+# considering the undef flag to avoid considering this special case.
+
+---
+name: partial_redundancy_coalesce_undef_copy_live_out
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: partial_redundancy_coalesce_undef_copy_live_out
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[XOR32ri:%[0-9]+]]:gr32 = XOR32ri undef [[XOR32ri]], 1, implicit-def dead $eflags
+  ; CHECK-NEXT:   dead [[MOV32rr:%[0-9]+]]:gr32 = MOV32rr [[COPY]]
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  bb.0:
+    liveins: $rdi
+
+    %0:gr32 = COPY $rdi
+
+  bb.1:
+    %1:gr32 = COPY undef %0
+    dead %1:gr32 = XOR32ri %1, 1, implicit-def dead $eflags
+    dead %2:gr32 = MOV32rr killed %0
+    %0:gr32 = COPY killed undef %1
+    JMP_1 %bb.1
+
+...