Merge branch 'main' into users/minglotus-6/typeprofrawformat

Author: mingmingl-llvm

.ci/generate-buildkite-pipeline-premerge

@@ -233,10 +233,7 @@ linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 
 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-# Temporary disable the windows job.
-# See https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840
-#windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
-windows_projects=""
+windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then

.ci/monolithic-windows.sh

@@ -38,6 +38,12 @@ targets="${2}"
 
 echo "--- cmake"
 pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt
+
+# The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
+# on fixing a build reliability issue on the build server, please
+# see https://github.com/llvm/llvm-project/pull/82393 and
+# https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40
+# for further information.
 cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
@@ -49,7 +55,10 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
-      -D MLIR_ENABLE_BINDINGS_PYTHON=ON
+      -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
+      -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO"
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

.github/new-prs-labeler.yml

@@ -822,6 +822,30 @@ backend:X86:
   - llvm/lib/TargetParser/X86*
   - llvm/utils/TableGen/X86*
 
+backend:PowerPC:
+  - llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC*
+  - llvm/include/llvm/BinaryFormat/XCOFF.h
+  - llvm/include/llvm/IR/IntrinsicsPowerPC.td
+  - llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
+  - llvm/lib/Target/PowerPC/**
+  - llvm/test/Analysis/**/PowerPC/**
+  - llvm/test/CodeGen/PowerPC/**
+  - llvm/test/CodeGen/MIR/PowerPC/**
+  - llvm/test/DebugInfo/XCOFF/**
+  - llvm/test/DebugInfo/PowerPC/**
+  - llvm/test/LTO/PowerPC/**
+  - llvm/test/MC/Disassembler/PowerPC/**
+  - llvm/test/MC/PowerPC/**
+  - llvm/test/MC/XCOFF/**
+  - llvm/test/Transforms/**/PowerPC/**
+  - clang/include/clang/Basic/BuiltinsPPC.*
+  - clang/lib/Basic/Targets/PPC.*
+  - clang/lib/CodeGen/Targets/PPC.cpp
+  - clang/lib/Driver/ToolChains/PPC*
+  - clang/lib/Driver/ToolChains/AIX*
+  - clang/lib/Driver/ToolChains/Arch/PPC.*
+  - clang/test/CodeGen/PowerPC/**
+
 third-party:unittests:
   - third-party/unittests/**
 

.github/workflows/approved-prs.yml

@@ -28,6 +28,7 @@ on:
       - 'polly/docs/**'
       - 'flang/docs/**'
       - 'flang/include/flang/Optimizer/Dialect/FIROps.td'
+      - '.github/workflows/docs.yml'
   pull_request:
     paths:
       - 'llvm/docs/**'
@@ -45,6 +46,7 @@ on:
       - 'polly/docs/**'
       - 'flang/docs/**'
       - 'flang/include/flang/Optimizer/Dialect/FIROps.td'
+      - '.github/workflows/docs.yml'
 
 jobs:
   check-docs-build:
@@ -162,8 +164,6 @@ jobs:
           TZ=UTC ninja -C polly-build docs-polly-html docs-polly-man
       - name: Build Flang docs
         if: steps.docs-changed-subprojects.outputs.flang_any_changed == 'true'
-        # TODO(boomanaiden154): Remove the SPHINX_WARNINGS_AS_ERRORS from the
-        # CMake invocation once the warnings in the flang docs build are fixed.
         run: |
-          cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON -DSPHINX_WARNINGS_AS_ERRORS=OFF ./llvm
-          TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
+          cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C flang-build docs-flang-html

.github/workflows/docs.yml

@@ -58,6 +58,10 @@ jobs:
   lit-tests:
     name: Lit Tests
     runs-on: ${{ matrix.os }}
+    container:
+      image: ${{(startsWith(matrix.os, 'ubuntu') && 'ghcr.io/llvm/ci-ubuntu-22.04:latest') || null}}
+      volumes:
+        - /mnt/:/mnt/
     strategy:
       fail-fast: false
       matrix:
@@ -77,6 +81,7 @@ jobs:
         with:
           python-version: ${{ inputs.python_version }}
       - name: Install Ninja
+        if: runner.os != 'Linux'
         uses: llvm/actions/install-ninja@main
       # actions/checkout deletes any existing files in the new git directory,
       # so this needs to either run before ccache-action or it has to use
@@ -108,8 +113,8 @@ jobs:
         run: |
           if [ "${{ runner.os }}" == "Linux" ]; then
             builddir="/mnt/build/"
-            sudo mkdir -p $builddir
-            sudo chown `whoami`:`whoami` $builddir
+            mkdir -p $builddir
+            extra_cmake_args="-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang"
           else
             builddir="$(pwd)"/build
           fi
@@ -123,6 +128,7 @@ jobs:
                 -DLLDB_INCLUDE_TESTS=OFF \
                 -DCMAKE_C_COMPILER_LAUNCHER=sccache \
                 -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+                $extra_cmake_args \
                 ${{ inputs.extra_cmake_args }}
           ninja -C "$builddir" '${{ inputs.build_target }}'
 

.github/workflows/llvm-project-tests.yml

@@ -28,6 +28,7 @@ jobs:
     name: Create a New Release
     runs-on: ubuntu-latest
     needs: validate-tag
+
     steps:
       - name: Install Dependencies
         run: |
@@ -40,8 +41,9 @@ jobs:
       - name: Create Release
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
         run: |
-          ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} create
+          ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} --user-token "$USER_TOKEN" create
   release-documentation:
     name: Build and Upload Release Documentation
     needs:

.github/workflows/release-tasks.yml

@@ -16,7 +16,7 @@ files needed to process intermediate representations and convert them into
 object files. Tools include an assembler, disassembler, bitcode analyzer, and
 bitcode optimizer.
 
-C-like languages use the [Clang](http://clang.llvm.org/) frontend. This
+C-like languages use the [Clang](https://clang.llvm.org/) frontend. This
 component compiles C, C++, Objective-C, and Objective-C++ code into LLVM bitcode
 -- and from there into object files, using LLVM.
 

README.md

@@ -16,6 +16,7 @@
 #include <map>
 #include <optional>
 #include <system_error>
+#include <unordered_map>
 
 namespace llvm {
 class raw_ostream;
@@ -111,6 +112,9 @@ class BoltAddressTranslation {
   /// addresses when aggregating profile
   bool enabledFor(llvm::object::ELFObjectFileBase *InputFile) const;
 
+  /// Save function and basic block hashes used for metadata dump.
+  void saveMetadata(BinaryContext &BC);
+
 private:
   /// Helper to update \p Map by inserting one or more BAT entries reflecting
   /// \p BB for function located at \p FuncAddress. At least one entry will be
@@ -140,6 +144,9 @@ class BoltAddressTranslation {
 
   std::map<uint64_t, MapTy> Maps;
 
+  using BBHashMap = std::unordered_map<uint32_t, size_t>;
+  std::unordered_map<uint64_t, std::pair<size_t, BBHashMap>> FuncHashes;
+
   /// Links outlined cold bocks to their original function
   std::map<uint64_t, uint64_t> ColdPartSource;
 

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -230,7 +230,7 @@ template <typename DebugVector, typename ListEntry, typename DebugAddressEntry>
 static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
                          DebugAddrWriter &AddrWriter, DWARFUnit &CU,
                          uint32_t &Index, const ListEntry BaseAddressx,
-                         const ListEntry OffsetPair, const ListEntry EndOfList,
+                         const ListEntry OffsetPair,
                          const std::function<void(uint32_t)> &Func) {
   if (Entries.size() < 2)
     return false;
@@ -241,7 +241,9 @@ static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
     const DebugAddressEntry &Entry = Entries[Index];
     if (Entry.LowPC == 0)
       break;
-    assert(Base <= Entry.LowPC && "Entry base is higher than low PC");
+    // In case rnglists or loclists are not sorted.
+    if (Base > Entry.LowPC)
+      break;
     uint32_t StartOffset = Entry.LowPC - Base;
     uint32_t EndOffset = Entry.HighPC - Base;
     if (encodeULEB128(EndOffset, TempBuffer) > 2)
@@ -266,8 +268,6 @@ static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
     encodeULEB128(OffsetEntry.EndOffset, OS);
     Func(OffsetEntry.Index);
   }
-  support::endian::write(OS, static_cast<uint8_t>(EndOfList),
-                         llvm::endianness::little);
   return true;
 }
 
@@ -276,19 +276,17 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
   std::lock_guard<std::mutex> Lock(WriterMutex);
 
   RangeEntries.push_back(CurrentOffset);
-  bool WrittenStartxLength = false;
   std::sort(
       Ranges.begin(), Ranges.end(),
       [](const DebugAddressRange &R1, const DebugAddressRange &R2) -> bool {
         return R1.LowPC < R2.LowPC;
       });
   for (unsigned I = 0; I < Ranges.size();) {
-    WrittenStartxLength = false;
     if (emitWithBase<DebugAddressRangesVector, dwarf::RnglistEntries,
-                     DebugAddressRange>(
-            *CUBodyStream, Ranges, *AddrWriter, *CU, I,
-            dwarf::DW_RLE_base_addressx, dwarf::DW_RLE_offset_pair,
-            dwarf::DW_RLE_end_of_list, [](uint32_t Index) -> void {}))
+                     DebugAddressRange>(*CUBodyStream, Ranges, *AddrWriter, *CU,
+                                        I, dwarf::DW_RLE_base_addressx,
+                                        dwarf::DW_RLE_offset_pair,
+                                        [](uint32_t Index) -> void {}))
       continue;
 
     const DebugAddressRange &Range = Ranges[I];
@@ -299,12 +297,11 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
     encodeULEB128(Index, *CUBodyStream);
     encodeULEB128(Range.HighPC - Range.LowPC, *CUBodyStream);
     ++I;
-    WrittenStartxLength = true;
   }
-  if (WrittenStartxLength)
-    support::endian::write(*CUBodyStream,
-                           static_cast<uint8_t>(dwarf::DW_RLE_end_of_list),
-                           llvm::endianness::little);
+
+  support::endian::write(*CUBodyStream,
+                         static_cast<uint8_t>(dwarf::DW_RLE_end_of_list),
+                         llvm::endianness::little);
   CurrentOffset = CUBodyBuffer->size();
   return RangeEntries.size() - 1;
 }
@@ -688,20 +685,18 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
   }
 
   std::vector<uint64_t> OffsetsArray;
-  bool WrittenStartxLength = false;
   auto writeExpression = [&](uint32_t Index) -> void {
     const DebugLocationEntry &Entry = LocList[Index];
     encodeULEB128(Entry.Expr.size(), LocBodyStream);
     LocBodyStream << StringRef(
         reinterpret_cast<const char *>(Entry.Expr.data()), Entry.Expr.size());
   };
   for (unsigned I = 0; I < LocList.size();) {
-    WrittenStartxLength = false;
     if (emitWithBase<DebugLocationsVector, dwarf::LoclistEntries,
-                     DebugLocationEntry>(
-            LocBodyStream, LocList, AddrWriter, CU, I,
-            dwarf::DW_LLE_base_addressx, dwarf::DW_LLE_offset_pair,
-            dwarf::DW_LLE_end_of_list, writeExpression))
+                     DebugLocationEntry>(LocBodyStream, LocList, AddrWriter, CU,
+                                         I, dwarf::DW_LLE_base_addressx,
+                                         dwarf::DW_LLE_offset_pair,
+                                         writeExpression))
       continue;
 
     const DebugLocationEntry &Entry = LocList[I];
@@ -713,13 +708,11 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
     encodeULEB128(Entry.HighPC - Entry.LowPC, LocBodyStream);
     writeExpression(I);
     ++I;
-    WrittenStartxLength = true;
   }
 
-  if (WrittenStartxLength)
-    support::endian::write(LocBodyStream,
-                           static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
-                           llvm::endianness::little);
+  support::endian::write(LocBodyStream,
+                         static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
+                         llvm::endianness::little);
 }
 
 void DebugLoclistWriter::addList(DIEBuilder &DIEBldr, DIE &Die,

bolt/lib/Core/DebugData.cpp

@@ -424,5 +424,20 @@ bool BoltAddressTranslation::enabledFor(
   }
   return false;
 }
+
+void BoltAddressTranslation::saveMetadata(BinaryContext &BC) {
+  for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) {
+    // We don't need a translation table if the body of the function hasn't
+    // changed
+    if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple()))
+      continue;
+    // Prepare function and block hashes
+    FuncHashes[BF.getAddress()].first = BF.computeHash();
+    BF.computeBlockHashes();
+    for (const BinaryBasicBlock &BB : BF)
+      FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(),
+                                                 BB.getHash());
+  }
+}
 } // namespace bolt
 } // namespace llvm