Merge remote-tracking branch 'origin/main' into android-mblen-wctrans

Author: rprichard

.ci/generate_test_report.py

@@ -28,11 +28,16 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
   ccache --clear
 fi
 
-function show-stats {
+function at-exit {
   mkdir -p artifacts
   ccache --print-stats > artifacts/ccache_stats.txt
+
+  # If building fails there will be no results files.
+  shopt -s nullglob
+  python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \
+    "linux-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
 }
-trap show-stats EXIT
+trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
@@ -42,6 +47,7 @@ lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-outpu
 echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
+pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \

.ci/monolithic-linux.sh

@@ -27,17 +27,23 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
 fi
 
 sccache --zero-stats
-function show-stats {
+function at-exit {
   mkdir -p artifacts
   sccache --show-stats >> artifacts/sccache_stats.txt
+
+  # If building fails there will be no results files.
+  shopt -s nullglob
+  python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \
+    "windows-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
 }
-trap show-stats EXIT
+trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
 
 echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
+pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
 
 # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
 # on fixing a build reliability issue on the build server, please

.ci/monolithic-windows.sh

@@ -0,0 +1 @@
+junitparser==3.2.0

.ci/requirements.txt

@@ -587,6 +587,12 @@ llvm:ir:
   - llvm/docs/LangRef.rst
   - llvm/unittests/IR/**
 
+llvm:SandboxIR:
+  - llvm/lib/SandboxIR/**
+  - llvm/include/llvm/SandboxIR/**
+  - llvm/docs/SandboxIR.md
+  - llvm/unittests/SandboxIR/**
+
 llvm:analysis:
   - llvm/lib/Analysis/**
   - llvm/include/llvm/Analysis/**

.github/new-prs-labeler.yml

@@ -45,12 +45,11 @@ env:
   LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-19"
   CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
 
-
 jobs:
   stage1:
     if: github.repository_owner == 'llvm'
-    runs-on: libcxx-runners-set
-    container: ghcr.io/libcxx/actions-builder:testing-2024-09-21
+    runs-on: libcxx-self-hosted-linux
+    container: ghcr.io/llvm/libcxx-linux-builder:0fd6f684b9c84c32d6cbfd9742402e788b2879f1
     continue-on-error: false
     strategy:
       fail-fast: false
@@ -86,8 +85,8 @@ jobs:
             **/crash_diagnostics/*
   stage2:
     if: github.repository_owner == 'llvm'
-    runs-on: libcxx-runners-set
-    container: ghcr.io/libcxx/actions-builder:testing-2024-09-21
+    runs-on: libcxx-self-hosted-linux
+    container: ghcr.io/llvm/libcxx-linux-builder:0fd6f684b9c84c32d6cbfd9742402e788b2879f1
     needs: [ stage1 ]
     continue-on-error: false
     strategy:
@@ -161,21 +160,21 @@ jobs:
           'generic-static',
           'bootstrapping-build'
         ]
-        machine: [ 'libcxx-runners-set' ]
+        machine: [ 'libcxx-self-hosted-linux' ]
         include:
         - config: 'generic-cxx26'
-          machine: libcxx-runners-set
+          machine: libcxx-self-hosted-linux
         - config: 'generic-asan'
-          machine: libcxx-runners-set
+          machine: libcxx-self-hosted-linux
         - config: 'generic-tsan'
-          machine: libcxx-runners-set
+          machine: libcxx-self-hosted-linux
         - config: 'generic-ubsan'
-          machine: libcxx-runners-set
+          machine: libcxx-self-hosted-linux
         # Use a larger machine for MSAN to avoid timeout and memory allocation issues.
         - config: 'generic-msan'
-          machine: libcxx-runners-set
+          machine: libcxx-self-hosted-linux
     runs-on: ${{ matrix.machine }}
-    container: ghcr.io/libcxx/actions-builder:testing-2024-09-21
+    container: ghcr.io/llvm/libcxx-linux-builder:0fd6f684b9c84c32d6cbfd9742402e788b2879f1
     steps:
       - uses: actions/checkout@v4
       - name: ${{ matrix.config }}
@@ -202,13 +201,13 @@ jobs:
       matrix:
         include:
         - config: generic-cxx03
-          os: macos-latest
+          os: macos-15
         - config: generic-cxx23
-          os: macos-latest
+          os: macos-15
         - config: generic-modules
-          os: macos-latest
+          os: macos-15
         - config: apple-configuration
-          os: macos-latest
+          os: macos-15
         - config: apple-system
           os: macos-13
         - config: apple-system-hardened

.github/workflows/libcxx-build-and-test.yaml

@@ -0,0 +1,71 @@
+# This file defines an action that builds the various Docker images used to run
+# libc++ CI whenever modifications to those Docker files are pushed to `main`.
+#
+# The images are pushed to the LLVM package registry at https://github.com/orgs/llvm/packages
+# and tagged appropriately. The selection of which Docker image version is used by the libc++
+# CI nodes at any given point is controlled from the workflow files themselves.
+
+name: Build Docker images for libc++ CI
+
+permissions:
+  contents: read
+  packages: write
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'libcxx/utils/ci/**'
+      - '.github/workflows/libcxx-build-containers.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'libcxx/utils/ci/**'
+      - '.github/workflows/libcxx-build-containers.yml'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'llvm'
+    permissions:
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build the Linux builder image
+      working-directory: libcxx/utils/ci
+      run: docker compose build actions-builder
+      env:
+        TAG: ${{ github.sha }}
+
+    # - name: Build the Android builder image
+    #   working-directory: libcxx/utils/ci
+    #   run: docker compose build android-buildkite-builder
+    #   env:
+    #     TAG: ${{ github.sha }}
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Push the Linux builder image
+      if: github.event_name == 'push'
+      working-directory: libcxx/utils/ci
+      run: |
+        docker compose push actions-builder
+      env:
+        TAG: ${{ github.sha }}
+
+    # - name: Push the Android builder image
+    #   if: github.event_name == 'push'
+    #   working-directory: libcxx/utils/ci
+    #   run: |
+    #     docker compose push android-buildkite-builder
+    #   env:
+    #     TAG: ${{ github.sha }}

.github/workflows/libcxx-build-containers.yml

@@ -43,17 +43,14 @@ class CFIReaderWriter {
 
   /// Generate .eh_frame_hdr from old and new .eh_frame sections.
   ///
-  /// Take FDEs from the \p NewEHFrame unless their initial_pc is listed
-  /// in \p FailedAddresses. All other entries are taken from the
+  /// Take FDEs from the \p NewEHFrame. All other entries are taken from the
   /// \p OldEHFrame.
   ///
   /// \p EHFrameHeaderAddress specifies location of .eh_frame_hdr,
   /// and is required for relative addressing used in the section.
-  std::vector<char>
-  generateEHFrameHeader(const DWARFDebugFrame &OldEHFrame,
-                        const DWARFDebugFrame &NewEHFrame,
-                        uint64_t EHFrameHeaderAddress,
-                        std::vector<uint64_t> &FailedAddresses) const;
+  std::vector<char> generateEHFrameHeader(const DWARFDebugFrame &OldEHFrame,
+                                          const DWARFDebugFrame &NewEHFrame,
+                                          uint64_t EHFrameHeaderAddress) const;
 
   using FDEsMap = std::map<uint64_t, const dwarf::FDE *>;
 

bolt/include/bolt/Core/Exceptions.h

@@ -556,14 +556,6 @@ class RewriteInstance {
     return ErrOrSection ? &ErrOrSection.get() : nullptr;
   }
 
-  /// Keep track of functions we fail to write in the binary. We need to avoid
-  /// rewriting CFI info for these functions.
-  std::vector<uint64_t> FailedAddresses;
-
-  /// Keep track of which functions didn't fit in their original space in the
-  /// last emission, so that we may either decide to split or not optimize them.
-  std::set<uint64_t> LargeFunctions;
-
   /// Section header string table.
   StringTableBuilder SHStrTab;
 

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -906,17 +906,6 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
   if (Sites.empty())
     return;
 
-  // Calculate callsite table size. Size of each callsite entry is:
-  //
-  //  sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
-  //
-  // or
-  //
-  //  sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
-  uint64_t CallSiteTableLength = llvm::size(Sites) * 4 * 3;
-  for (const auto &FragmentCallSite : Sites)
-    CallSiteTableLength += getULEB128Size(FragmentCallSite.second.Action);
-
   Streamer.switchSection(BC.MOFI->getLSDASection());
 
   const unsigned TTypeEncoding = BF.getLSDATypeEncoding();
@@ -975,36 +964,24 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
 
   Streamer.emitIntValue(TTypeEncoding, 1); // TType format
 
-  // See the comment in EHStreamer::emitExceptionTable() on to use
-  // uleb128 encoding (which can use variable number of bytes to encode the same
-  // value) to ensure type info table is properly aligned at 4 bytes without
-  // iteratively fixing sizes of the tables.
-  unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength);
-  unsigned TTypeBaseOffset =
-      sizeof(int8_t) +                 // Call site format
-      CallSiteTableLengthSize +        // Call site table length size
-      CallSiteTableLength +            // Call site table length
-      BF.getLSDAActionTable().size() + // Actions table size
-      BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size
-  unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset);
-  unsigned TotalSize = sizeof(int8_t) +      // LPStart format
-                       sizeof(int8_t) +      // TType format
-                       TTypeBaseOffsetSize + // TType base offset size
-                       TTypeBaseOffset;      // TType base offset
-  unsigned SizeAlign = (4 - TotalSize) & 3;
-
-  if (TTypeEncoding != dwarf::DW_EH_PE_omit)
-    // Account for any extra padding that will be added to the call site table
-    // length.
-    Streamer.emitULEB128IntValue(TTypeBaseOffset,
-                                 /*PadTo=*/TTypeBaseOffsetSize + SizeAlign);
+  MCSymbol *TTBaseLabel = nullptr;
+  if (TTypeEncoding != dwarf::DW_EH_PE_omit) {
+    TTBaseLabel = BC.Ctx->createTempSymbol("TTBase");
+    MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol("TTBaseRef");
+    Streamer.emitAbsoluteSymbolDiffAsULEB128(TTBaseLabel, TTBaseRefLabel);
+    Streamer.emitLabel(TTBaseRefLabel);
+  }
 
   // Emit the landing pad call site table. We use signed data4 since we can emit
   // a landing pad in a different part of the split function that could appear
   // earlier in the address space than LPStart.
   Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
-  Streamer.emitULEB128IntValue(CallSiteTableLength);
 
+  MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol("CSTStart");
+  MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol("CSTEnd");
+  Streamer.emitAbsoluteSymbolDiffAsULEB128(CSTEndLabel, CSTStartLabel);
+
+  Streamer.emitLabel(CSTStartLabel);
   for (const auto &FragmentCallSite : Sites) {
     const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;
     const MCSymbol *BeginLabel = CallSite.Start;
@@ -1020,6 +997,7 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
     emitLandingPad(CallSite.LP);
     Streamer.emitULEB128IntValue(CallSite.Action);
   }
+  Streamer.emitLabel(CSTEndLabel);
 
   // Write out action, type, and type index tables at the end.
   //
@@ -1038,6 +1016,8 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
   assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
          "indirect type table size mismatch");
 
+  Streamer.emitValueToAlignment(Align(TTypeAlignment));
+
   for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {
     const uint64_t TypeAddress = TypeTable[Index];
     switch (TTypeEncoding & 0x70) {
@@ -1063,6 +1043,10 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {
     }
     }
   }
+
+  if (TTypeEncoding != dwarf::DW_EH_PE_omit)
+    Streamer.emitLabel(TTBaseLabel);
+
   for (uint8_t const &Byte : BF.getLSDATypeIndexTable())
     Streamer.emitIntValue(Byte, 1);
 }

bolt/lib/Core/BinaryEmitter.cpp

@@ -665,16 +665,13 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
   return true;
 }
 
-std::vector<char> CFIReaderWriter::generateEHFrameHeader(
-    const DWARFDebugFrame &OldEHFrame, const DWARFDebugFrame &NewEHFrame,
-    uint64_t EHFrameHeaderAddress,
-    std::vector<uint64_t> &FailedAddresses) const {
+std::vector<char>
+CFIReaderWriter::generateEHFrameHeader(const DWARFDebugFrame &OldEHFrame,
+                                       const DWARFDebugFrame &NewEHFrame,
+                                       uint64_t EHFrameHeaderAddress) const {
   // Common PC -> FDE map to be written into .eh_frame_hdr.
   std::map<uint64_t, uint64_t> PCToFDE;
 
-  // Presort array for binary search.
-  llvm::sort(FailedAddresses);
-
   // Initialize PCToFDE using NewEHFrame.
   for (dwarf::FrameEntry &Entry : NewEHFrame.entries()) {
     const dwarf::FDE *FDE = dyn_cast<dwarf::FDE>(&Entry);
@@ -689,13 +686,7 @@ std::vector<char> CFIReaderWriter::generateEHFrameHeader(
       continue;
 
     // Add the address to the map unless we failed to write it.
-    if (!std::binary_search(FailedAddresses.begin(), FailedAddresses.end(),
-                            FuncAddress)) {
-      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: FDE for function at 0x"
-                        << Twine::utohexstr(FuncAddress) << " is at 0x"
-                        << Twine::utohexstr(FDEAddress) << '\n');
-      PCToFDE[FuncAddress] = FDEAddress;
-    }
+    PCToFDE[FuncAddress] = FDEAddress;
   };
 
   LLVM_DEBUG(dbgs() << "BOLT-DEBUG: new .eh_frame contains "