Merge branch 'main' into main

Author: StarryCSF

.ci/metrics/metrics.py

@@ -349,19 +349,7 @@ def github_get_metrics(
                     running_count[metric_name] += 1
                 continue
 
-            job_result = int(job.conclusion == "success")
-            if job_result:
-                # We still might want to mark the job as a failure if one of the steps
-                # failed. This is required due to use setting continue-on-error in
-                # the premerge pipeline to prevent sending emails while we are
-                # testing the infrastructure.
-                # TODO(boomanaiden154): Remove this once the premerge pipeline is no
-                # longer in a testing state and we can directly assert the workflow
-                # result.
-                for step in job.steps:
-                    if step.conclusion != "success" and step.conclusion != "skipped":
-                        job_result = 0
-                        break
+            job_result = int(job.conclusion == "success" or job.conclusion == "skipped")
 
             created_at = job.created_at
             started_at = job.started_at

.ci/monolithic-linux.sh

@@ -38,6 +38,7 @@ function at-exit {
 
   ccache --print-stats > artifacts/ccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
+  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
 
   # If building fails there will be no results files.
   shopt -s nullglob
@@ -46,7 +47,7 @@ function at-exit {
     python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":linux: Linux x64 Test Results" \
       "linux-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":linux: Linux x64 Test Results" \
+    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }

.ci/monolithic-windows.sh

@@ -33,6 +33,7 @@ function at-exit {
   mkdir -p artifacts
   sccache --show-stats >> artifacts/sccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
+  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
 
   # If building fails there will be no results files.
   shopt -s nullglob
@@ -41,7 +42,7 @@ function at-exit {
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":windows: Windows x64 Test Results" \
       "windows-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":windows: Windows x64 Test Results" \
+    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }

.github/CODEOWNERS

@@ -128,7 +128,7 @@
 /mlir/**/Transforms/SROA.* @moxinilian
 
 # BOLT
-/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @yota9
+/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @yota9 @paschalis-mpeis
 
 # Bazel build system.
 /utils/bazel/ @rupprecht @keith @aaronmondal

.github/new-prs-labeler.yml

@@ -554,6 +554,12 @@ flang:fir-hlfir:
 flang:codegen:
   - flang/**/CodeGen/**
 
+llvm:codegen:
+  - llvm/lib/CodeGen/*
+  - llvm/lib/CodeGen/MIRParser/*
+  - llvm/lib/CodeGen/LiveDebugValues/*
+  - llvm/lib/CodeGen/AsmPrinter/*
+
 llvm:globalisel:
   - llvm/**/GlobalISel/**
   - llvm/utils/TableGen/GlobalISel*
@@ -703,6 +709,10 @@ mlgo:
   - llvm/test/CodeGen/MLRegAlloc/**
   - llvm/utils/mlgo-utils/**
   - llvm/docs/MLGO.rst
+  - llvm/include/llvm/Analysis/IR2Vec.h
+  - llvm/lib/Analysis/IR2Vec.cpp
+  - llvm/lib/Analysis/models/**
+  - llvm/test/Analysis/IR2Vec/**
 
 tools:llvm-exegesis:
   - llvm/tools/llvm-exegesis/**

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -85,7 +85,7 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-ARG RUNNER_VERSION=2.324.0
+ARG RUNNER_VERSION=2.325.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \

.github/workflows/containers/github-action-ci/Dockerfile

@@ -86,7 +86,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.324.0
+ENV GITHUB_RUNNER_VERSION=2.325.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \

.github/workflows/docs.yml

@@ -205,7 +205,7 @@ jobs:
           steps.docs-changed-subprojects.outputs.workflow_any_changed == 'true'
         run: |
           cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON ./llvm
-          TZ=UTC ninja -C flang-build docs-flang-html
+          TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
           mkdir built-docs/flang
           cp -r flang-build/docs/* built-docs/flang/
       - name: Upload docs

.github/workflows/libcxx-build-and-test.yaml

@@ -184,7 +184,7 @@ jobs:
             **/crash_diagnostics/*
 
   macos:
-    needs: [ stage3 ]
+    needs: [ stage2 ]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/pr-code-format.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Install clang-format
         uses: aminya/setup-cpp@17c11551771948abc5752bbf3183482567c7caf0 # v1.1.1
         with:
-          clangformat: 19.1.6
+          clangformat: 20.1.5
 
       - name: Setup Python env
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0

bolt/Maintainers.txt

@@ -20,6 +20,10 @@ N: Vladislav Khmelevsky
 E: [email protected]
 D: AArch64 backend
 
+N: Paschalis Mpeis
+E: [email protected]
+D: AArch64 backend
+
 N: Job Noorman
 E: [email protected]
 D: RISC-V backend

bolt/docs/Heatmaps.md

@@ -89,7 +89,13 @@ For the generation, the default bucket size was used with a line size of 128.
 Some useful options are:
 
 ```
--line-size=<uint>   - number of entries per line (default 256)
+-line-size=<uint>   - number of entries per line (default 256).
+                      Use a smaller value (e.g. 128) if the heatmap doesn't fit
+                      the screen horizontally.
+-block-size=<initial size>[,<zoom-out size>,...] - heatmap bucket size,
+                      optionally followed by zoom-out sizes to produce coarse-
+                      grained heatmaps. Size can be specified in human-readable
+                      format with [kKmMgG][i][B] suffix. Default 64B, 4K, 256K.
 -max-address=<uint> - maximum address considered valid for heatmap (default 4GB)
 -print-mappings     - print mappings in the legend, between characters/blocks and text sections (default false)
 ```

bolt/include/bolt/Core/BinaryFunction.h

@@ -2348,6 +2348,7 @@ class BinaryFunction {
       releaseCFG();
       CurrentState = State::Emitted;
     }
+    clearList(Relocations);
   }
 
   /// Process LSDA information for the function.

bolt/include/bolt/Profile/DataAggregator.h

@@ -370,33 +370,46 @@ class DataAggregator : public DataReader {
   /// memory.
   ///
   /// File format syntax:
-  /// {B|F|f|T} [<start_id>:]<start_offset> [<end_id>:]<end_offset> [<ft_end>]
-  ///       <count> [<mispred_count>]
+  /// E <event>
+  /// S <start> <count>
+  /// T <start> <end> <ft_end> <count>
+  /// B <start> <end> <count> <mispred_count>
+  /// [Ff] <start> <end> <count>
   ///
-  /// B - indicates an aggregated branch
-  /// F - an aggregated fall-through
+  /// where <start>, <end>, <ft_end> have the format [<id>:]<offset>
+  ///
+  /// E - name of the sampling event used for subsequent entries
+  /// S - indicates an aggregated basic sample at <start>
+  /// B - indicates an aggregated branch from <start> to <end>
+  /// F - an aggregated fall-through from <start> to <end>
   /// f - an aggregated fall-through with external origin - used to disambiguate
   ///       between a return hitting a basic block head and a regular internal
   ///       jump to the block
-  /// T - an aggregated trace: branch with a fall-through (from, to, ft_end)
-  ///
-  /// <start_id> - build id of the object containing the start address. We can
-  /// skip it for the main binary and use "X" for an unknown object. This will
-  /// save some space and facilitate human parsing.
-  ///
-  /// <start_offset> - hex offset from the object base load address (0 for the
-  /// main executable unless it's PIE) to the start address.
+  /// T - an aggregated trace: branch from <start> to <end> with a fall-through
+  ///       to <ft_end>
   ///
-  /// <end_id>, <end_offset> - same for the end address.
+  /// <id> - build id of the object containing the address. We can skip it for
+  /// the main binary and use "X" for an unknown object. This will save some
+  /// space and facilitate human parsing.
   ///
-  /// <ft_end> - same for the fallthrough_end address.
+  /// <offset> - hex offset from the object base load address (0 for the
+  /// main executable unless it's PIE) to the address.
   ///
-  /// <count> - total aggregated count of the branch or a fall-through.
+  /// <count> - total aggregated count.
   ///
   /// <mispred_count> - the number of times the branch was mispredicted.
-  /// Omitted for fall-throughs.
   ///
   /// Example:
+  /// Basic samples profile:
+  /// E cycles
+  /// S 41be50 3
+  /// E br_inst_retired.near_taken
+  /// S 41be60 6
+  ///
+  /// Trace profile combining branches and fall-throughs:
+  /// T 4b196f 4b19e0 4b19ef 2
+  ///
+  /// Legacy branch profile with separate branches and fall-throughs:
   /// F 41be50 41be50 3
   /// F 41be90 41be90 4
   /// B 4b1942 39b57f0 3 0

bolt/include/bolt/Profile/Heatmap.h

@@ -85,6 +85,9 @@ class Heatmap {
   void printSectionHotness(raw_ostream &OS) const;
 
   size_t size() const { return Map.size(); }
+
+  /// Increase bucket size to \p NewSize, recomputing the heatmap.
+  void resizeBucket(uint64_t NewSize);
 };
 
 } // namespace bolt

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -23,6 +23,15 @@ enum HeatmapModeKind {
   HM_Optional   // perf2bolt --heatmap
 };
 
+using HeatmapBlockSizes = std::vector<unsigned>;
+struct HeatmapBlockSpecParser : public llvm::cl::parser<HeatmapBlockSizes> {
+  explicit HeatmapBlockSpecParser(llvm::cl::Option &O)
+      : llvm::cl::parser<HeatmapBlockSizes>(O) {}
+  // Return true on error.
+  bool parse(llvm::cl::Option &O, llvm::StringRef ArgName, llvm::StringRef Arg,
+             HeatmapBlockSizes &Val);
+};
+
 extern HeatmapModeKind HeatmapMode;
 extern bool BinaryAnalysisMode;
 
@@ -47,7 +56,8 @@ extern llvm::cl::opt<bool> EqualizeBBCounts;
 extern llvm::cl::opt<bool> ForcePatch;
 extern llvm::cl::opt<bool> RemoveSymtab;
 extern llvm::cl::opt<unsigned> ExecutionCountThreshold;
-extern llvm::cl::opt<unsigned> HeatmapBlock;
+extern llvm::cl::opt<HeatmapBlockSizes, false, HeatmapBlockSpecParser>
+    HeatmapBlock;
 extern llvm::cl::opt<unsigned long long> HeatmapMaxAddress;
 extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HeatmapPrintMappings;

bolt/lib/Core/BinaryContext.cpp

@@ -1032,10 +1032,8 @@ void BinaryContext::adjustCodePadding() {
 
     if (!hasValidCodePadding(BF)) {
       if (HasRelocations) {
-        if (opts::Verbosity >= 1) {
-          this->outs() << "BOLT-INFO: function " << BF
-                       << " has invalid padding. Ignoring the function.\n";
-        }
+        this->errs() << "BOLT-WARNING: function " << BF
+                     << " has invalid padding. Ignoring the function\n";
         BF.setIgnored();
       } else {
         BF.setMaxSize(BF.getSize());

bolt/lib/Core/BinaryFunction.cpp

@@ -1526,8 +1526,6 @@ Error BinaryFunction::disassemble() {
   if (uint64_t Offset = getFirstInstructionOffset())
     Labels[Offset] = BC.Ctx->createNamedTempSymbol();
 
-  clearList(Relocations);
-
   if (!IsSimple) {
     clearList(Instructions);
     return createNonFatalBOLTError("");
@@ -3244,16 +3242,26 @@ void BinaryFunction::setTrapOnEntry() {
 }
 
 void BinaryFunction::setIgnored() {
+  IsIgnored = true;
+
   if (opts::processAllFunctions()) {
     // We can accept ignored functions before they've been disassembled.
-    // In that case, they would still get disassembled and emited, but not
+    // In that case, they would still get disassembled and emitted, but not
     // optimized.
-    assert(CurrentState == State::Empty &&
-           "cannot ignore non-empty functions in current mode");
-    IsIgnored = true;
+    if (CurrentState != State::Empty) {
+      BC.errs() << "BOLT-ERROR: cannot ignore non-empty function " << *this
+                << " in current mode\n";
+      exit(1);
+    }
     return;
   }
 
+  IsSimple = false;
+  LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n');
+
+  if (CurrentState == State::Empty)
+    return;
+
   clearDisasmState();
 
   // Clear CFG state too.
@@ -3273,9 +3281,11 @@ void BinaryFunction::setIgnored() {
 
   CurrentState = State::Empty;
 
-  IsIgnored = true;
-  IsSimple = false;
-  LLVM_DEBUG(dbgs() << "Ignoring " << getPrintName() << '\n');
+  // Fix external references in the original function body.
+  if (BC.HasRelocations) {
+    LLVM_DEBUG(dbgs() << "Scanning refs in " << *this << '\n');
+    scanExternalRefs();
+  }
 }
 
 void BinaryFunction::duplicateConstantIslands() {
@@ -3764,7 +3774,6 @@ void BinaryFunction::postProcessBranches() {
 
 MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) {
   assert(Offset && "cannot add primary entry point");
-  assert(CurrentState == State::Empty || CurrentState == State::Disassembled);
 
   const uint64_t EntryPointAddress = getAddress() + Offset;
   MCSymbol *LocalSymbol = getOrCreateLocalLabel(EntryPointAddress);
@@ -3773,6 +3782,8 @@ MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) {
   if (EntrySymbol)
     return EntrySymbol;
 
+  assert(CurrentState == State::Empty || CurrentState == State::Disassembled);
+
   if (BinaryData *EntryBD = BC.getBinaryDataAtAddress(EntryPointAddress)) {
     EntrySymbol = EntryBD->getSymbol();
   } else {

bolt/lib/Passes/ADRRelaxationPass.cpp

@@ -81,17 +81,15 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) {
         It = BB.eraseInstruction(std::prev(It));
       } else if (std::next(It) != BB.end() && BC.MIB->isNoop(*std::next(It))) {
         BB.eraseInstruction(std::next(It));
-      } else if (!opts::StrictMode && !BF.isSimple()) {
+      } else if (!BF.isSimple()) {
         // If the function is not simple, it may contain a jump table undetected
         // by us. This jump table may use an offset from the branch instruction
         // to land in the desired place. If we add new instructions, we
         // invalidate this offset, so we have to rely on linker-inserted NOP to
         // replace it with ADRP, and abort if it is not present.
         auto L = BC.scopeLock();
-        BC.errs() << formatv(
-            "BOLT-ERROR: Cannot relax adr in non-simple function "
-            "{0}. Use --strict option to override\n",
-            BF.getOneName());
+        BC.errs() << "BOLT-ERROR: cannot relax ADR in non-simple function "
+                  << BF << '\n';
         PassFailed = true;
         return;
       }