Merge from 'main' to 'sycl-web' (100 commits)

  CONFLICT (content): Merge conflict in clang/include/clang/Basic/CodeGenOptions.h
  CONFLICT (content): Merge conflict in clang/lib/CodeGen/BackendUtil.cpp

Author: fineg74

.github/workflows/pr-code-format.yml

@@ -11,14 +11,33 @@ jobs:
       - name: Fetch LLVM sources
         uses: actions/checkout@v4
         with:
-          fetch-depth: 2 # Fetches only the last 2 commits
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout through merge base
+        uses: rmacklin/fetch-through-merge-base@v0
+        with:
+          base_ref: ${{ github.event.pull_request.base.ref }}
+          head_ref: ${{ github.event.pull_request.head.sha }}
+          deepen_length: 500
 
       - name: Get changed files
         id: changed-files
         uses: tj-actions/changed-files@v39
         with:
           separator: ","
-          fetch_depth: 2000 # Fetches only the last 2000 commits
+          skip_initial_fetch: true
+
+      # We need to make sure that we aren't executing/using any code from the
+      # PR for security reasons as we're using pull_request_target. Checkout
+      # the target branch with the necessary files.
+      - name: Fetch code formatting utils
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            llvm/utils/git/requirements_formatting.txt
+            llvm/utils/git/code-format-helper.py
+          sparse-checkout-cone-mode: false
+          path: code-format-tools
 
       - name: "Listed files"
         run: |
@@ -35,10 +54,10 @@ jobs:
         with:
           python-version: '3.11'
           cache: 'pip'
-          cache-dependency-path: 'llvm/utils/git/requirements_formatting.txt'
+          cache-dependency-path: 'code-format-tools/llvm/utils/git/requirements_formatting.txt'
 
       - name: Install python dependencies
-        run: pip install -r llvm/utils/git/requirements_formatting.txt
+        run: pip install -r code-format-tools/llvm/utils/git/requirements_formatting.txt
 
       - name: Run code formatter
         env:
@@ -47,7 +66,7 @@ jobs:
           END_REV: ${{ github.event.pull_request.head.sha }}
           CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          python llvm/utils/git/code-format-helper.py \
+          python ./code-format-tools/llvm/utils/git/code-format-helper.py \
             --token ${{ secrets.GITHUB_TOKEN }} \
             --issue-number $GITHUB_PR_NUMBER \
             --start-rev $START_REV \

bolt/lib/Passes/BinaryPasses.cpp

@@ -582,65 +582,47 @@ bool CheckLargeFunctions::shouldOptimize(const BinaryFunction &BF) const {
 }
 
 void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
-  std::vector<std::pair<MCInst *, uint32_t>> PreservedOffsetAnnotations;
-  std::vector<std::pair<MCInst *, MCSymbol *>> PreservedLabelAnnotations;
-
-  for (auto &It : BC.getBinaryFunctions()) {
-    BinaryFunction &BF = It.second;
-
-    for (FunctionFragment &FF : BF.getLayout().fragments()) {
+  for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+    for (FunctionFragment &FF : BF->getLayout().fragments()) {
+      // Reset at the start of the new fragment.
       int64_t CurrentGnuArgsSize = 0;
 
       for (BinaryBasicBlock *const BB : FF) {
-        // First convert GnuArgsSize annotations into CFIs. This may change
-        // instr pointers, so do it before recording ptrs for preserved
-        // annotations
-        if (BF.usesGnuArgsSize()) {
-          for (auto II = BB->begin(); II != BB->end(); ++II) {
-            if (!BC.MIB->isInvoke(*II))
-              continue;
+        for (auto II = BB->begin(); II != BB->end(); ++II) {
+
+          // Convert GnuArgsSize annotations into CFIs.
+          if (BF->usesGnuArgsSize() && BC.MIB->isInvoke(*II)) {
             const int64_t NewGnuArgsSize = BC.MIB->getGnuArgsSize(*II);
             assert(NewGnuArgsSize >= 0 &&
-                   "expected non-negative GNU_args_size");
+                   "Expected non-negative GNU_args_size.");
             if (NewGnuArgsSize != CurrentGnuArgsSize) {
-              auto InsertII = BF.addCFIInstruction(
+              auto InsertII = BF->addCFIInstruction(
                   BB, II,
                   MCCFIInstruction::createGnuArgsSize(nullptr, NewGnuArgsSize));
               CurrentGnuArgsSize = NewGnuArgsSize;
               II = std::next(InsertII);
             }
           }
-        }
 
-        // Now record preserved annotations separately and then strip
-        // annotations.
-        for (auto II = BB->begin(); II != BB->end(); ++II) {
-          if (BF.requiresAddressTranslation() && BC.MIB->getOffset(*II))
-            PreservedOffsetAnnotations.emplace_back(&(*II),
-                                                    *BC.MIB->getOffset(*II));
-          if (MCSymbol *Label = BC.MIB->getLabel(*II))
-            PreservedLabelAnnotations.emplace_back(&*II, Label);
+          // Preserve selected annotations and strip the rest.
+          std::optional<uint32_t> Offset = BF->requiresAddressTranslation()
+                                               ? BC.MIB->getOffset(*II)
+                                               : std::nullopt;
+          MCSymbol *Label = BC.MIB->getLabel(*II);
+
           BC.MIB->stripAnnotations(*II);
+
+          if (Offset)
+            BC.MIB->setOffset(*II, *Offset);
+          if (Label)
+            BC.MIB->setLabel(*II, Label);
         }
       }
     }
   }
-  for (BinaryFunction *BF : BC.getInjectedBinaryFunctions())
-    for (BinaryBasicBlock &BB : *BF)
-      for (MCInst &Instruction : BB) {
-        if (MCSymbol *Label = BC.MIB->getLabel(Instruction))
-          PreservedLabelAnnotations.emplace_back(&Instruction, Label);
-        BC.MIB->stripAnnotations(Instruction);
-      }
 
   // Release all memory taken by annotations
   BC.MIB->freeAnnotations();
-
-  // Reinsert preserved annotations we need during code emission.
-  for (const std::pair<MCInst *, uint32_t> &Item : PreservedOffsetAnnotations)
-    BC.MIB->setOffset(*Item.first, Item.second);
-  for (auto [Instr, Label] : PreservedLabelAnnotations)
-    BC.MIB->setLabel(*Instr, Label);
 }
 
 // Check for dirty state in MCSymbol objects that might be a consequence

bolt/utils/nfc-stat-parser.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import re
+import sys
+import os
+from statistics import geometric_mean
+
+TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="BOLT NFC stat parser",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper"
+    )
+    parser.add_argument(
+        "--check_longer_than",
+        default=0.5,
+        type=float,
+        help="Only warn on tests longer than X seconds for at least one side",
+    )
+    parser.add_argument(
+        "--threshold_single",
+        default=10,
+        type=float,
+        help="Threshold for a single test result swing, abs percent",
+    ),
+    parser.add_argument(
+        "--threshold_agg",
+        default=5,
+        type=float,
+        help="Threshold for geomean test results swing, abs percent",
+    ),
+    parser.add_argument("--verbose", "-v", action="store_true")
+    args = parser.parse_args()
+
+    def fmt_delta(value, exc_threshold, above_bound=True):
+        formatted_value = format(value, "+.2%")
+        if not above_bound:
+            formatted_value += "?"
+        elif exc_threshold and sys.stdout.isatty():  # terminal supports colors
+            return f"\033[1m{formatted_value}\033[0m"
+        return formatted_value
+
+    # Ratios for geomean computation
+    time_ratios = []
+    mem_ratios = []
+    # Whether any test exceeds the single test threshold (mem or time)
+    threshold_single = False
+    # Whether geomean exceeds aggregate test threshold (mem or time)
+    threshold_agg = False
+
+    if args.verbose:
+        print(f"# Individual test threshold: +-{args.threshold_single}%")
+        print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%")
+        print(
+            f"# Checking time swings for tests with runtime >"
+            f"{args.check_longer_than}s - otherwise marked as ?"
+        )
+        print("Test/binary BOLT_wall_time BOLT_max_rss")
+
+    for input_file in args.input:
+        input_dir = os.path.dirname(input_file)
+        with open(input_file) as timing_file:
+            timing_reader = csv.reader(timing_file, delimiter=";")
+            for row in timing_reader:
+                test_name = row[0]
+                m = TIMING_LOG_RE.match(row[0])
+                if m:
+                    test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}"
+                else:
+                    # Prepend input dir to unparsed test name
+                    test_name = input_dir + "#" + test_name
+                time_a, time_b = float(row[1]), float(row[3])
+                mem_a, mem_b = int(row[2]), int(row[4])
+                # Check if time is above bound for at least one side
+                time_above_bound = any(
+                    [x > args.check_longer_than for x in [time_a, time_b]]
+                )
+                # Compute B/A ratios (for % delta and geomean)
+                time_ratio = time_b / time_a if time_a else float('nan')
+                mem_ratio = mem_b / mem_a if mem_a else float('nan')
+                # Keep ratios for geomean
+                if time_above_bound and time_ratio > 0:  # must be >0 for gmean
+                    time_ratios += [time_ratio]
+                mem_ratios += [mem_ratio]
+                # Deltas: (B/A)-1 = (B-A)/A
+                time_delta = time_ratio - 1
+                mem_delta = mem_ratio - 1
+                # Check individual test results vs single test threshold
+                time_exc = (
+                    100.0 * abs(time_delta) > args.threshold_single and time_above_bound
+                )
+                mem_exc = 100.0 * abs(mem_delta) > args.threshold_single
+                if time_exc or mem_exc:
+                    threshold_single = True
+                # Print deltas with formatting in verbose mode
+                if args.verbose or time_exc or mem_exc:
+                    print(
+                        test_name,
+                        fmt_delta(time_delta, time_exc, time_above_bound),
+                        fmt_delta(mem_delta, mem_exc),
+                    )
+
+    time_gmean_delta = geometric_mean(time_ratios) - 1
+    mem_gmean_delta = geometric_mean(mem_ratios) - 1
+    time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg
+    mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg
+    if time_agg_threshold or mem_agg_threshold:
+        threshold_agg = True
+    if time_agg_threshold or mem_agg_threshold or args.verbose:
+        print(
+            "Geomean",
+            fmt_delta(time_gmean_delta, time_agg_threshold),
+            fmt_delta(mem_gmean_delta, mem_agg_threshold),
+        )
+    exit(threshold_single or threshold_agg)
+
+
+if __name__ == "__main__":
+    main()