Merge remote-tracking branch 'upstream/main' into libcxx/ranges/join_with

Author: JMazurkiewicz

.ci/compute_projects_test.py

@@ -147,7 +147,7 @@ def test_flang(self):
 
     def test_invalid_subproject(self):
         env_variables = compute_projects.get_env_variables(
-            [".ci/compute_projects.py"], "Linux"
+            ["third-party/benchmark/CMakeLists.txt"], "Linux"
         )
         self.assertEqual(env_variables["projects_to_build"], "")
         self.assertEqual(env_variables["project_check_targets"], "")
@@ -163,7 +163,7 @@ def test_top_level_file(self):
 
     def test_exclude_runtiems_in_projects(self):
         env_variables = compute_projects.get_env_variables(
-            [".ci/compute_projects.py", "libcxx/CMakeLists.txt"], "Linux"
+            ["libcxx/CMakeLists.txt"], "Linux"
         )
         self.assertEqual(env_variables["projects_to_build"], "")
         self.assertEqual(env_variables["project_check_targets"], "")
@@ -192,10 +192,10 @@ def test_ci(self):
         env_variables = compute_projects.get_env_variables(
             [".ci/compute_projects.py"], "Linux"
         )
-        self.assertEqual(env_variables["projects_to_build"], "clang;lld;llvm;lldb")
+        self.assertEqual(env_variables["projects_to_build"], "clang;lld;lldb;llvm")
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-lld check-llvm check-lldb",
+            "check-clang check-lld check-lldb check-llvm",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"

.ci/generate_test_report_lib.py

@@ -122,6 +122,17 @@ def plural(num_tests):
             ]
         )
 
+    if failures or return_code != 0:
+        report.extend(
+            [
+                "",
+                "If these failures are unrelated to your changes (for example "
+                "tests are broken or flaky at HEAD), please open an issue at "
+                "https://github.com/llvm/llvm-project/issues and add the "
+                "`infrastructure` label.",
+            ]
+        )
+
     report = "\n".join(report)
     if len(report.encode("utf-8")) > size_limit:
         return generate_report(

.ci/generate_test_report_lib_test.py

@@ -109,7 +109,9 @@ def test_no_failures_build_failed(self):
 
               All tests passed but another part of the build **failed**.
 
-              [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
+              [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+              
+              If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -169,7 +171,9 @@ def test_report_single_file_single_testsuite(self):
           ```
           Other output goes here
           ```
-          </details>"""
+          </details>
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -203,7 +207,9 @@ def test_report_single_file_single_testsuite(self):
         ```
         DEF/test_2 output goes here
         ```
-        </details>"""
+        </details>
+        
+        If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
         ),
         "error",
     )
@@ -311,7 +317,9 @@ def test_report_dont_list_failures(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+          Failed tests and their output was too large to report. Download the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
@@ -352,13 +360,16 @@ def test_report_dont_list_failures_link_to_log(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
+          Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),
         )
 
     def test_report_size_limit(self):
+        test_output = "f" * 1000
         self.assertEqual(
             generate_test_report_lib.generate_report(
                 "Foo",
@@ -371,14 +382,16 @@ def test_report_size_limit(self):
           <testsuites time="0.02">
           <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
           <testcase classname="Bar/test_1" name="test_1" time="0.02">
-            <failure><![CDATA[Some long output goes here...]]></failure>
+            <failure><![CDATA[{output}]]></failure>
           </testcase>
           </testsuite>
-          </testsuites>"""
+          </testsuites>""".format(
+                                output=test_output
+                            )
                         )
                     )
                 ],
-                size_limit=128,
+                size_limit=512,
             ),
             (
                 dedent(
@@ -387,7 +400,9 @@ def test_report_size_limit(self):
 
           * 1 test failed
 
-          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+          Failed tests and their output was too large to report. Download the build's log file to see the details.
+          
+          If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
                 ),
                 "error",
             ),

.ci/metrics/metrics.py

@@ -29,8 +29,8 @@
 # name.
 GITHUB_JOB_TO_TRACK = {
     "github_llvm_premerge_checks": {
-        "Build and Test Linux (Test Only - Please Ignore Results)": "premerge_linux",
-        "Build and Test Windows (Test Only - Please Ignore Results)": "premerge_windows",
+        "Build and Test Linux": "premerge_linux",
+        "Build and Test Windows": "premerge_windows",
     }
 }
 

.github/new-prs-labeler.yml

@@ -709,6 +709,9 @@ tools:llvm-exegesis:
   - llvm/test/tools/llvm-exegesis/**
   - llvm/unittests/tools/llvm-exegesis/**
 
+tools:llvm-reduce:
+  - llvm/tools/llvm-reduce/**
+
 platform:windows:
   - lld/COFF/**
   - clang/lib/Driver/MSVC.cpp

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -39,33 +39,15 @@ RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & \
 
 # install tools as described in https://llvm.org/docs/GettingStartedVS.html
 # and a few more that were not documented...
-RUN choco install -y ninja git
+RUN choco install -y ninja git sccache
 # Pin an older version of Python; the current Python 3.10 fails when
 # doing "pip install" for the other dependencies, as it fails to find libxml
 # while compiling some package.
 RUN choco install -y python3 --version 3.9.7
 
-# ActivePerl is currently not installable via Chocolatey, see
-# http://disq.us/p/2ipditb. Install StrawberryPerl instead. Unfortunately,
-# StrawberryPerl not only installs Perl, but also a redundant C/C++ compiler
-# toolchain, and a copy of pkg-config which can cause misdetections for other
-# built products, see
-# https://github.com/StrawberryPerl/Perl-Dist-Strawberry/issues/11 for further
-# details. Remove the redundant and unnecessary parts of the StrawberryPerl
-# install.
-RUN choco install -y strawberryperl && \
-    rmdir /q /s c:\strawberry\c && \
-    del /q c:\strawberry\perl\bin\pkg-config*
-
-# libcxx requires clang(-cl) to be available
-RUN choco install -y sccache llvm
+# Testing requires psutil
 RUN pip install psutil
 
-RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-x86_64.zip && \
-    powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && \
-    del llvm-mingw-*-ucrt-x86_64.zip && \
-    ren llvm-mingw-20230320-ucrt-x86_64 llvm-mingw
-
 # configure Python encoding
 ENV PYTHONIOENCODING=UTF-8
 
@@ -103,11 +85,6 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-# handle for debugging of files beeing locked by some processes.
-RUN choco install -y handle
-
-RUN pip3 install pywin32 buildbot-worker==2.8.4
-
 ARG RUNNER_VERSION=2.324.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 

.github/workflows/libcxx-build-and-test.yaml

@@ -37,7 +37,7 @@ jobs:
   stage1:
     if: github.repository_owner == 'llvm'
     runs-on: libcxx-self-hosted-linux
-    container: ghcr.io/llvm/libcxx-linux-builder:2b57ebb50b6d418e70382e655feaa619b558e254
+    container: ghcr.io/llvm/libcxx-linux-builder:b060022103f551d8ca1dad84122ef73927c86512
     continue-on-error: false
     strategy:
       fail-fast: false

.github/workflows/premerge.yaml

@@ -16,7 +16,6 @@ on:
       - closed
   push:
     branches:
-      - 'main'
       - 'release/**'
 
 concurrency:
@@ -25,7 +24,7 @@ concurrency:
 
 jobs:
   premerge-checks-linux:
-    name: Build and Test Linux (Test Only - Please Ignore Results)
+    name: Build and Test Linux
     if: >-
         github.repository_owner == 'llvm' &&
         (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -43,9 +42,6 @@ jobs:
         # Mark the job as a success even if the step fails so that people do
         # not get notified while the new premerge pipeline is in an
         # experimental state.
-        # TODO(boomanaiden154): Remove this once the pipeline is stable and we
-        # are ready for people to start recieving notifications.
-        continue-on-error: true
         run: |
           git config --global --add safe.directory '*'
 
@@ -74,7 +70,7 @@ jobs:
           include-hidden-files: 'true'
 
   premerge-checks-windows:
-    name: Build and Test Windows (Test Only - Please Ignore Results)
+    name: Build and Test Windows
     if: >-
         github.repository_owner == 'llvm' &&
         (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -110,9 +106,6 @@ jobs:
         # Mark the job as a success even if the step fails so that people do
         # not get notified while the new premerge pipeline is in an
         # experimental state.
-        # TODO(boomanaiden154): Remove this once the pipeline is stable and we
-        # are ready for people to start recieving notifications.
-        continue-on-error: true
         if: ${{ steps.vars.outputs.windows-projects != '' }}
         shell: cmd
         run: |

bolt/CMakeLists.txt

@@ -46,6 +46,10 @@ if(BOLT_BUILT_STANDALONE)
   set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin)
   set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX})
 
+  separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
+  add_definitions(${LLVM_DEFINITIONS_LIST})
+  list(APPEND CMAKE_REQUIRED_DEFINITIONS ${LLVM_DEFINITIONS_LIST})
+
   include(AddLLVM)
   include(TableGen)
   include_directories(${LLVM_INCLUDE_DIRS})

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -562,35 +562,56 @@ class MCPlusBuilder {
     return {};
   }
 
-  virtual ErrorOr<MCPhysReg> getAuthenticatedReg(const MCInst &Inst) const {
-    llvm_unreachable("not implemented");
-    return getNoRegister();
-  }
-
-  virtual bool isAuthenticationOfReg(const MCInst &Inst,
-                                     MCPhysReg AuthenticatedReg) const {
+  /// Returns the register where an authenticated pointer is written to by Inst,
+  /// or std::nullopt if not authenticating any register.
+  ///
+  /// Sets IsChecked if the instruction always checks authenticated pointer,
+  /// i.e. it either writes a successfully authenticated pointer or terminates
+  /// the program abnormally (such as "ldra x0, [x1]!" on AArch64, which crashes
+  /// on authentication failure even if FEAT_FPAC is not implemented).
+  virtual std::optional<MCPhysReg>
+  getWrittenAuthenticatedReg(const MCInst &Inst, bool &IsChecked) const {
     llvm_unreachable("not implemented");
-    return false;
+    return std::nullopt;
   }
 
-  virtual MCPhysReg getSignedReg(const MCInst &Inst) const {
+  /// Returns the register signed by Inst, or std::nullopt if not signing any
+  /// register.
+  ///
+  /// The returned register is assumed to be both input and output operand,
+  /// as it is done on AArch64.
+  virtual std::optional<MCPhysReg> getSignedReg(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
-    return getNoRegister();
+    return std::nullopt;
   }
 
-  virtual ErrorOr<MCPhysReg> getRegUsedAsRetDest(const MCInst &Inst) const {
+  /// Returns the register used as a return address. Returns std::nullopt if
+  /// not applicable, such as reading the return address from a system register
+  /// or from the stack.
+  ///
+  /// Sets IsAuthenticatedInternally if the instruction accepts a signed
+  /// pointer as its operand and authenticates it internally.
+  ///
+  /// Should only be called when isReturn(Inst) is true.
+  virtual std::optional<MCPhysReg>
+  getRegUsedAsRetDest(const MCInst &Inst,
+                      bool &IsAuthenticatedInternally) const {
     llvm_unreachable("not implemented");
-    return getNoRegister();
+    return std::nullopt;
   }
 
   /// Returns the register used as the destination of an indirect branch or call
   /// instruction. Sets IsAuthenticatedInternally if the instruction accepts
   /// a signed pointer as its operand and authenticates it internally.
+  ///
+  /// Should only be called if isIndirectCall(Inst) or isIndirectBranch(Inst)
+  /// returns true.
   virtual MCPhysReg
   getRegUsedAsIndirectBranchDest(const MCInst &Inst,
                                  bool &IsAuthenticatedInternally) const {
     llvm_unreachable("not implemented");
-    return getNoRegister();
+    return 0; // Unreachable. A valid register should be returned by the
+              // target implementation.
   }
 
   /// Returns the register containing an address safely materialized by `Inst`
@@ -602,14 +623,14 @@ class MCPlusBuilder {
   ///    controlled, under the Pointer Authentication threat model.
   ///
   /// If the instruction does not write to any register satisfying the above
-  /// two conditions, NoRegister is returned.
+  /// two conditions, std::nullopt is returned.
   ///
   /// The Pointer Authentication threat model assumes an attacker is able to
   /// modify any writable memory, but not executable code (due to W^X).
-  virtual MCPhysReg
+  virtual std::optional<MCPhysReg>
   getMaterializedAddressRegForPtrAuth(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
-    return getNoRegister();
+    return std::nullopt;
   }
 
   /// Analyzes if this instruction can safely perform address arithmetics
@@ -622,10 +643,13 @@ class MCPlusBuilder {
   /// controlled, provided InReg and executable code are not. Please note that
   /// registers other than InReg as well as the contents of memory which is
   /// writable by the process should be considered attacker-controlled.
+  ///
+  /// The instruction should not write any values derived from InReg anywhere,
+  /// except for OutReg.
   virtual std::optional<std::pair<MCPhysReg, MCPhysReg>>
   analyzeAddressArithmeticsForPtrAuth(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
-    return std::make_pair(getNoRegister(), getNoRegister());
+    return std::nullopt;
   }
 
   /// Analyzes if a pointer is checked to be authenticated successfully
@@ -670,10 +694,10 @@ class MCPlusBuilder {
   ///
   /// Use this function for simple, single-instruction patterns instead of
   /// its getAuthCheckedReg(BB) counterpart.
-  virtual MCPhysReg getAuthCheckedReg(const MCInst &Inst,
-                                      bool MayOverwrite) const {
+  virtual std::optional<MCPhysReg> getAuthCheckedReg(const MCInst &Inst,
+                                                     bool MayOverwrite) const {
     llvm_unreachable("not implemented");
-    return getNoRegister();
+    return std::nullopt;
   }
 
   virtual bool isTerminator(const MCInst &Inst) const;