Merge branch 'master' into cxx-link-stdlib

Author: MForster

CMakeLists.txt

@@ -12,6 +12,11 @@ if(POLICY CMP0067)
   cmake_policy(SET CMP0067 NEW)
 endif()
 
+# Convert relative paths to absolute for subdirectory `target_sources`
+if(POLICY CMP0076)
+  cmake_policy(SET CMP0076 NEW)
+endif()
+
 # Add path for custom CMake modules.
 list(APPEND CMAKE_MODULE_PATH
     "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
@@ -142,14 +147,20 @@ set(SWIFT_COMPILER_VERSION "" CACHE STRING
 set(CLANG_COMPILER_VERSION "" CACHE STRING
     "The internal version of the Clang compiler")
 
-# Indicate whether Swift should attempt to use the lld linker.
-set(SWIFT_ENABLE_LLD_LINKER TRUE CACHE BOOL
-    "Enable using the lld linker when available")
-
-# Indicate whether Swift should attempt to use the gold linker.
-# This is not used on Darwin.
-set(SWIFT_ENABLE_GOLD_LINKER TRUE CACHE BOOL
-    "Enable using the gold linker when available")
+# Which default linker to use. Prefer LLVM_USE_LINKER if it set, otherwise use
+# our own defaults. This should only be possible in a unified (not stand alone)
+# build environment.
+if(LLVM_USE_LINKER)
+  set(SWIFT_USE_LINKER_default "${LLVM_USE_LINKER}")
+elseif(CMAKE_SYSTEM_NAME STREQUAL Windows AND NOT CMAKE_HOST_SYSTEM_NAME STREQUAL Windows)
+  set(SWIFT_USE_LINKER_default "lld")
+elseif(CMAKE_SYSTEM_NAME STREQUAL Darwin)
+  set(SWIFT_USE_LINKER_default "")
+else()
+  set(SWIFT_USE_LINKER_default "gold")
+endif()
+set(SWIFT_USE_LINKER ${SWIFT_USE_LINKER_default} CACHE STRING
+    "Build Swift with a non-default linker")
 
 set(SWIFT_TOOLS_ENABLE_LTO OFF CACHE STRING "Build Swift tools with LTO. One
     must specify the form of LTO by setting this to one of: 'full', 'thin'. This
@@ -402,6 +413,8 @@ endif()
 
 if(MSVC OR "${CMAKE_SIMULATE_ID}" STREQUAL MSVC)
   include(ClangClCompileRules)
+elseif(UNIX)
+  include(UnixCompileRules)
 endif()
 
 if(CMAKE_C_COMPILER_ID MATCHES Clang)
@@ -421,19 +434,6 @@ if(SWIFT_BUILD_SYNTAXPARSERLIB OR SWIFT_BUILD_SOURCEKIT)
   endif()
 endif()
 
-#
-# Assume a new enough ar to generate the index at construction time. This avoids
-# having to invoke ranlib as a secondary command.
-#
-
-set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> crs <TARGET> <LINK_FLAGS> <OBJECTS>")
-set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> qs <TARGET> <LINK_FLAGS> <OBJECTS>")
-set(CMAKE_C_ARCHIVE_FINISH "")
-
-set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> crs <TARGET> <LINK_FLAGS> <OBJECTS>")
-set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> qs <TARGET> <LINK_FLAGS> <OBJECTS>")
-set(CMAKE_CXX_ARCHIVE_FINISH "")
-
 #
 # Include CMake modules
 #
@@ -458,6 +458,12 @@ if(SWIFT_PATH_TO_CMARK_BUILD)
 endif()
 message(STATUS "")
 
+if("${SWIFT_NATIVE_LLVM_TOOLS_PATH}" STREQUAL "")
+  set(SWIFT_CROSS_COMPILING FALSE)
+else()
+  set(SWIFT_CROSS_COMPILING TRUE)
+endif()
+
 include(SwiftSharedCMakeConfig)
 
 # NOTE: We include this before SwiftComponents as it relies on some LLVM CMake
@@ -487,16 +493,6 @@ if(NOT SWIFT_LIPO)
   find_toolchain_tool(SWIFT_LIPO "${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" lipo)
 endif()
 
-# Reset CMAKE_SYSTEM_PROCESSOR if not cross-compiling.
-# CMake refuses to use `uname -m` on OS X
-# http://public.kitware.com/Bug/view.php?id=10326
-if(NOT CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_PROCESSOR STREQUAL "i386")
-  execute_process(
-      COMMAND "uname" "-m"
-      OUTPUT_VARIABLE CMAKE_SYSTEM_PROCESSOR
-      OUTPUT_STRIP_TRAILING_WHITESPACE)
-endif()
-
 get_filename_component(SWIFT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} REALPATH)
 set(SWIFT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(SWIFT_CMAKE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
@@ -555,9 +551,6 @@ if(XCODE)
   swift_common_xcode_cxx_config()
 endif()
 
-include(SwiftCheckCXXNativeRegex)
-check_cxx_native_regex(SWIFT_HAVE_WORKING_STD_REGEX)
-
 # If SWIFT_HOST_VARIANT_SDK not given, try to detect from the CMAKE_SYSTEM_NAME.
 if(SWIFT_HOST_VARIANT_SDK)
   set(SWIFT_HOST_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}")

README.md

@@ -9,7 +9,7 @@
 | **Ubuntu 16.04** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-incremental-RA-linux-ubuntu-16_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-incremental-RA-linux-ubuntu-16_04)|[![Build Status](https://ci.swift.org/job/oss-swift-package-linux-ubuntu-16_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-linux-ubuntu-16_04)|
 | **Ubuntu 18.04** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-incremental-RA-linux-ubuntu-18_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-incremental-RA-linux-ubuntu-18_04)|[![Build Status](https://ci.swift.org/job/oss-swift-package-linux-ubuntu-18_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-linux-ubuntu-18_04)|
 | **Ubuntu 20.04** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-package-ubuntu-20_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-ubuntu-20_04)|[![Build Status](https://ci.swift.org/job/oss-swift-package-ubuntu-20_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-ubuntu-20_04)|
-| **CentOS 8** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-package-centos-8/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-centos-8)|[![Build Status](https://ci.swift.org/job/oss-swift-package-linux-ubuntu-18_04/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-centos-8)|
+| **CentOS 8** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-package-centos-8/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-centos-8)|[![Build Status](https://ci.swift.org/job/oss-swift-package-centos-8/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-centos-8)|
 | **Amazon Linux 2** | x86_64 | [![Build Status](https://ci.swift.org/job/oss-swift-package-amazon-linux-2/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-amazon-linux-2)|[![Build Status](https://ci.swift.org/job/oss-swift-package-amazon-linux-2/lastCompletedBuild/badge/icon)](https://ci.swift.org/job/oss-swift-package-amazon-linux-2)|
 
 **Swift Community-Hosted CI Platforms**
@@ -85,6 +85,9 @@ Once you are able to build things successfully and have a compile-test-debug
 loop going, check out the [development tips](docs/DevelopmentTips.md) for
 better productivity while working on the compiler.
 
+You can also skim [docs/README.md](/docs/README.md) to understand what
+high-level documentation is available.
+
 ### System Requirements
 
 macOS, Ubuntu Linux LTS, and the latest Ubuntu Linux release are currently
@@ -94,7 +97,7 @@ Please make sure you use Python 2.x. Python 3.x is not supported currently.
 
 #### macOS
 
-To build for macOS, you need [Xcode 11.4](https://developer.apple.com/xcode/resources/).
+To build for macOS, you need [Xcode 12 beta](https://developer.apple.com/xcode/resources/).
 The required version of Xcode changes frequently, and is often a beta release.
 Check this document or the host information on <https://ci.swift.org> for the
 current required version.
@@ -151,8 +154,6 @@ with version 2 shipped with Ubuntu.
 
 **Note:** For Ubuntu 20.04, use `libpython2-dev` in place of the libpython-dev package above.
 
-Build instructions for Ubuntu 14.04 LTS can be found [here](docs/Ubuntu14.md).
-
 ### Getting Sources for Swift and Related Projects
 
 First create a directory for all of the Swift sources:
@@ -240,7 +241,7 @@ various products. These incremental builds are a big timesaver when developing
 and debugging.
 
     cd ${SWIFT_BUILD_DIR}
-    ninja swift
+    ninja swift-frontend
 
 This will build the Swift compiler, but will not rebuild the standard library or
 any other target. Building the `swift-stdlib` target as an additional layer of
@@ -257,7 +258,7 @@ To open the Swift project in Xcode, open `${SWIFT_BUILD_DIR}/Swift.xcodeproj`.
 It will auto-create a *lot* of schemes for all of the available targets. A
 common debug flow would involve:
 
- - Select the 'swift' scheme.
+ - Select the 'swift-frontend' scheme.
  - Pull up the scheme editor (⌘⇧<).
  - Select the 'Arguments' tab and click the '+'.
  - Add the command line options.

benchmark/CMakeLists.txt

@@ -108,6 +108,7 @@ set(SWIFT_BENCH_MODULES
     single-source/LuhnAlgoLazy
     single-source/MapReduce
     single-source/Memset
+    single-source/Mirror
     single-source/MonteCarloE
     single-source/MonteCarloPi
     single-source/NSDictionaryCastToSwift

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -105,7 +105,7 @@ macro(configure_build)
 endmacro()
 
 macro(configure_sdks_darwin)
-  set(macosx_arch "x86_64")
+  set(macosx_arch "x86_64" "arm64")
   set(iphoneos_arch "arm64" "arm64e" "armv7")
   set(appletvos_arch "arm64")
   set(watchos_arch "armv7k")
@@ -609,11 +609,7 @@ function (swift_benchmark_compile_archopts)
 
   if(is_darwin)
     # If host == target.
-    if("${BENCH_COMPILE_ARCHOPTS_PLATFORM}" STREQUAL "macosx")
-      set(OUTPUT_EXEC "${benchmark-bin-dir}/Benchmark_${BENCH_COMPILE_ARCHOPTS_OPT}")
-    else()
-      set(OUTPUT_EXEC "${benchmark-bin-dir}/Benchmark_${BENCH_COMPILE_ARCHOPTS_OPT}-${target}")
-    endif()
+    set(OUTPUT_EXEC "${benchmark-bin-dir}/Benchmark_${BENCH_COMPILE_ARCHOPTS_OPT}-${target}")
   else()
     # If we are on Linux, we do not support cross compiling.
     set(OUTPUT_EXEC "${benchmark-bin-dir}/Benchmark_${BENCH_COMPILE_ARCHOPTS_OPT}")
@@ -660,8 +656,11 @@ function (swift_benchmark_compile_archopts)
           "-m${triple_platform}-version-min=${ver}"
           "-lobjc"
           "-L${SWIFT_LIBRARY_PATH}/${BENCH_COMPILE_ARCHOPTS_PLATFORM}"
+          "-L${sdk}/usr/lib/swift"
           "-Xlinker" "-rpath"
           "-Xlinker" "${SWIFT_LINK_RPATH}"
+          "-Xlinker" "-rpath"
+          "-Xlinker" "/usr/lib/swift"
           ${bench_library_objects}
           ${bench_driver_objects}
           ${ld64_add_ast_path_opts}
@@ -694,7 +693,7 @@ function(swift_benchmark_compile)
   cmake_parse_arguments(SWIFT_BENCHMARK_COMPILE "" "PLATFORM" "" ${ARGN})
 
   if(NOT SWIFT_BENCHMARK_BUILT_STANDALONE)
-    set(stdlib_dependencies "swift")
+    set(stdlib_dependencies "swift-frontend")
     foreach(stdlib_dependency ${UNIVERSAL_LIBRARY_NAMES_${SWIFT_BENCHMARK_COMPILE_PLATFORM}})
       string(FIND "${stdlib_dependency}" "Unittest" find_output)
       if("${find_output}" STREQUAL "-1")

benchmark/scripts/Benchmark_Driver

@@ -73,7 +73,25 @@ class BenchmarkDriver(object):
     def test_harness(self):
         """Full path to test harness binary."""
         suffix = self.args.optimization if hasattr(self.args, "optimization") else "O"
-        return os.path.join(self.args.tests, "Benchmark_" + suffix)
+        if hasattr(self.args, "architecture") and self.args.architecture:
+            suffix += "-" + self.args.architecture + "*"
+        pattern = os.path.join(self.args.tests, "Benchmark_" + suffix)
+        executables = []
+        if hasattr(self._subprocess, "test_mode") and self._subprocess.test_mode:
+            executables = [pattern]
+        else:
+            executables = glob.glob(pattern)
+        if len(executables) == 0:
+            raise ValueError(
+                "No benchmark executable for file name pattern " +
+                pattern + " found")
+        if len(executables) > 1:
+            raise ValueError(
+                "Multiple benchmark executables for file name pattern " +
+                pattern + " found\n" +
+                str(executables) +
+                "\nSpecify the architecture to select the right benchmark executable")
+        return executables[0]
 
     def _git(self, cmd):
         """Execute the Git command in the `swift-repo`."""

benchmark/scripts/run_smoke_bench

@@ -48,12 +48,13 @@ VERBOSE = False
 class DriverArgs(object):
     """Arguments for BenchmarkDriver."""
 
-    def __init__(self, tests, optimization="O"):
+    def __init__(self, tests, optimization="O", architecture=None):
         """Initialize with path to the build-dir and optimization level."""
         self.benchmarks = None
         self.filters = None
         self.tests = os.path.join(tests, "bin")
         self.optimization = optimization
+        self.architecture = architecture
 
 
 def log(msg):
@@ -126,6 +127,12 @@ def main():
         help="The number of re-runs until it's assumed to be a real change",
         default=8,
     )
+    argparser.add_argument(
+        "-arch",
+        type=str,
+        help="The architecture. The default is x86_64",
+        default="x86_64",
+    )
     argparser.add_argument(
         "-platform", type=str, help="The benchmark build platform", default="macosx"
     )
@@ -158,6 +165,7 @@ def test_opt_levels(args):
                 args.num_samples,
                 args.num_reruns,
                 output_file,
+                args.arch
             ):
                 changes = True
 
@@ -167,6 +175,7 @@ def test_opt_levels(args):
                 opt_level,
                 args.oldbuilddir[0],
                 args.newbuilddir[0],
+                args.arch,
                 args.platform,
                 output_file,
             ):
@@ -177,6 +186,7 @@ def test_opt_levels(args):
             "swiftlibs",
             args.oldbuilddir[0],
             args.newbuilddir[0],
+            args.arch,
             args.platform,
             output_file,
         ):
@@ -221,7 +231,8 @@ def merge(results, other_results):
 
 
 def test_performance(
-    opt_level, old_dir, new_dir, threshold, num_samples, num_reruns, output_file
+    opt_level, old_dir, new_dir, threshold, num_samples, num_reruns,
+    output_file, arch
 ):
     """Detect performance changes in benchmarks.
 
@@ -231,7 +242,7 @@ def test_performance(
 
     i, unchanged_length_count = 0, 0
     old, new = [
-        BenchmarkDriver(DriverArgs(dir, optimization=opt_level))
+        BenchmarkDriver(DriverArgs(dir, optimization=opt_level, architecture=arch))
         for dir in [old_dir, new_dir]
     ]
     results = [measure(driver, driver.tests, i) for driver in [old, new]]
@@ -260,18 +271,20 @@ def test_performance(
     )
 
 
-def report_code_size(opt_level, old_dir, new_dir, platform, output_file):
+def report_code_size(opt_level, old_dir, new_dir, architecture, platform, output_file):
     if opt_level == "swiftlibs":
         files = glob.glob(os.path.join(old_dir, "lib", "swift", platform, "*.dylib"))
     else:
         files = glob.glob(
-            os.path.join(old_dir, opt_level + "-*" + platform + "*", "*.o")
+            os.path.join(old_dir, opt_level + "-" + architecture + "*" +
+                         platform + "*", "*.o")
         )
 
     idx = 1
     old_lines = ""
     new_lines = ""
     for oldfile in files:
+        new_dir = os.path.join(new_dir, '')
         newfile = oldfile.replace(old_dir, new_dir, 1)
         if os.path.isfile(newfile):
             oldsize = get_codesize(oldfile)
@@ -369,8 +382,8 @@ performance team (@eeckstein).
 
 
 def check_added(args, output_file=None):
-    old = BenchmarkDriver(DriverArgs(args.oldbuilddir[0]))
-    new = BenchmarkDriver(DriverArgs(args.newbuilddir[0]))
+    old = BenchmarkDriver(DriverArgs(args.oldbuilddir[0], architecture=args.arch))
+    new = BenchmarkDriver(DriverArgs(args.newbuilddir[0], architecture=args.arch))
     added = set(new.tests).difference(set(old.tests))
     new.tests = list(added)
     doctor = BenchmarkDoctor(args, driver=new)

benchmark/scripts/test_Benchmark_Driver.py

@@ -198,6 +198,9 @@ def record_and_respond(self, args, stdin, stdout, stderr, shell):
         self.calls.append(args)
         return self.respond.get(args, "")
 
+    def test_mode():
+        return True
+
 
 class TestBenchmarkDriverInitialization(unittest.TestCase):
     def setUp(self):
@@ -206,13 +209,19 @@ def setUp(self):
 
     def test_test_harness(self):
         self.assertEqual(
-            BenchmarkDriver(self.args, tests=["ignored"]).test_harness,
+            BenchmarkDriver(
+                self.args,
+                tests=["ignored"],
+                _subprocess=self.subprocess_mock).test_harness,
             "/benchmarks/Benchmark_O",
         )
         self.args.tests = "/path"
         self.args.optimization = "Suffix"
         self.assertEqual(
-            BenchmarkDriver(self.args, tests=["ignored"]).test_harness,
+            BenchmarkDriver(
+                self.args,
+                tests=["ignored"],
+                _subprocess=self.subprocess_mock).test_harness,
             "/path/Benchmark_Suffix",
         )
 
@@ -271,6 +280,7 @@ def test_log_file(self):
                 swift_repo=None,
             ),
             tests=["ignored"],
+            _subprocess=self.subprocess_mock
         )
         self.assertEqual(driver.log_file, "/path/Benchmark_Suffix-" + now + ".log")
 

benchmark/single-source/DataBenchmarks.swift

@@ -13,7 +13,7 @@
 import TestsUtils
 import Foundation
 
-let d: [BenchmarkCategory] =  [.validation, .api, .Data]
+let d: [BenchmarkCategory] =  [.validation, .api, .Data, .cpubench]
 
 public let DataBenchmarks = [
   BenchmarkInfo(name: "DataCreateEmpty",