Build and install 'clang' and 'clangd' in the macOS & linux toolchains

- Build script now builds clang_tools_extra as part of LLVM's build.
- Build script now has a new libc++ build step to allow libc++ headers to be installed in the
  resulting toolchain.
- 'clang', 'clangd', 'clang-headers' & 'compiler-rt' targets are now installed for
  the package build configurations for macOS and linux.
- 'clang-resource-dir-symlink' is used in the package build configuration for macOS and linux
  to avoid duplication of Clang's headers and compiler-rt archives.

rdar://24912710

Author: hyp

utils/build-presets.ini

@@ -728,8 +728,8 @@ install-swiftpm
 install-xctest
 install-libicu
 install-prefix=/usr
-swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;swift-remote-mirror;sdk-overlay;parser-lib;license;sourcekit-inproc
-llvm-install-components=llvm-cov;llvm-profdata;IndexStore
+swift-install-components=autolink-driver;compiler;clang-resource-dir-symlink;stdlib;swift-remote-mirror;sdk-overlay;parser-lib;license;sourcekit-inproc
+llvm-install-components=llvm-cov;llvm-profdata;IndexStore;clang;clang-headers;compiler-rt;clangd
 build-swift-static-stdlib
 build-swift-static-sdk-overlay
 build-swift-stdlib-unittest-extra
@@ -1080,8 +1080,8 @@ test-installable-package
 # If someone uses this for incremental builds, force reconfiguration.
 reconfigure
 
-swift-install-components=compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;license;sourcekit-xpc-service;swift-remote-mirror;swift-remote-mirror-headers
-llvm-install-components=llvm-cov;llvm-profdata;IndexStore
+swift-install-components=compiler;clang-resource-dir-symlink;stdlib;sdk-overlay;parser-lib;license;sourcekit-xpc-service;swift-remote-mirror;swift-remote-mirror-headers
+llvm-install-components=llvm-cov;llvm-profdata;IndexStore;clang;clang-headers;compiler-rt;clangd
 
 # Path to the .tar.gz package we would create.
 installable-package=%(installable_package)s

utils/build-script-impl

@@ -1218,6 +1218,12 @@ if [ ! -d "${CLANG_SOURCE_DIR}" ] ; then
     make_relative_symlink "${WORKSPACE}/clang" "${CLANG_SOURCE_DIR}"
 fi
 
+# Don't symlink clang-tools-extra into the tree as the 'clang' symlink prevents
+# make_relative_symlink from working correctly.
+if [ -e "${WORKSPACE}/clang-tools-extra" ] ; then
+    CLANG_TOOLS_EXTRA_SOURCE_DIR="${WORKSPACE}/clang-tools-extra"
+fi
+
 # Symlink compiler-rt into the llvm tree, if it exists.
 COMPILER_RT_SOURCE_DIR="${LLVM_SOURCE_DIR}/projects/compiler-rt"
 if [ -e "${WORKSPACE}/compiler-rt" ] ; then
@@ -1226,7 +1232,16 @@ if [ -e "${WORKSPACE}/compiler-rt" ] ; then
     fi
 fi
 
+# Build libcxx, unless it doesn't exist.
+LIBCXX_SOURCE_DIR="${WORKSPACE}/libcxx"
+if [[ ! -e "${LIBCXX_SOURCE_DIR}" ]] ; then
+  SKIP_BUILD_LIBCXX=1
+fi
+
 PRODUCTS=(cmark llvm)
+if [[ ! "${SKIP_BUILD_LIBCXX}" ]] ; then
+     PRODUCTS=("${PRODUCTS[@]}" libcxx)
+fi
 if [[ ! "${SKIP_BUILD_LIBICU}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" libicu)
 fi
@@ -1572,6 +1587,10 @@ function build_directory_bin() {
             llvm)
                 echo "${root}/${LLVM_BUILD_TYPE}/bin"
                 ;;
+            libcxx)
+                # Reuse LLVM's build type.
+                echo "${root}/${LLVM_BUILD_TYPE}/bin"
+                ;;
             swift)
                 echo "${root}/${SWIFT_BUILD_TYPE}/bin"
                 ;;
@@ -1719,6 +1738,10 @@ function cmake_config_opt() {
             llvm)
                 echo "--config ${LLVM_BUILD_TYPE}"
                 ;;
+            libcxx)
+                # Reuse LLVM's build type.
+                echo "--config ${LLVM_BUILD_TYPE}"
+                ;;
             swift)
                 echo "--config ${SWIFT_BUILD_TYPE}"
                 ;;
@@ -2163,6 +2186,12 @@ for host in "${ALL_HOSTS[@]}"; do
                     "${llvm_cmake_options[@]}"
                 )
 
+                if [[ ! -z "${CLANG_TOOLS_EXTRA_SOURCE_DIR}" ]] ; then
+                  cmake_options+=(
+                    -DLLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR="${CLANG_TOOLS_EXTRA_SOURCE_DIR}"
+                  )
+                fi
+
                 if [[ "${BUILD_TOOLCHAIN_ONLY}" ]]; then
                     cmake_options+=(
                     -DLLVM_BUILD_TOOLS=NO
@@ -2202,6 +2231,22 @@ for host in "${ALL_HOSTS[@]}"; do
 
                 ;;
 
+            libcxx)
+                build_targets=(cxx-headers)
+                cmake_options=(
+                    "${cmake_options[@]}"
+                    -DCMAKE_C_FLAGS="$(llvm_c_flags ${host})"
+                    -DCMAKE_CXX_FLAGS="$(llvm_c_flags ${host})"
+                    -DCMAKE_C_FLAGS_RELWITHDEBINFO="-O2 -DNDEBUG"
+                    -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O2 -DNDEBUG"
+                    -DCMAKE_BUILD_TYPE:STRING="${LLVM_BUILD_TYPE}"
+                    -DLLVM_INCLUDE_DOCS:BOOL=TRUE
+                    -DLLVM_CONFIG_PATH="$(build_directory "${LOCAL_HOST}" llvm)/bin/llvm-config"
+                    "${llvm_cmake_options[@]}"
+                )
+
+                ;;
+
             swift)
 
                 if [[ ! "${SKIP_BUILD_ANDROID}" ]]; then
@@ -3028,6 +3073,9 @@ for host in "${ALL_HOSTS[@]}"; do
             llvm)
                 continue # We don't test LLVM
                 ;;
+            libcxx)
+                continue # We don't test libc++
+                ;;
             swift)
                 executable_target=
                 results_targets=
@@ -3524,6 +3572,9 @@ for host in "${ALL_HOSTS[@]}"; do
                 fi
                 INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/;/ install-/g')
                 ;;
+            libcxx)
+                INSTALL_TARGETS=install-cxx-headers
+                ;;
             swift)
                 if [[ -z "${INSTALL_SWIFT}" ]] ; then
                     continue