Optional support for round-trip testing the new Swift parser.

When enabled, compile in support for round-trip testing the new
SwiftSyntax-provided Swift parser alongside the existing parser. Right
now, this means parsing every source file with the new parser and
ensuring that the resulting syntax tree can reproduce the input source
precisely. Over time, this is expected to grow.

Opt in to this behavior by passing the following to build-script:

    build-script --early-swiftsyntax --extra-cmake-options=-DSWIFT_SWIFT_PARSER_MODE:STRING=ROUNDTRIP

Author: DougGregor

CMakeLists.txt

@@ -868,6 +868,23 @@ if(XCODE)
   set(SWIFT_SDKS "OSX")
 endif()
 
+# When we have the early SwiftSyntax build, we can include its parser.
+if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR)
+  # An option that specifies how the SwiftSyntax-based Swift parser is
+  # integrated.
+  option(SWIFT_SWIFT_PARSER_MODE [=[
+How to use the SwiftSyntax-based Swift parser. Possible values are
+    OFF:           do not use this parser at all
+    ROUNDTRIP:     ensure that the SwiftSyntax-based Swift parser round-trips
+]=] OFF)
+
+  if(SWIFT_SWIFT_PARSER_MODE)
+    set(SWIFT_SWIFT_PARSER TRUE)
+    include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake)
+  endif()
+endif()
+
+
 # FIXME: the parameters we specify in SWIFT_SDKS are lacking architecture specifics,
 # so we need to hard-code it. For example, the SDK for Android is just 'ANDROID',
 # and we have to specify SWIFT_SDK_ANDROID_ARCHITECTURES separately.

cmake/modules/AddSwift.cmake

@@ -608,6 +608,11 @@ function(add_swift_host_library name)
 
   translate_flags(ASHL "${options}")
 
+  # Once the new Swift parser is linked, everything has Swift modules.
+  if (SWIFT_SWIFT_PARSER AND ASHL_SHARED)
+    set(ASHL_HAS_SWIFT_MODULES ON)
+  endif()
+
   if(NOT ASHL_SHARED AND NOT ASHL_STATIC AND NOT ASHL_OBJECT)
     message(FATAL_ERROR "One of SHARED/STATIC/OBJECT must be specified")
   endif()
@@ -834,6 +839,11 @@ function(add_swift_host_tool executable)
       JOB_POOL_LINK swift_link_job_pool)
   endif()
 
+  # Once the new Swift parser is linked in, every host tool has Swift modules.
+  if (SWIFT_SWIFT_PARSER)
+    set(ASHT_HAS_SWIFT_MODULES ON)
+  endif()
+
   if (ASHT_HAS_SWIFT_MODULES)
       _add_swift_runtime_link_flags(${executable} "../lib" "${ASHT_BOOTSTRAPPING}")
   endif()

lib/Parse/CMakeLists.txt

@@ -35,6 +35,34 @@ target_link_libraries(swiftParse PRIVATE
   swiftSyntaxParse
 )
 
+if (SWIFT_SWIFT_PARSER)
+  # Link against the SwiftSyntax parser and libraries it depends on. The actual
+  # formulation of this is a hack to work around a CMake bug in Ninja file
+  # generation that results in multiple Ninja targets producing the same file in
+  # a downstream SourceKit target. This should be expressed as:
+  #
+  #   target_link_libraries(swiftParse
+  #     PRIVATE
+  #     SwiftSyntax::SwiftParser
+  #   )
+  target_link_libraries(swiftParse
+    PRIVATE
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftParser>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftDiagnostics>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntax>
+  )
+
+  target_include_directories(swiftParse
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../swift-syntax/Sources/SwiftParser
+  )
+
+  target_compile_definitions(swiftParse
+    PRIVATE
+    SWIFT_SWIFT_PARSER_ROUNDTRIP
+  )
+endif()
+
 add_dependencies(swiftParse swift-parse-syntax-generated-headers)
 
 set_swift_llvm_is_available(swiftParse)

lib/Parse/ParseRequests.cpp

@@ -26,6 +26,10 @@
 #include "swift/Syntax/SyntaxNodes.h"
 #include "swift/SyntaxParse/SyntaxTreeCreator.h"
 
+#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP
+#include "SwiftParserCompilerSupport.h"
+#endif
+
 using namespace swift;
 
 namespace swift {
@@ -189,6 +193,22 @@ SourceFileParsingResult ParseSourceFileRequest::evaluate(Evaluator &evaluator,
   if (auto tokens = parser.takeTokenReceiver()->finalize())
     tokensRef = ctx.AllocateCopy(*tokens);
 
+#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP
+  if (ctx.SourceMgr.getCodeCompletionBufferID() != bufferID) {
+    auto bufferRange = ctx.SourceMgr.getRangeForBuffer(*bufferID);
+    unsigned int flags = SPCC_RoundTrip;
+
+    int roundTripResult =
+      swift_parser_consistencyCheck(
+        bufferRange.str().data(), bufferRange.getByteLength(),
+        SF->getFilename().str().c_str(), flags);
+
+    // FIXME: Produce an error on round-trip failure.
+    if (roundTripResult)
+      abort();
+  }
+#endif
+
   return SourceFileParsingResult{ctx.AllocateCopy(decls), tokensRef,
                                  parser.CurrentTokenHash, syntaxRoot};
 }

utils/swift_build_support/swift_build_support/products/swift.py

@@ -10,14 +10,16 @@
 #
 # ----------------------------------------------------------------------------
 
+import os
+
 from . import cmark
 from . import earlyswiftdriver
+from . import earlyswiftsyntax
 from . import libcxx
 from . import llvm
 from . import product
 from ..cmake import CMakeOptions
 
-
 class Swift(product.Product):
 
     def __init__(self, args, toolchain, source_dir, build_dir):
@@ -55,6 +57,9 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         # Add experimental distributed flag.
         self.cmake_options.extend(self._enable_experimental_distributed)
 
+        # Add path for the early SwiftSyntax build.
+        self.cmake_options.extend(self._early_swiftsyntax_flags)
+
         # Add static vprintf flag
         self.cmake_options.extend(self._enable_stdlib_static_vprintf)
 
@@ -167,6 +172,18 @@ def _enable_experimental_distributed(self):
         return [('SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED:BOOL',
                  self.args.enable_experimental_distributed)]
 
+    @property
+    def _early_swiftsyntax_flags(self):
+        result=[]
+        if self.args.build_early_swiftsyntax:
+            build_root = os.path.dirname(self.build_dir)
+            early_swiftsyntax_build_dir = os.path.join(
+                '..', build_root, '%s-%s' % ('earlyswiftsyntax',
+                                             self.args.host_target))
+            result.append(('SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR:PATH',
+                           early_swiftsyntax_build_dir))
+        return result
+
     @property
     def _enable_stdlib_static_vprintf(self):
         return [('SWIFT_STDLIB_STATIC_PRINT',
@@ -194,7 +211,11 @@ def _swift_tools_ld64_lto_codegen_only_for_supporting_targets(self):
 
     @classmethod
     def get_dependencies(cls):
-        return [cmark.CMark,
+        deps = [cmark.CMark,
                 earlyswiftdriver.EarlySwiftDriver,
                 llvm.LLVM,
                 libcxx.LibCXX]
+        if self.args.build_early_swiftsyntax:
+            deps.append(earlyswiftsyntax.EarlySwiftSyntax)
+
+        return deps