Merge pull request #60888 from DougGregor/round-trip-new-swift-parser

Optional support for round-trip testing the new Swift parser.

Author: DougGregor

CMakeLists.txt

@@ -868,6 +868,23 @@ if(XCODE)
   set(SWIFT_SDKS "OSX")
 endif()
 
+# When we have the early SwiftSyntax build, we can include its parser.
+if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR)
+  # An option that specifies how the SwiftSyntax-based Swift parser is
+  # integrated.
+  option(SWIFT_SWIFT_PARSER_MODE [=[
+How to use the SwiftSyntax-based Swift parser. Possible values are
+    OFF:           do not use this parser at all
+    ROUNDTRIP:     ensure that the SwiftSyntax-based Swift parser round-trips
+]=] OFF)
+
+  if(SWIFT_SWIFT_PARSER_MODE)
+    set(SWIFT_SWIFT_PARSER TRUE)
+    include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake)
+  endif()
+endif()
+
+
 # FIXME: the parameters we specify in SWIFT_SDKS are lacking architecture specifics,
 # so we need to hard-code it. For example, the SDK for Android is just 'ANDROID',
 # and we have to specify SWIFT_SDK_ANDROID_ARCHITECTURES separately.

cmake/modules/AddSwift.cmake

@@ -504,7 +504,7 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
 
     # Workaround to make lldb happy: we have to explicitly add all swift compiler modules
     # to the linker command line.
-    set(swift_ast_path_flags "-Wl")
+    set(swift_ast_path_flags " -Wl")
     get_property(modules GLOBAL PROPERTY swift_compiler_modules)
     foreach(module ${modules})
       get_target_property(module_file "SwiftModule${module}" "module_file")
@@ -608,6 +608,11 @@ function(add_swift_host_library name)
 
   translate_flags(ASHL "${options}")
 
+  # Once the new Swift parser is linked, everything has Swift modules.
+  if (SWIFT_SWIFT_PARSER AND ASHL_SHARED)
+    set(ASHL_HAS_SWIFT_MODULES ON)
+  endif()
+
   if(NOT ASHL_SHARED AND NOT ASHL_STATIC AND NOT ASHL_OBJECT)
     message(FATAL_ERROR "One of SHARED/STATIC/OBJECT must be specified")
   endif()
@@ -834,6 +839,11 @@ function(add_swift_host_tool executable)
       JOB_POOL_LINK swift_link_job_pool)
   endif()
 
+  # Once the new Swift parser is linked in, every host tool has Swift modules.
+  if (SWIFT_SWIFT_PARSER)
+    set(ASHT_HAS_SWIFT_MODULES ON)
+  endif()
+
   if (ASHT_HAS_SWIFT_MODULES)
       _add_swift_runtime_link_flags(${executable} "../lib" "${ASHT_BOOTSTRAPPING}")
   endif()

cmake/modules/AddSwiftUnittests.cmake

@@ -78,5 +78,9 @@ function(add_swift_unittest test_dirname)
         LINK_FLAGS " -fsanitize=thread")
     endif()
   endif()
+
+  if (SWIFT_SWIFT_PARSER)
+    _add_swift_runtime_link_flags(${test_dirname} "../../lib" "")
+  endif()
 endfunction()
 

lib/Parse/CMakeLists.txt

@@ -35,6 +35,34 @@ target_link_libraries(swiftParse PRIVATE
   swiftSyntaxParse
 )
 
+if (SWIFT_SWIFT_PARSER)
+  # Link against the SwiftSyntax parser and libraries it depends on. The actual
+  # formulation of this is a hack to work around a CMake bug in Ninja file
+  # generation that results in multiple Ninja targets producing the same file in
+  # a downstream SourceKit target. This should be expressed as:
+  #
+  #   target_link_libraries(swiftParse
+  #     PRIVATE
+  #     SwiftSyntax::SwiftParser
+  #   )
+  target_link_libraries(swiftParse
+    PRIVATE
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftParser>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftDiagnostics>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntax>
+  )
+
+  target_include_directories(swiftParse
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../swift-syntax/Sources/SwiftParser
+  )
+
+  target_compile_definitions(swiftParse
+    PRIVATE
+    SWIFT_SWIFT_PARSER_ROUNDTRIP
+  )
+endif()
+
 add_dependencies(swiftParse swift-parse-syntax-generated-headers)
 
 set_swift_llvm_is_available(swiftParse)

lib/Parse/ParseRequests.cpp

@@ -26,6 +26,10 @@
 #include "swift/Syntax/SyntaxNodes.h"
 #include "swift/SyntaxParse/SyntaxTreeCreator.h"
 
+#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP
+#include "SwiftParserCompilerSupport.h"
+#endif
+
 using namespace swift;
 
 namespace swift {
@@ -189,6 +193,22 @@ SourceFileParsingResult ParseSourceFileRequest::evaluate(Evaluator &evaluator,
   if (auto tokens = parser.takeTokenReceiver()->finalize())
     tokensRef = ctx.AllocateCopy(*tokens);
 
+#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP
+  if (ctx.SourceMgr.getCodeCompletionBufferID() != bufferID) {
+    auto bufferRange = ctx.SourceMgr.getRangeForBuffer(*bufferID);
+    unsigned int flags = SPCC_RoundTrip;
+
+    int roundTripResult =
+      swift_parser_consistencyCheck(
+        bufferRange.str().data(), bufferRange.getByteLength(),
+        SF->getFilename().str().c_str(), flags);
+
+    // FIXME: Produce an error on round-trip failure.
+    if (roundTripResult)
+      abort();
+  }
+#endif
+
   return SourceFileParsingResult{ctx.AllocateCopy(decls), tokensRef,
                                  parser.CurrentTokenHash, syntaxRoot};
 }

utils/swift_build_support/swift_build_support/products/earlyswiftdriver.py

@@ -12,6 +12,7 @@
 
 import os
 
+from . import earlyswiftsyntax
 from . import product
 from .. import shell
 from .. import toolchain
@@ -57,7 +58,11 @@ def should_build(self, host_target):
 
     @classmethod
     def get_dependencies(cls):
-        return []
+        # FIXME: This isn't a real dependency, but is necessary to linearize the
+        # dependency graph from Swift to EarlySwiftSyntax. If we properly
+        # express the dependency from Swift -> EarlySwiftSyntax, build_graph.py
+        # asserts that there are multiple roots to the graph.
+        return [earlyswiftsyntax.EarlySwiftSyntax]
 
     def should_clean(self, host_target):
         return self.args.clean_early_swift_driver

utils/swift_build_support/swift_build_support/products/swift.py

@@ -10,6 +10,8 @@
 #
 # ----------------------------------------------------------------------------
 
+import os
+
 from . import cmark
 from . import earlyswiftdriver
 from . import libcxx
@@ -55,6 +57,9 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         # Add experimental distributed flag.
         self.cmake_options.extend(self._enable_experimental_distributed)
 
+        # Add path for the early SwiftSyntax build.
+        self.cmake_options.extend(self._early_swiftsyntax_flags)
+
         # Add static vprintf flag
         self.cmake_options.extend(self._enable_stdlib_static_vprintf)
 
@@ -167,6 +172,18 @@ def _enable_experimental_distributed(self):
         return [('SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED:BOOL',
                  self.args.enable_experimental_distributed)]
 
+    @property
+    def _early_swiftsyntax_flags(self):
+        result = []
+        if self.args.build_early_swiftsyntax:
+            build_root = os.path.dirname(self.build_dir)
+            early_swiftsyntax_build_dir = os.path.join(
+                '..', build_root, '%s-%s' % ('earlyswiftsyntax',
+                                             self.args.host_target))
+            result.append(('SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR:PATH',
+                           early_swiftsyntax_build_dir))
+        return result
+
     @property
     def _enable_stdlib_static_vprintf(self):
         return [('SWIFT_STDLIB_STATIC_PRINT',

utils/swift_build_support/tests/products/test_swift.py

@@ -56,6 +56,7 @@ def setUp(self):
             enable_experimental_differentiable_programming=False,
             enable_experimental_concurrency=False,
             enable_experimental_distributed=False,
+            build_early_swiftsyntax=False,
             build_swift_stdlib_static_print=False,
             build_swift_stdlib_unicode_data=True,
             swift_freestanding_is_darwin=False,