Re-apply "SwiftSyntax: Teach SwiftSyntax to use SourceKitd to serialize syntax trees. (#14424)" (#14465)

Author: nkcsgexi

CMakeLists.txt

@@ -11,6 +11,8 @@ set_property(GLOBAL PROPERTY JOB_POOLS local_jobs=${localhost_logical_cores})
 # Put linking in that category
 set_property(GLOBAL PROPERTY JOB_POOL_LINK local_jobs)
 
+ENABLE_LANGUAGE(C)
+
 # First include general CMake utilities.
 include(SwiftUtils)
 include(CheckSymbolExists)
@@ -748,10 +750,10 @@ if(swift_build_android AND NOT "${SWIFT_ANDROID_NDK_PATH}" STREQUAL "")
 
   set(SWIFT_ANDROID_PREBUILT_PATH
       "${SWIFT_ANDROID_NDK_PATH}/toolchains/arm-linux-androideabi-${SWIFT_ANDROID_NDK_GCC_VERSION}/prebuilt/${_swift_android_prebuilt_suffix}")
-  
+
   # Resolve the correct linker based on the file name of CMAKE_LINKER (being 'ld' or 'ld.gold' the options)
   get_filename_component(SWIFT_ANDROID_LINKER_NAME "${CMAKE_LINKER}" NAME)
-  set(SWIFT_SDK_ANDROID_ARCH_armv7_LINKER 
+  set(SWIFT_SDK_ANDROID_ARCH_armv7_LINKER
       "${SWIFT_ANDROID_NDK_PATH}/toolchains/arm-linux-androideabi-${SWIFT_ANDROID_NDK_GCC_VERSION}/prebuilt/${_swift_android_prebuilt_suffix}/bin/arm-linux-androideabi-${SWIFT_ANDROID_LINKER_NAME}")
 
   configure_sdk_unix(ANDROID "Android" "android" "android" "armv7" "armv7-none-linux-androideabi" "${SWIFT_ANDROID_SDK_PATH}")
@@ -879,7 +881,7 @@ endif()
 ###############
 #
 # We have to include stdlib/ before tools/.
-# Do not move add_subdirectory(stdlib) after add_subdirectory(tools)!                                             
+# Do not move add_subdirectory(stdlib) after add_subdirectory(tools)!
 #
 # We must include stdlib/ before tools/ because stdlib/CMakeLists.txt
 # declares the swift-stdlib-* set of targets. These targets will then
@@ -898,7 +900,7 @@ add_subdirectory(stdlib)
 if(SWIFT_INCLUDE_TOOLS)
   add_subdirectory(include)
   add_subdirectory(lib)
-  
+
   # Always include this after including stdlib/!
   # Refer to the large comment above the add_subdirectory(stdlib) call.
   # https://bugs.swift.org/browse/SR-5975

cmake/modules/AddSwift.cmake

@@ -255,7 +255,7 @@ function(_add_variant_c_compile_flags)
   else()
     list(APPEND result "-DNDEBUG")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-DSWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -320,7 +320,7 @@ function(_add_variant_swift_compile_flags
   if (SWIFT_ENABLE_GUARANTEED_NORMAL_ARGUMENTS)
     list(APPEND result "-Xfrontend" "-enable-guaranteed-normal-arguments")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-D" "SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -1605,7 +1605,7 @@ function(add_swift_library name)
         if(SWIFTLIB_IS_SDK_OVERLAY)
           list(APPEND swiftlib_swift_compile_flags_all "-Fsystem" "${SWIFT_SDK_${sdk}_PATH}/System/Library/PrivateFrameworks/")
         endif()
-       
+
        if("${sdk}" STREQUAL "IOS_SIMULATOR")
          if("${name}" STREQUAL "swiftMediaPlayer")
            message("DISABLING AUTOLINK FOR swiftMediaPlayer")
@@ -1854,7 +1854,7 @@ function(add_swift_library name)
           LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX}
           RUNTIME DESTINATION bin)
       swift_is_installing_component(dev is_installing)
-      
+
       if(NOT is_installing)
         set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${name})
       else()
@@ -2177,7 +2177,7 @@ function(add_swift_host_tool executable)
 
     swift_is_installing_component(${ADDSWIFTHOSTTOOL_SWIFT_COMPONENT}
       is_installing)
-  
+
     if(NOT is_installing)
       set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${executable})
     else()

test/SwiftSyntax/DeserializeFile.swift

@@ -6,6 +6,7 @@
 import StdlibUnittest
 import Foundation
 import SwiftSyntax
+import SwiftSourceKit
 
 func getInput(_ file: String) -> URL {
   var result = URL(fileURLWithPath: #file)

test/SwiftSyntax/ParseFile.swift

@@ -2,11 +2,11 @@
 // REQUIRES: executable_test
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
-// REQUIRES: rdar36740859
 
 import Foundation
 import StdlibUnittest
 import SwiftSyntax
+import SwiftSourceKit
 
 var ParseFile = TestSuite("ParseFile")
 
@@ -31,7 +31,8 @@ ParseFile.test("ParseSingleFile") {
   let currentFile = URL(fileURLWithPath: #file)
   expectDoesNotThrow({
     let currentFileContents = try String(contentsOf: currentFile)
-    let parsed = try SourceFileSyntax.parse(currentFile)
+    let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
+      SourceKitdService.encodeSourceFileSyntax(currentFile))
     expectEqual("\(parsed)", currentFileContents)
   })
 }

test/SwiftSyntax/VisitorTest.swift

@@ -3,12 +3,10 @@
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
 
-// FIXME: This test fails occassionally in CI with invalid json.
-// REQUIRES: disabled
-
 import StdlibUnittest
 import Foundation
 import SwiftSyntax
+import SwiftSourceKit
 
 func getInput(_ file: String) -> URL {
   var result = URL(fileURLWithPath: #file)
@@ -29,7 +27,8 @@ VisitorTests.test("Basic") {
     }
   }
   expectDoesNotThrow({
-    let parsed = try SourceFileSyntax.parse(getInput("visitor.swift"))
+    let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
+      SourceKitdService.encodeSourceFileSyntax(getInput("visitor.swift")))
     let counter = FuncCounter()
     let hashBefore = parsed.hashValue
     counter.visit(parsed)

test/lit.cfg

@@ -286,6 +286,7 @@ else:
     test_resource_dir = os.path.join(config.swift_lib_dir, 'swift')
     resource_dir_opt = ""
 stdlib_resource_dir_opt = resource_dir_opt
+sourcekitd_framework_dir = config.swift_lib_dir
 lit_config.note('Using resource dir: ' + test_resource_dir)
 
 # Parse the variant triple.
@@ -670,12 +671,13 @@ if run_vendor == 'apple':
             (run_cpu, run_os, run_vers, clang_mcp_opt))
 
         config.target_build_swift = (
-            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s"
+            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s -F %s -Xlinker -rpath -Xlinker %s"
             % (xcrun_prefix, config.swiftc, target_options,
                extra_frameworks_dir, extra_frameworks_dir,
                sdk_overlay_linker_opt, config.swift_test_options,
                config.swift_driver_test_options,
-               swift_execution_tests_extra_flags))
+               swift_execution_tests_extra_flags, sourcekitd_framework_dir,
+               sourcekitd_framework_dir))
         config.target_run = ""
 
         if 'interpret' in lit_config.params:

tools/CMakeLists.txt

@@ -1,3 +1,14 @@
+include(CheckIncludeFiles)
+check_include_files("xpc/xpc.h" HAVE_XPC_H)
+
+swift_is_installing_component(sourcekit-inproc SOURCEKIT_INSTALLING_INPROC)
+
+if(HAVE_XPC_H AND SWIFT_BUILD_SOURCEKIT AND NOT SOURCEKIT_INSTALLING_INPROC)
+  set(BUILD_SOURCEKIT_XPC_SERVICE TRUE)
+else()
+  set(BUILD_SOURCEKIT_XPC_SERVICE FALSE)
+endif()
+
 # Add generated libSyntax headers to global dependencies.
 list(APPEND LLVM_COMMON_DEPENDS swift-syntax-generated-headers)
 
@@ -16,19 +27,27 @@ add_swift_tool_subdirectory(swift-api-digester)
 add_swift_tool_subdirectory(swift-syntax-test)
 add_swift_tool_subdirectory(swift-refactor)
 
+if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  set(BUILD_FOUNDATION TRUE)
+else()
+  set(BUILD_FOUNDATION FALSE)
+endif()
+
 if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
+  if(BUILD_SOURCEKIT_XPC_SERVICE AND BUILD_FOUNDATION)
+    add_subdirectory(SwiftSourceKitClient)
+  endif()
 endif()
 
-
 if(SWIFT_HOST_VARIANT STREQUAL "macosx")
   # Only build Darwin-specific tools when deploying to OS X.
   add_swift_tool_subdirectory(swift-stdlib-tool)
 
   # SwiftSyntax depends on both the standard library (because it's a
   # Swift module), and the SDK overlays (because it depends on Foundation).
   # Ensure we only build SwiftSyntax when we're building both.
-  if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  if(BUILD_FOUNDATION)
     add_subdirectory(SwiftSyntax)
   endif()
 endif()

tools/SourceKit/CMakeLists.txt

@@ -24,8 +24,6 @@ configure_file(
 set(SOURCEKIT_DEPLOYMENT_OS "${SWIFT_HOST_VARIANT}")
 set(SOURCEKIT_DEPLOYMENT_TARGET "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_DEPLOYMENT_VERSION}")
 
-swift_is_installing_component(sourcekit-inproc SOURCEKIT_INSTALLING_INPROC)
-
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND NOT CMAKE_CROSSCOMPILING)
   set(CMAKE_OSX_SYSROOT "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_PATH}")
   set(CMAKE_OSX_ARCHITECTURES "${SWIFT_HOST_VARIANT_ARCH}")

tools/SourceKit/tools/sourcekitd/bin/CMakeLists.txt

@@ -1,4 +1,4 @@
   add_subdirectory(InProc)
-if (HAVE_XPC_H)
+if(HAVE_XPC_H)
   add_subdirectory(XPC)
 endif()

tools/SourceKit/tools/sourcekitd/lib/API/CMakeLists.txt

@@ -1,7 +1,3 @@
-include(CheckIncludeFiles)
-
-check_include_files("xpc/xpc.h" HAVE_XPC_H)
-
 # If we were going to build for APPLE but don't have XPC, just build inproc.
 if(APPLE AND NOT HAVE_XPC_H)
   set(SWIFT_SOURCEKIT_USE_INPROC_LIBRARY TRUE)

tools/SwiftSourceKitClient/CMakeLists.txt

@@ -0,0 +1,17 @@
+set(EXTRA_COMPILE_FLAGS "-F" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+set(EXTRA_LINKER_FLAGS "-Xlinker" "-rpath" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}"
+    "-Xlinker" "-F" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+
+add_swift_library(swiftSwiftSourceKit SHARED
+  SourceKitdClient.swift
+  SourceKitdRequest.swift
+  SourceKitdResponse.swift
+  SourceKitdUID.swift
+
+  DEPENDS SourceKitService
+  SWIFT_COMPILE_FLAGS ${EXTRA_COMPILE_FLAGS}
+  LINK_FLAGS ${EXTRA_LINKER_FLAGS}
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT swift-syntax
+  TARGET_SDKS OSX
+  IS_STDLIB)

tools/SwiftSourceKitClient/SourceKitdClient.swift

@@ -0,0 +1,48 @@
+//===--------------------- SourceKitdClient.swift -------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a wrapper of SourceKitd service.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+import Foundation
+
+public class SourceKitdService {
+
+  public init() {
+    sourcekitd_initialize()
+  }
+  deinit {
+    sourcekitd_shutdown()
+  }
+  public func sendSyn(request: SourceKitdRequest) -> SourceKitdResponse {
+    return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+  }
+}
+
+extension SourceKitdService {
+  /// Parses the Swift file at the provided URL into a `Syntax` tree in Json
+  /// serialization format by querying SourceKitd service.
+  /// - Parameter url: The URL you wish to parse.
+  /// - Returns: The syntax tree in Json format string.
+  public static func encodeSourceFileSyntax(_ url: URL) throws -> String {
+    let Service = SourceKitdService()
+    let Request = SourceKitdRequest(uid: .source_request_editor_open)
+    let Path = url.path
+    Request.addParameter(.key_sourcefile, value: Path)
+    Request.addParameter(.key_name, value: Path)
+    Request.addParameter(.key_enable_syntax_tree, value: 1)
+
+    // FIXME: SourceKitd error handling.
+    let Resp = Service.sendSyn(request: Request)
+    return Resp.value.getString(.key_serialized_syntax_tree)
+  }
+}