SwiftSyntax: Teach SwiftSyntax to use SourceKitd to serialize syntax trees. (#14424)

When using SwiftSyntax as a standalone tool, we invoke Swiftc
internally to get serialized syntax trees. This is not ideal for
several reasons: (1) we have to hard-code the relative path of swiftc
to invoke it; (2) we have to rely on standard input/output to pass the
tree across the process boundaries; and (3) we have to maintain two
different ways to get syntax tree (swiftc and sourcekitd).

This patch attempts to teach SwiftSyntax to use SourceKitd to get the
tree just like other clients. We first add a SourceKitd client library
written in Swift; and next teach SwiftSyntax to adopt this SourceKitd
client-side library. For platforms other than MacOS, we still use Swiftc
to get syntax trees. This client library also allows us to add 
SourceKitd tests in Swift.

This patch also re-enables several flaky tests.

Author: nkcsgexi

cmake/modules/AddSwift.cmake

@@ -255,7 +255,7 @@ function(_add_variant_c_compile_flags)
   else()
     list(APPEND result "-DNDEBUG")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-DSWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -320,7 +320,7 @@ function(_add_variant_swift_compile_flags
   if (SWIFT_ENABLE_GUARANTEED_NORMAL_ARGUMENTS)
     list(APPEND result "-Xfrontend" "-enable-guaranteed-normal-arguments")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-D" "SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -1605,7 +1605,7 @@ function(add_swift_library name)
         if(SWIFTLIB_IS_SDK_OVERLAY)
           list(APPEND swiftlib_swift_compile_flags_all "-Fsystem" "${SWIFT_SDK_${sdk}_PATH}/System/Library/PrivateFrameworks/")
         endif()
-       
+
        if("${sdk}" STREQUAL "IOS_SIMULATOR")
          if("${name}" STREQUAL "swiftMediaPlayer")
            message("DISABLING AUTOLINK FOR swiftMediaPlayer")
@@ -1854,7 +1854,7 @@ function(add_swift_library name)
           LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX}
           RUNTIME DESTINATION bin)
       swift_is_installing_component(dev is_installing)
-      
+
       if(NOT is_installing)
         set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${name})
       else()
@@ -2177,7 +2177,7 @@ function(add_swift_host_tool executable)
 
     swift_is_installing_component(${ADDSWIFTHOSTTOOL_SWIFT_COMPONENT}
       is_installing)
-  
+
     if(NOT is_installing)
       set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${executable})
     else()

test/SwiftSyntax/ParseFile.swift

@@ -2,7 +2,6 @@
 // REQUIRES: executable_test
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
-// REQUIRES: rdar36740859
 
 import Foundation
 import StdlibUnittest

test/SwiftSyntax/VisitorTest.swift

@@ -3,9 +3,6 @@
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
 
-// FIXME: This test fails occassionally in CI with invalid json.
-// REQUIRES: disabled
-
 import StdlibUnittest
 import Foundation
 import SwiftSyntax

test/lit.cfg

@@ -286,6 +286,7 @@ else:
     test_resource_dir = os.path.join(config.swift_lib_dir, 'swift')
     resource_dir_opt = ""
 stdlib_resource_dir_opt = resource_dir_opt
+sourcekitd_framework_dir = config.swift_lib_dir
 lit_config.note('Using resource dir: ' + test_resource_dir)
 
 # Parse the variant triple.
@@ -670,12 +671,13 @@ if run_vendor == 'apple':
             (run_cpu, run_os, run_vers, clang_mcp_opt))
 
         config.target_build_swift = (
-            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s"
+            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s -F %s -Xlinker -rpath -Xlinker %s"
             % (xcrun_prefix, config.swiftc, target_options,
                extra_frameworks_dir, extra_frameworks_dir,
                sdk_overlay_linker_opt, config.swift_test_options,
                config.swift_driver_test_options,
-               swift_execution_tests_extra_flags))
+               swift_execution_tests_extra_flags, sourcekitd_framework_dir,
+               sourcekitd_framework_dir))
         config.target_run = ""
 
         if 'interpret' in lit_config.params:

tools/CMakeLists.txt

@@ -16,19 +16,27 @@ add_swift_tool_subdirectory(swift-api-digester)
 add_swift_tool_subdirectory(swift-syntax-test)
 add_swift_tool_subdirectory(swift-refactor)
 
+if (SWIFT_BUILD_SOURCEKIT AND SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  set(BUILD_SOURCEKIT_SWIFT_CLIENT TRUE)
+else()
+  set(BUILD_SOURCEKIT_SWIFT_CLIENT FALSE)
+endif()
+
 if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
+  if(BUILD_SOURCEKIT_SWIFT_CLIENT)
+    add_subdirectory(SwiftSourceKitClient)
+  endif()
 endif()
 
-
 if(SWIFT_HOST_VARIANT STREQUAL "macosx")
   # Only build Darwin-specific tools when deploying to OS X.
   add_swift_tool_subdirectory(swift-stdlib-tool)
 
   # SwiftSyntax depends on both the standard library (because it's a
   # Swift module), and the SDK overlays (because it depends on Foundation).
   # Ensure we only build SwiftSyntax when we're building both.
-  if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  if(BUILD_SOURCEKIT_SWIFT_CLIENT)
     add_subdirectory(SwiftSyntax)
   endif()
 endif()

tools/SwiftSourceKitClient/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(EXTRA_COMPILE_FLAGS "-F" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+set(EXTRA_LINKER_FLAGS "-Xlinker" "-rpath" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}"
+    "-Xlinker" "-F" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+
+add_swift_library(swiftSwiftSourceKit SHARED
+  # This file should be listed the first.  Module name is inferred from the
+  # filename.
+  SourceKitdClient.swift
+  SourceKitdRequest.swift
+  SourceKitdResponse.swift
+  SourceKitdUID.swift
+
+  SWIFT_COMPILE_FLAGS ${EXTRA_COMPILE_FLAGS}
+  LINK_FLAGS ${EXTRA_LINKER_FLAGS}
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT swift-syntax
+  TARGET_SDKS OSX
+  IS_STDLIB)

tools/SwiftSourceKitClient/SourceKitdClient.swift

@@ -0,0 +1,28 @@
+//===--------------------- SourceKitdClient.swift -------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a wrapper of SourceKitd service.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+
+public class SourceKitdService {
+
+  public init() {
+    sourcekitd_initialize()
+  }
+  deinit {
+    sourcekitd_shutdown()
+  }
+  public func sendSyn(request: SourceKitdRequest) -> SourceKitdResponse {
+    return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+  }
+}

tools/SwiftSourceKitClient/SourceKitdRequest.swift

@@ -0,0 +1,154 @@
+//===--------------- SourceKitdRequest.swift ------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a convenient way to build a sourcekitd request.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+
+public struct SourceKitdRequest: CustomStringConvertible {
+
+  public class Dictionary: CustomStringConvertible {
+    let dict: sourcekitd_object_t
+
+    public init() {
+      dict = sourcekitd_request_dictionary_create(nil, nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(UnsafeMutableRawPointer(dict))
+    }
+
+    public func add(_ key: SourceKitdUID, value: String) {
+      sourcekitd_request_dictionary_set_string(dict, key.uid, value)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Int) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, Int64(value))
+    }
+
+    public func add(_ key: SourceKitdUID, value: SourceKitdUID) {
+      sourcekitd_request_dictionary_set_uid(dict, key.uid, value.uid)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Array) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.arr)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Dictionary) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.dict)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Bool) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, value ? 1 : 0)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(dict)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  public class Array: CustomStringConvertible {
+    let arr: sourcekitd_object_t
+    private let Append: Int = -1
+
+    public init() {
+      arr = sourcekitd_request_array_create(nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(arr)
+    }
+
+    public func add(_ value: String) {
+      sourcekitd_request_array_set_string(arr, Append, value)
+    }
+
+    public func add(_ value: Int) {
+      sourcekitd_request_array_set_int64(arr, Append, Int64(value))
+    }
+
+    public func add(_ value: SourceKitdUID) {
+      sourcekitd_request_array_set_uid(arr, Append, value.uid)
+    }
+
+    public func add(_ value: Dictionary) {
+      sourcekitd_request_array_set_value(arr, Append, value.dict)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(arr)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  private let req = Dictionary()
+
+  public init(uid: SourceKitdUID) {
+    req.add(SourceKitdUID.key_request, value: uid)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: String) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: Int) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: SourceKitdUID) {
+    req.add(key, value: value)
+  }
+
+  public func addArrayParameter(_ key: SourceKitdUID) -> Array {
+    let arr = Array()
+    req.add(key, value: arr)
+    return arr
+  }
+
+  public func addDictionaryParameter(_ key: SourceKitdUID) -> Dictionary {
+    let dict = Dictionary()
+    req.add(key, value: dict)
+    return dict
+  }
+
+  public var description: String {
+    return req.description
+  }
+
+  public var rawRequest: sourcekitd_object_t {
+    return req.dict
+  }
+
+  public func addCompilerArgsToRequest(_ compilerArguments: [String]?,
+                                       _ bufferName: String? = nil) {
+    let args = self.addArrayParameter(SourceKitdUID.key_compilerargs)
+
+    if let compilerArguments = compilerArguments {
+      for argument in compilerArguments {
+        switch argument {
+        // Exclude some arguments which SourceKit doesn't want or need.
+        case "-Xfrontend":
+          break
+        default:
+          args.add(argument)
+        }
+      }
+    }
+  }
+}