SwiftSyntax: Teach SwiftSyntax to use SourceKit to serialize syntax trees.

cmake/modules/AddSwift.cmake

@@ -255,7 +255,7 @@ function(_add_variant_c_compile_flags)
   else()
     list(APPEND result "-DNDEBUG")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-DSWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -320,7 +320,7 @@ function(_add_variant_swift_compile_flags
   if (SWIFT_ENABLE_GUARANTEED_NORMAL_ARGUMENTS)
     list(APPEND result "-Xfrontend" "-enable-guaranteed-normal-arguments")
   endif()
-  
+
   if(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS)
     list(APPEND result "-D" "SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS")
   endif()
@@ -1605,7 +1605,7 @@ function(add_swift_library name)
         if(SWIFTLIB_IS_SDK_OVERLAY)
           list(APPEND swiftlib_swift_compile_flags_all "-Fsystem" "${SWIFT_SDK_${sdk}_PATH}/System/Library/PrivateFrameworks/")
         endif()
-       
+
        if("${sdk}" STREQUAL "IOS_SIMULATOR")
          if("${name}" STREQUAL "swiftMediaPlayer")
            message("DISABLING AUTOLINK FOR swiftMediaPlayer")
@@ -1854,7 +1854,7 @@ function(add_swift_library name)
           LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX}
           RUNTIME DESTINATION bin)
       swift_is_installing_component(dev is_installing)
-      
+
       if(NOT is_installing)
         set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${name})
       else()
@@ -2177,7 +2177,7 @@ function(add_swift_host_tool executable)
 
     swift_is_installing_component(${ADDSWIFTHOSTTOOL_SWIFT_COMPONENT}
       is_installing)
-  
+
     if(NOT is_installing)
       set_property(GLOBAL APPEND PROPERTY SWIFT_BUILDTREE_EXPORTS ${executable})
     else()

test/SwiftSyntax/ParseFile.swift

@@ -2,7 +2,6 @@
 // REQUIRES: executable_test
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
-// REQUIRES: rdar36740859
 
 import Foundation
 import StdlibUnittest

test/SwiftSyntax/VisitorTest.swift

@@ -3,9 +3,6 @@
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
 
-// FIXME: This test fails occassionally in CI with invalid json.
-// REQUIRES: disabled
-
 import StdlibUnittest
 import Foundation
 import SwiftSyntax

test/lit.cfg

@@ -286,6 +286,7 @@ else:
     test_resource_dir = os.path.join(config.swift_lib_dir, 'swift')
     resource_dir_opt = ""
 stdlib_resource_dir_opt = resource_dir_opt
+sourcekitd_framework_dir = config.swift_lib_dir
 lit_config.note('Using resource dir: ' + test_resource_dir)
 
 # Parse the variant triple.
@@ -670,12 +671,13 @@ if run_vendor == 'apple':
             (run_cpu, run_os, run_vers, clang_mcp_opt))
 
         config.target_build_swift = (
-            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s"
+            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s -F %s -Xlinker -rpath -Xlinker %s"
             % (xcrun_prefix, config.swiftc, target_options,
                extra_frameworks_dir, extra_frameworks_dir,
                sdk_overlay_linker_opt, config.swift_test_options,
                config.swift_driver_test_options,
-               swift_execution_tests_extra_flags))
+               swift_execution_tests_extra_flags, sourcekitd_framework_dir,
+               sourcekitd_framework_dir))
         config.target_run = ""
 
         if 'interpret' in lit_config.params:

tools/CMakeLists.txt

@@ -16,19 +16,27 @@ add_swift_tool_subdirectory(swift-api-digester)
 add_swift_tool_subdirectory(swift-syntax-test)
 add_swift_tool_subdirectory(swift-refactor)
 
+if (SWIFT_BUILD_SOURCEKIT AND SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  set(BUILD_SOURCEKIT_SWIFT_CLIENT TRUE)
+else()
+  set(BUILD_SOURCEKIT_SWIFT_CLIENT FALSE)
+endif()
+
 if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
+  if(BUILD_SOURCEKIT_SWIFT_CLIENT)
+    add_subdirectory(SwiftSourceKitClient)
+  endif()
 endif()
 
-
 if(SWIFT_HOST_VARIANT STREQUAL "macosx")
   # Only build Darwin-specific tools when deploying to OS X.
   add_swift_tool_subdirectory(swift-stdlib-tool)
 
   # SwiftSyntax depends on both the standard library (because it's a
   # Swift module), and the SDK overlays (because it depends on Foundation).
   # Ensure we only build SwiftSyntax when we're building both.
-  if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  if(BUILD_SOURCEKIT_SWIFT_CLIENT)
     add_subdirectory(SwiftSyntax)
   endif()
 endif()

tools/SwiftSourceKitClient/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(EXTRA_COMPILE_FLAGS "-F" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+set(EXTRA_LINKER_FLAGS "-Xlinker" "-rpath" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}"
+    "-Xlinker" "-F" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+
+add_swift_library(swiftSwiftSourceKit SHARED
+  # This file should be listed the first.  Module name is inferred from the
+  # filename.
+  SourceKitdClient.swift
+  SourceKitdRequest.swift
+  SourceKitdResponse.swift
+  SourceKitdUID.swift
+
+  SWIFT_COMPILE_FLAGS ${EXTRA_COMPILE_FLAGS}
+  LINK_FLAGS ${EXTRA_LINKER_FLAGS}
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT swift-syntax
+  TARGET_SDKS OSX
+  IS_STDLIB)

tools/SwiftSourceKitClient/SourceKitdClient.swift

@@ -0,0 +1,28 @@
+//===--------------------- SourceKitdClient.swift -------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a wrapper of SourceKitd service.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+
+public class SourceKitdService {
+
+  public init() {
+    sourcekitd_initialize()
+  }
+  deinit {
+    sourcekitd_shutdown()
+  }
+  public func sendSyn(request: SourceKitdRequest) -> SourceKitdResponse {
+    return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+  }
+}

tools/SwiftSourceKitClient/SourceKitdRequest.swift

@@ -0,0 +1,154 @@
+//===--------------- SourceKitdRequest.swift ------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a convenient way to build a sourcekitd request.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+
+public struct SourceKitdRequest: CustomStringConvertible {
+
+  public class Dictionary: CustomStringConvertible {
+    let dict: sourcekitd_object_t
+
+    public init() {
+      dict = sourcekitd_request_dictionary_create(nil, nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(UnsafeMutableRawPointer(dict))
+    }
+
+    public func add(_ key: SourceKitdUID, value: String) {
+      sourcekitd_request_dictionary_set_string(dict, key.uid, value)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Int) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, Int64(value))
+    }
+
+    public func add(_ key: SourceKitdUID, value: SourceKitdUID) {
+      sourcekitd_request_dictionary_set_uid(dict, key.uid, value.uid)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Array) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.arr)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Dictionary) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.dict)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Bool) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, value ? 1 : 0)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(dict)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  public class Array: CustomStringConvertible {
+    let arr: sourcekitd_object_t
+    private let Append: Int = -1
+
+    public init() {
+      arr = sourcekitd_request_array_create(nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(arr)
+    }
+
+    public func add(_ value: String) {
+      sourcekitd_request_array_set_string(arr, Append, value)
+    }
+
+    public func add(_ value: Int) {
+      sourcekitd_request_array_set_int64(arr, Append, Int64(value))
+    }
+
+    public func add(_ value: SourceKitdUID) {
+      sourcekitd_request_array_set_uid(arr, Append, value.uid)
+    }
+
+    public func add(_ value: Dictionary) {
+      sourcekitd_request_array_set_value(arr, Append, value.dict)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(arr)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  private let req = Dictionary()
+
+  public init(uid: SourceKitdUID) {
+    req.add(SourceKitdUID.key_request, value: uid)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: String) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: Int) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: SourceKitdUID) {
+    req.add(key, value: value)
+  }
+
+  public func addArrayParameter(_ key: SourceKitdUID) -> Array {
+    let arr = Array()
+    req.add(key, value: arr)
+    return arr
+  }
+
+  public func addDictionaryParameter(_ key: SourceKitdUID) -> Dictionary {
+    let dict = Dictionary()
+    req.add(key, value: dict)
+    return dict
+  }
+
+  public var description: String {
+    return req.description
+  }
+
+  public var rawRequest: sourcekitd_object_t {
+    return req.dict
+  }
+
+  public func addCompilerArgsToRequest(_ compilerArguments: [String]?,
+                                       _ bufferName: String? = nil) {
+    let args = self.addArrayParameter(SourceKitdUID.key_compilerargs)
+
+    if let compilerArguments = compilerArguments {
+      for argument in compilerArguments {
+        switch argument {
+        // Exclude some arguments which SourceKit doesn't want or need.
+        case "-Xfrontend":
+          break
+        default:
+          args.add(argument)
+        }
+      }
+    }
+  }
+}