[WIP] Re-apply "SwiftSyntax: Teach SwiftSyntax to use SourceKitd to serialize syntax trees. (#14424)"

CMakeLists.txt

@@ -11,6 +11,8 @@ set_property(GLOBAL PROPERTY JOB_POOLS local_jobs=${localhost_logical_cores})
 # Put linking in that category
 set_property(GLOBAL PROPERTY JOB_POOL_LINK local_jobs)
 
+ENABLE_LANGUAGE(C)
+
 # First include general CMake utilities.
 include(SwiftUtils)
 include(CheckSymbolExists)

test/SwiftSyntax/DeserializeFile.swift

@@ -6,6 +6,7 @@
 import StdlibUnittest
 import Foundation
 import SwiftSyntax
+import SwiftLang
 
 func getInput(_ file: String) -> URL {
   var result = URL(fileURLWithPath: #file)
@@ -19,7 +20,7 @@ var DecodeTests = TestSuite("DecodeSyntax")
 
 DecodeTests.test("Basic") {
   expectDoesNotThrow({
-    let content = try SourceFileSyntax.encodeSourceFileSyntax(getInput("visitor.swift"))
+    let content = try SwiftLang.parse(getInput("visitor.swift"))
     let source = try String(contentsOf: getInput("visitor.swift"))
     let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
     expectEqual("\(parsed)", source)

test/SwiftSyntax/ParseFile.swift

@@ -2,11 +2,11 @@
 // REQUIRES: executable_test
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
-// REQUIRES: rdar36740859
 
 import Foundation
 import StdlibUnittest
 import SwiftSyntax
+import SwiftLang
 
 var ParseFile = TestSuite("ParseFile")
 
@@ -31,7 +31,8 @@ ParseFile.test("ParseSingleFile") {
   let currentFile = URL(fileURLWithPath: #file)
   expectDoesNotThrow({
     let currentFileContents = try String(contentsOf: currentFile)
-    let parsed = try SourceFileSyntax.parse(currentFile)
+    let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
+      SwiftLang.parse(currentFile))
     expectEqual("\(parsed)", currentFileContents)
   })
 }

test/SwiftSyntax/VisitorTest.swift

@@ -3,12 +3,10 @@
 // REQUIRES: OS=macosx
 // REQUIRES: objc_interop
 
-// FIXME: This test fails occassionally in CI with invalid json.
-// REQUIRES: disabled
-
 import StdlibUnittest
 import Foundation
 import SwiftSyntax
+import SwiftLang
 
 func getInput(_ file: String) -> URL {
   var result = URL(fileURLWithPath: #file)
@@ -29,7 +27,8 @@ VisitorTests.test("Basic") {
     }
   }
   expectDoesNotThrow({
-    let parsed = try SourceFileSyntax.parse(getInput("visitor.swift"))
+    let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
+      SwiftLang.parse(getInput("visitor.swift")))
     let counter = FuncCounter()
     let hashBefore = parsed.hashValue
     counter.visit(parsed)

test/lit.cfg

@@ -286,6 +286,7 @@ else:
     test_resource_dir = os.path.join(config.swift_lib_dir, 'swift')
     resource_dir_opt = ""
 stdlib_resource_dir_opt = resource_dir_opt
+sourcekitd_framework_dir = config.swift_lib_dir
 lit_config.note('Using resource dir: ' + test_resource_dir)
 
 # Parse the variant triple.
@@ -670,12 +671,13 @@ if run_vendor == 'apple':
             (run_cpu, run_os, run_vers, clang_mcp_opt))
 
         config.target_build_swift = (
-            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s"
+            "%s %s %s -F %s -Xlinker -rpath -Xlinker %s %s %s %s %s -F %s -Xlinker -rpath -Xlinker %s"
             % (xcrun_prefix, config.swiftc, target_options,
                extra_frameworks_dir, extra_frameworks_dir,
                sdk_overlay_linker_opt, config.swift_test_options,
                config.swift_driver_test_options,
-               swift_execution_tests_extra_flags))
+               swift_execution_tests_extra_flags, sourcekitd_framework_dir,
+               sourcekitd_framework_dir))
         config.target_run = ""
 
         if 'interpret' in lit_config.params:

tools/CMakeLists.txt

@@ -1,3 +1,14 @@
+include(CheckIncludeFiles)
+check_include_files("xpc/xpc.h" HAVE_XPC_H)
+
+swift_is_installing_component(sourcekit-inproc SOURCEKIT_INSTALLING_INPROC)
+
+if(HAVE_XPC_H AND SWIFT_BUILD_SOURCEKIT AND NOT SOURCEKIT_INSTALLING_INPROC)
+  set(BUILD_SOURCEKIT_XPC_SERVICE TRUE)
+else()
+  set(BUILD_SOURCEKIT_XPC_SERVICE FALSE)
+endif()
+
 # Add generated libSyntax headers to global dependencies.
 list(APPEND LLVM_COMMON_DEPENDS swift-syntax-generated-headers)
 
@@ -16,19 +27,24 @@ add_swift_tool_subdirectory(swift-api-digester)
 add_swift_tool_subdirectory(swift-syntax-test)
 add_swift_tool_subdirectory(swift-refactor)
 
+if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  set(BUILD_FOUNDATION TRUE)
+else()
+  set(BUILD_FOUNDATION FALSE)
+endif()
+
 if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
 endif()
 
-
 if(SWIFT_HOST_VARIANT STREQUAL "macosx")
   # Only build Darwin-specific tools when deploying to OS X.
   add_swift_tool_subdirectory(swift-stdlib-tool)
 
   # SwiftSyntax depends on both the standard library (because it's a
   # Swift module), and the SDK overlays (because it depends on Foundation).
   # Ensure we only build SwiftSyntax when we're building both.
-  if(SWIFT_BUILD_STDLIB AND SWIFT_BUILD_SDK_OVERLAY)
+  if(BUILD_FOUNDATION)
     add_subdirectory(SwiftSyntax)
   endif()
 endif()

tools/SourceKit/CMakeLists.txt

@@ -24,8 +24,6 @@ configure_file(
 set(SOURCEKIT_DEPLOYMENT_OS "${SWIFT_HOST_VARIANT}")
 set(SOURCEKIT_DEPLOYMENT_TARGET "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_DEPLOYMENT_VERSION}")
 
-swift_is_installing_component(sourcekit-inproc SOURCEKIT_INSTALLING_INPROC)
-
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND NOT CMAKE_CROSSCOMPILING)
   set(CMAKE_OSX_SYSROOT "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_PATH}")
   set(CMAKE_OSX_ARCHITECTURES "${SWIFT_HOST_VARIANT_ARCH}")

tools/SourceKit/tools/CMakeLists.txt

@@ -10,3 +10,6 @@ if(HAVE_UNICODE_LIBEDIT)
   add_swift_tool_subdirectory(sourcekitd-repl)
 endif()
 add_swift_tool_subdirectory(complete-test)
+if(BUILD_FOUNDATION)
+  add_subdirectory(SwiftSourceKitClient)
+endif()

tools/SourceKit/tools/SwiftSourceKitClient/CMakeLists.txt

@@ -0,0 +1,19 @@
+set(EXTRA_COMPILE_FLAGS "-F" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+set(EXTRA_LINKER_FLAGS "-Xlinker" "-rpath" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}"
+    "-Xlinker" "-F" "-Xlinker" "${SWIFT_LIBRARY_OUTPUT_INTDIR}")
+
+add_swift_library(swiftSwiftLang SHARED
+  SwiftLang.swift
+  SourceKitdClient.swift
+  SourceKitdRequest.swift
+  SourceKitdResponse.swift
+  SourceKitdUID.swift
+
+  DEPENDS sourcekitd-test
+  PRIVATE_LINK_LIBRARIES sourcekitd
+  SWIFT_COMPILE_FLAGS ${EXTRA_COMPILE_FLAGS}
+  LINK_FLAGS ${EXTRA_LINKER_FLAGS}
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT sourcekit-xpc-service
+  TARGET_SDKS OSX
+  IS_STDLIB)

tools/SourceKit/tools/SwiftSourceKitClient/SourceKitdClient.swift

@@ -0,0 +1,29 @@
+//===--------------------- SourceKitdClient.swift -------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a wrapper of SourceKitd service.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+import Foundation
+
+public class SourceKitdService {
+
+  public init() {
+    sourcekitd_initialize()
+  }
+  deinit {
+    sourcekitd_shutdown()
+  }
+  public func sendSyn(request: SourceKitdRequest) -> SourceKitdResponse {
+    return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+  }
+}

tools/SourceKit/tools/SwiftSourceKitClient/SourceKitdRequest.swift

@@ -0,0 +1,154 @@
+//===--------------- SourceKitdRequest.swift ------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides a convenient way to build a sourcekitd request.
+//===----------------------------------------------------------------------===//
+
+import sourcekitd
+
+public struct SourceKitdRequest: CustomStringConvertible {
+
+  public class Dictionary: CustomStringConvertible {
+    let dict: sourcekitd_object_t
+
+    public init() {
+      dict = sourcekitd_request_dictionary_create(nil, nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(UnsafeMutableRawPointer(dict))
+    }
+
+    public func add(_ key: SourceKitdUID, value: String) {
+      sourcekitd_request_dictionary_set_string(dict, key.uid, value)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Int) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, Int64(value))
+    }
+
+    public func add(_ key: SourceKitdUID, value: SourceKitdUID) {
+      sourcekitd_request_dictionary_set_uid(dict, key.uid, value.uid)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Array) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.arr)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Dictionary) {
+      sourcekitd_request_dictionary_set_value(dict, key.uid, value.dict)
+    }
+
+    public func add(_ key: SourceKitdUID, value: Bool) {
+      sourcekitd_request_dictionary_set_int64(dict, key.uid, value ? 1 : 0)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(dict)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  public class Array: CustomStringConvertible {
+    let arr: sourcekitd_object_t
+    private let Append: Int = -1
+
+    public init() {
+      arr = sourcekitd_request_array_create(nil, 0)
+    }
+
+    deinit {
+      sourcekitd_request_release(arr)
+    }
+
+    public func add(_ value: String) {
+      sourcekitd_request_array_set_string(arr, Append, value)
+    }
+
+    public func add(_ value: Int) {
+      sourcekitd_request_array_set_int64(arr, Append, Int64(value))
+    }
+
+    public func add(_ value: SourceKitdUID) {
+      sourcekitd_request_array_set_uid(arr, Append, value.uid)
+    }
+
+    public func add(_ value: Dictionary) {
+      sourcekitd_request_array_set_value(arr, Append, value.dict)
+    }
+
+    public var description: String {
+      let utf8Str = sourcekitd_request_description_copy(arr)!
+      let result = String(cString: utf8Str)
+      free(utf8Str)
+      return result
+    }
+
+  }
+
+  private let req = Dictionary()
+
+  public init(uid: SourceKitdUID) {
+    req.add(SourceKitdUID.key_request, value: uid)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: String) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: Int) {
+    req.add(key, value: value)
+  }
+
+  public func addParameter(_ key: SourceKitdUID, value: SourceKitdUID) {
+    req.add(key, value: value)
+  }
+
+  public func addArrayParameter(_ key: SourceKitdUID) -> Array {
+    let arr = Array()
+    req.add(key, value: arr)
+    return arr
+  }
+
+  public func addDictionaryParameter(_ key: SourceKitdUID) -> Dictionary {
+    let dict = Dictionary()
+    req.add(key, value: dict)
+    return dict
+  }
+
+  public var description: String {
+    return req.description
+  }
+
+  public var rawRequest: sourcekitd_object_t {
+    return req.dict
+  }
+
+  public func addCompilerArgsToRequest(_ compilerArguments: [String]?,
+                                       _ bufferName: String? = nil) {
+    let args = self.addArrayParameter(SourceKitdUID.key_compilerargs)
+
+    if let compilerArguments = compilerArguments {
+      for argument in compilerArguments {
+        switch argument {
+        // Exclude some arguments which SourceKit doesn't want or need.
+        case "-Xfrontend":
+          break
+        default:
+          args.add(argument)
+        }
+      }
+    }
+  }
+}