[5.1] Update swift-5.1-branch to match the swiftpm branch point

.gitignore

@@ -1,7 +1,9 @@
 .DS_Store
 default.profraw
+Package.resolved
 /.build
 /Packages
 /*.xcodeproj
 /*.sublime-project
 /*.sublime-workspace
+/.swiftpm

Editors/README.md

@@ -39,6 +39,7 @@ You will need the path to the `sourcekit-lsp` executable and the Swift toolchain
           "scopes": ["source.swift"],
           "syntaxes": [
             "Packages/Swift/Syntaxes/Swift.tmLanguage",
+            "Packages/Decent Swift Syntax/Swift.sublime-syntax",
           ],
           "languageId": "swift"
         },

Editors/vscode/README.md

@@ -22,7 +22,7 @@ $ cd Editors/vscode
 $ npm run createDevPackage
 ```
 
-You can install the package from the command-line if you have the `code` executable on your `PATH`:
+You can install the package from the command-line using the `code` command if available (see [Launching from the command line](https://code.visualstudio.com/docs/setup/mac#_launching-from-the-command-line)).
 
 ```
 code --install-extension out/sourcekit-lsp-vscode-dev.vsix

Package.swift

@@ -7,8 +7,7 @@ let package = Package(
     products: [
     ],
     dependencies: [
-      .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
-      .package(url: "https://github.com/apple/indexstore-db.git", .branch("master")),
+      // See 'Dependencies' below.
     ],
     targets: [
       .target(
@@ -82,3 +81,28 @@ let package = Package(
         dependencies: ["SKSupport", "SKTestSupport"]),
     ]
 )
+
+// MARK: Dependencies
+
+// When building with the swift build-script, use local dependencies whose contents are controlled
+// by the external environment. This allows sourcekit-lsp to take advantage of the automation used
+// for building the swift toolchain, such as `update-checkout`, or cross-repo PR tests.
+
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
+
+if getenv("SWIFTCI_USE_LOCAL_DEPS") == nil {
+  // Building standalone.
+  package.dependencies += [
+    .package(url: "https://github.com/apple/indexstore-db.git", .branch("swift-5.1-branch")),
+    .package(url: "https://github.com/apple/swift-package-manager.git", .branch("swift-5.1-branch")),
+  ]
+} else {
+  package.dependencies += [
+    .package(path: "../indexstore-db"),
+    .package(path: "../swiftpm"),
+  ]
+}

README.md

@@ -7,7 +7,7 @@ SourceKit-LSP is an implementation of the [Language Server Protocol](https://mic
 
 SourceKit-LSP is under heavy development! The best way to try it out is to build it from source. You will also need a Swift development toolchain and an editor that supports LSP.
 
-1. Install the `swift-DEVELOPMENT-SNAPSHOT-2019-01-22-a` toolchain snapshot from https://swift.org/download/#snapshots. Set the environment variable `SOURCEKIT_TOOLCHAIN_PATH` to the absolute path to the toolchain or otherwise configure your editor to use this toolchain. See [Toolchains](#toolchains) for more information.
+1. Install the latest master toolchain snapshot from https://swift.org/download/#snapshots. Set the environment variable `SOURCEKIT_TOOLCHAIN_PATH` to the absolute path to the toolchain or otherwise configure your editor to use this toolchain. See [Toolchains](#toolchains) for more information.
 
 2. Build the language server executable `sourcekit-lsp` using `swift build`. See [Building](#building-sourcekit-lsp) for more information.
 
@@ -23,17 +23,20 @@ SourceKit-LSP is built using the [Swift Package Manager](https://github.com/appl
 For a standard debug build:
 
 ```sh
+$ swift package update
 $ swift build
 ```
 
 After building, the server will be located at `.build/debug/sourcekit-lsp`, or a similar path, if you passed any custom options to `swift build`. Editors will generally need to be provided with this path in order to run the newly built server - see [Editors](Editors) for more information about configuration.
 
+SourceKit-LSP is designed to build against the latest SwiftPM, so if you run into any issue make sure you have the most up-to-date dependencies by running `swift package update`.
+
 ### Building on Linux
 
 The C++ code in the index requires `libdispatch`, but unlike Swift code, it cannot find it automatically on Linux. You can work around this by adding a search path manually.
 
 ```sh
-$ swift build -Xcxx -I<path_to_swift_toolchain>/usr/lib/swift
+$ swift build -Xcxx -I<path_to_swift_toolchain>/usr/lib/swift -I<path_to_swift_toolchain>/usr/lib/swift/Block
 ```
 
 ### Using the Generated Xcode Project
@@ -73,7 +76,7 @@ SourceKit-LSP depends on tools such as `sourcekitd` and `clangd`, which it loads
 
 ### Recommended Toolchain
 
-Use the `swift-DEVELOPMENT-SNAPSHOT-2019-01-22-a` toolchain snapshot from https://swift.org/download/#snapshots. SourceKit-LSP is still early in its development and we are actively adding functionality to the toolchain to support it.
+Use the latest master toolchain snapshot from https://swift.org/download/#snapshots. SourceKit-LSP is still early in its development and we are actively adding functionality to the toolchain to support it.
 
 ### Selecting the Toolchain
 
@@ -104,7 +107,7 @@ SourceKit-LSP is still in early development, so you may run into rough edges wit
 | Feature | Status | Notes |
 |---------|:------:|-------|
 | Swift | ✅ | |
-| C/C++/ObjC | ❌ | [clangd](https://clang.llvm.org/extra/clangd.html) is not available in the recommended toolchain. You can try out C/C++/ObjC support by building clangd from source and putting it in `PATH`.
+| C/C++/ObjC | ❌ | As of `swift-DEVELOPMENT-SNAPSHOT-2019-02-14-a` [clangd](https://clang.llvm.org/extra/clangd.html) is available but hitting frequent assertion failures.
 | Code completion | ✅ | |
 | Quick Help (Hover) | ✅ | |
 | Diagnostics | ✅ | |

Sources/SKCore/ToolchainRegistry.swift

@@ -14,7 +14,6 @@ import SKSupport
 import Basic
 import SPMUtility
 import Dispatch
-import POSIX
 import Foundation
 
 /// Set of known toolchains.
@@ -43,7 +42,7 @@ public final class ToolchainRegistry {
 
   /// The currently selected toolchain identifier on Darwin.
   public internal(set) lazy var darwinToolchainOverride: String? = {
-    if let id = getenv("TOOLCHAINS"), !id.isEmpty, id != "default" {
+    if let id = ProcessEnv.vars["TOOLCHAINS"], !id.isEmpty, id != "default" {
       return id
     }
     return nil
@@ -278,7 +277,7 @@ extension ToolchainRegistry {
   {
     var shouldSetDefault = setDefault
     for envVar in environmentVariables {
-      if let pathStr = getenv(envVar),
+      if let pathStr = ProcessEnv.vars[envVar],
          let path = try? AbsolutePath(validating: pathStr),
          let toolchain = try? _registerToolchain(path, fileSystem),
          shouldSetDefault
@@ -301,7 +300,7 @@ extension ToolchainRegistry {
 
   func _scanForToolchains(pathVariables: [String], _ fileSystem: FileSystem) {
     pathVariables.lazy.flatMap { envVar in
-      getEnvSearchPaths(pathString: getenv(envVar), currentWorkingDirectory: nil)
+      getEnvSearchPaths(pathString: ProcessEnv.vars[envVar], currentWorkingDirectory: nil)
     }
     .forEach { path in
       _ = try? _registerToolchain(path, fileSystem)

Sources/SourceKit/SourceKitServer.swift

@@ -169,8 +169,9 @@ public final class SourceKitServer: LanguageServer {
         return nil
       }
 
+      let pid = Int(ProcessInfo.processInfo.processIdentifier)
       let resp = try service.sendSync(InitializeRequest(
-        processId: Int(getpid()),
+        processId: pid,
         rootPath: nil,
         rootURL: (workspace?.rootPath).map { URL(fileURLWithPath: $0.pathString) },
         initializationOptions: InitializationOptions(),

Sources/SourceKit/sourcekitd/SwiftSourceKitFramework.swift

@@ -42,7 +42,11 @@ final class SwiftSourceKitFramework {
 
   init(dylib path: AbsolutePath) throws {
     self.path = path
+    #if os(Windows)
+    self.dylib = try dlopen(path.pathString, mode: [])
+    #else
     self.dylib = try dlopen(path.pathString, mode: [.lazy, .local, .first, .deepBind])
+    #endif
 
     func dlsym_required<T>(_ handle: DLHandle, symbol: String) throws -> T {
       guard let sym: T = dlsym(handle, symbol: symbol) else {
@@ -54,7 +58,7 @@ final class SwiftSourceKitFramework {
     // Workaround rdar://problem/43656704 by not constructing the value directly.
     // self.api = sourcekitd_functions_t(
     let ptr = UnsafeMutablePointer<sourcekitd_functions_t>.allocate(capacity: 1)
-    bzero(UnsafeMutableRawPointer(ptr), MemoryLayout<sourcekitd_functions_t>.stride)
+    memset(UnsafeMutableRawPointer(ptr), 0, MemoryLayout<sourcekitd_functions_t>.stride)
     var api = ptr.pointee
     ptr.deallocate()
 

Tests/SKCoreTests/ToolchainRegistryTests.swift

@@ -14,7 +14,6 @@
 import Basic
 import SPMUtility
 import XCTest
-import POSIX
 
 final class ToolchainRegistryTests: XCTestCase {
   func testDefaultBasic() {
@@ -186,8 +185,8 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("SOURCEKIT_PATH", value: "/bogus:\(binPath):/bogus2")
-    defer { try! setenv("SOURCEKIT_PATH", value: "") }
+    try! ProcessEnv.setVar("SOURCEKIT_PATH", value: "/bogus:\(binPath):/bogus2")
+    defer { try! ProcessEnv.setVar("SOURCEKIT_PATH", value: "") }
 
     tr.scanForToolchains(fs)
 
@@ -205,7 +204,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tc.libIndexStore)
 
     let binPath2 = AbsolutePath("/other/my_toolchain/bin")
-    try! setenv("SOME_TEST_ENV_PATH", value: "/bogus:\(binPath2):/bogus2")
+    try! ProcessEnv.setVar("SOME_TEST_ENV_PATH", value: "/bogus:\(binPath2):/bogus2")
     makeToolchain(binPath: binPath2, fs, sourcekitd: true)
     tr.scanForToolchains(pathVariables: ["NOPE", "SOME_TEST_ENV_PATH", "MORE_NOPE"], fs)
 
@@ -228,7 +227,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("TEST_SOURCEKIT_TOOLCHAIN_PATH_1", value: binPath.parentDirectory.pathString)
+    try! ProcessEnv.setVar("TEST_SOURCEKIT_TOOLCHAIN_PATH_1", value: binPath.parentDirectory.pathString)
 
     tr.scanForToolchains(environmentVariables: ["TEST_SOURCEKIT_TOOLCHAIN_PATH_1"], fs)
 
@@ -255,7 +254,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2", value: binPath.parentDirectory.pathString)
+    try! ProcessEnv.setVar("TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2", value: binPath.parentDirectory.pathString)
 
     tr.scanForToolchains(
       environmentVariables: ["TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2"],

Tests/SKSupportTests/SupportPerfTests.swift

@@ -13,7 +13,6 @@
 @testable import SKSupport
 import SKTestSupport
 import Basic
-import POSIX
 import XCTest
 
 final class SupportPerfTests: PerfTestCase {
@@ -24,8 +23,7 @@ final class SupportPerfTests: PerfTestCase {
       "\t", "\n"
     ] + (32...126).map { Character(UnicodeScalar($0)) }
 
-    // The debug performance is shockingly bad.
-    #if DEBUG
+    #if DEBUG || !ENABLE_PERF_TESTS
     let iterations = 1_000
     #else
     let iterations = 10_000
@@ -49,13 +47,11 @@ final class SupportPerfTests: PerfTestCase {
   }
 
   func testLineTableSingleCharEditPerf() {
-
     let characters: [Character] = [
       "\t", "\n"
       ] + (32...126).map { Character(UnicodeScalar($0)) }
 
-    // The debug performance is shockingly bad.
-    #if DEBUG
+    #if DEBUG || !ENABLE_PERF_TESTS
     let iterations = 1_000
     let size = 1_000
     #else
@@ -75,9 +71,9 @@ final class SupportPerfTests: PerfTestCase {
       self.startMeasuring()
 
       for _ in 1...iterations {
-        let line = (0..<(t.count-1)).randomElement(using: &lcg)!
-        let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg)!
-        let len = Bool.random() ? 1 : 0
+        let line = (0 ..< (t.count-1)).randomElement(using: &lcg) ?? 0
+        let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg) ?? 0
+        let len = t[line].isEmpty ? 0 : Bool.random() ? 1 : 0
         var newText = String(characters.randomElement(using: &lcg)!)
         if len == 1 && Bool.random(using: &lcg) {
           newText = "" // deletion

Tests/SKSupportTests/SupportTests.swift

@@ -13,7 +13,6 @@
 import XCTest
 @testable import SKSupport
 import Basic
-import POSIX
 
 final class SupportTests: XCTestCase {
 
@@ -184,8 +183,8 @@ final class SupportTests: XCTestCase {
 
     testLogger.currentLevel = .default
 
-    try! setenv("TEST_ENV_LOGGGING_1", value: "1")
-    try! setenv("TEST_ENV_LOGGGING_0", value: "0")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_1", value: "1")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_0", value: "0")
     // .warning
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_1")
 
@@ -221,7 +220,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // invalid - no change
-    try! setenv("TEST_ENV_LOGGGING_err", value: "")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -233,7 +232,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // invalid - no change
-    try! setenv("TEST_ENV_LOGGGING_err", value: "a3")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "a3")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -245,7 +244,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // too high - max out at .debug
-    try! setenv("TEST_ENV_LOGGGING_err", value: "1000")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "1000")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -260,16 +259,16 @@ final class SupportTests: XCTestCase {
       ])
 
     // By string.
-    try! setenv("TEST_ENV_LOGGGING_string", value: "error")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "error")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .error)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "warning")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "warning")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .warning)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "info")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "info")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .info)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "debug")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "debug")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .debug)
 

Tests/SourceKitTests/LocalClangTests.swift

@@ -20,7 +20,7 @@ import XCTest
 final class LocalClangTests: XCTestCase {
 
   /// Whether to fail tests if clangd cannot be found.
-  static let requireClangd: Bool = false
+  static let requireClangd: Bool = false // Note: Swift CI doesn't build clangd on all jobs
 
   /// Whether clangd exists in the toolchain.
   var haveClangd: Bool = false