Merge remote-tracking branch 'origin/master' into swift-5.1-branch

Semantically, we want our branch point to match swiftpm, but all of the
changes since then also look good to take into the branch, so just do
one last merge from master.

Author: benlangmuir

.gitignore

@@ -1,7 +1,9 @@
 .DS_Store
 default.profraw
+Package.resolved
 /.build
 /Packages
 /*.xcodeproj
 /*.sublime-project
 /*.sublime-workspace
+/.swiftpm

Editors/README.md

@@ -39,6 +39,7 @@ You will need the path to the `sourcekit-lsp` executable and the Swift toolchain
           "scopes": ["source.swift"],
           "syntaxes": [
             "Packages/Swift/Syntaxes/Swift.tmLanguage",
+            "Packages/Decent Swift Syntax/Swift.sublime-syntax",
           ],
           "languageId": "swift"
         },

Editors/vscode/README.md

@@ -22,7 +22,7 @@ $ cd Editors/vscode
 $ npm run createDevPackage
 ```
 
-You can install the package from the command-line if you have the `code` executable on your `PATH`:
+You can install the package from the command-line using the `code` command if available (see [Launching from the command line](https://code.visualstudio.com/docs/setup/mac#_launching-from-the-command-line)).
 
 ```
 code --install-extension out/sourcekit-lsp-vscode-dev.vsix

Package.resolved

@@ -7,8 +7,7 @@ let package = Package(
     products: [
     ],
     dependencies: [
-      .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
-      .package(url: "https://github.com/apple/indexstore-db.git", .branch("master")),
+      // See 'Dependencies' below.
     ],
     targets: [
       .target(
@@ -82,3 +81,28 @@ let package = Package(
         dependencies: ["SKSupport", "SKTestSupport"]),
     ]
 )
+
+// MARK: Dependencies
+
+// When building with the swift build-script, use local dependencies whose contents are controlled
+// by the external environment. This allows sourcekit-lsp to take advantage of the automation used
+// for building the swift toolchain, such as `update-checkout`, or cross-repo PR tests.
+
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
+
+if getenv("SWIFTCI_USE_LOCAL_DEPS") == nil {
+  // Building standalone.
+  package.dependencies += [
+    .package(url: "https://github.com/apple/indexstore-db.git", .branch("master")),
+    .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
+  ]
+} else {
+  package.dependencies += [
+    .package(path: "../indexstore-db"),
+    .package(path: "../swiftpm"),
+  ]
+}

Package.swift

@@ -7,7 +7,7 @@ SourceKit-LSP is an implementation of the [Language Server Protocol](https://mic
 
 SourceKit-LSP is under heavy development! The best way to try it out is to build it from source. You will also need a Swift development toolchain and an editor that supports LSP.
 
-1. Install the `swift-DEVELOPMENT-SNAPSHOT-2019-01-22-a` toolchain snapshot from https://swift.org/download/#snapshots. Set the environment variable `SOURCEKIT_TOOLCHAIN_PATH` to the absolute path to the toolchain or otherwise configure your editor to use this toolchain. See [Toolchains](#toolchains) for more information.
+1. Install the latest master toolchain snapshot from https://swift.org/download/#snapshots. Set the environment variable `SOURCEKIT_TOOLCHAIN_PATH` to the absolute path to the toolchain or otherwise configure your editor to use this toolchain. See [Toolchains](#toolchains) for more information.
 
 2. Build the language server executable `sourcekit-lsp` using `swift build`. See [Building](#building-sourcekit-lsp) for more information.
 
@@ -23,17 +23,20 @@ SourceKit-LSP is built using the [Swift Package Manager](https://github.com/appl
 For a standard debug build:
 
 ```sh
+$ swift package update
 $ swift build
 ```
 
 After building, the server will be located at `.build/debug/sourcekit-lsp`, or a similar path, if you passed any custom options to `swift build`. Editors will generally need to be provided with this path in order to run the newly built server - see [Editors](Editors) for more information about configuration.
 
+SourceKit-LSP is designed to build against the latest SwiftPM, so if you run into any issue make sure you have the most up-to-date dependencies by running `swift package update`.
+
 ### Building on Linux
 
 The C++ code in the index requires `libdispatch`, but unlike Swift code, it cannot find it automatically on Linux. You can work around this by adding a search path manually.
 
 ```sh
-$ swift build -Xcxx -I<path_to_swift_toolchain>/usr/lib/swift
+$ swift build -Xcxx -I<path_to_swift_toolchain>/usr/lib/swift -I<path_to_swift_toolchain>/usr/lib/swift/Block
 ```
 
 ### Using the Generated Xcode Project
@@ -73,7 +76,7 @@ SourceKit-LSP depends on tools such as `sourcekitd` and `clangd`, which it loads
 
 ### Recommended Toolchain
 
-Use the `swift-DEVELOPMENT-SNAPSHOT-2019-01-22-a` toolchain snapshot from https://swift.org/download/#snapshots. SourceKit-LSP is still early in its development and we are actively adding functionality to the toolchain to support it.
+Use the latest master toolchain snapshot from https://swift.org/download/#snapshots. SourceKit-LSP is still early in its development and we are actively adding functionality to the toolchain to support it.
 
 ### Selecting the Toolchain
 
@@ -104,7 +107,7 @@ SourceKit-LSP is still in early development, so you may run into rough edges wit
 | Feature | Status | Notes |
 |---------|:------:|-------|
 | Swift | ✅ | |
-| C/C++/ObjC | ❌ | [clangd](https://clang.llvm.org/extra/clangd.html) is not available in the recommended toolchain. You can try out C/C++/ObjC support by building clangd from source and putting it in `PATH`.
+| C/C++/ObjC | ❌ | As of `swift-DEVELOPMENT-SNAPSHOT-2019-02-14-a` [clangd](https://clang.llvm.org/extra/clangd.html) is available but hitting frequent assertion failures.
 | Code completion | ✅ | |
 | Quick Help (Hover) | ✅ | |
 | Diagnostics | ✅ | |

README.md

@@ -14,7 +14,6 @@ import SKSupport
 import Basic
 import SPMUtility
 import Dispatch
-import POSIX
 import Foundation
 
 /// Set of known toolchains.
@@ -43,7 +42,7 @@ public final class ToolchainRegistry {
 
   /// The currently selected toolchain identifier on Darwin.
   public internal(set) lazy var darwinToolchainOverride: String? = {
-    if let id = getenv("TOOLCHAINS"), !id.isEmpty, id != "default" {
+    if let id = ProcessEnv.vars["TOOLCHAINS"], !id.isEmpty, id != "default" {
       return id
     }
     return nil
@@ -278,7 +277,7 @@ extension ToolchainRegistry {
   {
     var shouldSetDefault = setDefault
     for envVar in environmentVariables {
-      if let pathStr = getenv(envVar),
+      if let pathStr = ProcessEnv.vars[envVar],
          let path = try? AbsolutePath(validating: pathStr),
          let toolchain = try? _registerToolchain(path, fileSystem),
          shouldSetDefault
@@ -301,7 +300,7 @@ extension ToolchainRegistry {
 
   func _scanForToolchains(pathVariables: [String], _ fileSystem: FileSystem) {
     pathVariables.lazy.flatMap { envVar in
-      getEnvSearchPaths(pathString: getenv(envVar), currentWorkingDirectory: nil)
+      getEnvSearchPaths(pathString: ProcessEnv.vars[envVar], currentWorkingDirectory: nil)
     }
     .forEach { path in
       _ = try? _registerToolchain(path, fileSystem)

Sources/SKCore/ToolchainRegistry.swift

@@ -169,8 +169,9 @@ public final class SourceKitServer: LanguageServer {
         return nil
       }
 
+      let pid = Int(ProcessInfo.processInfo.processIdentifier)
       let resp = try service.sendSync(InitializeRequest(
-        processId: Int(getpid()),
+        processId: pid,
         rootPath: nil,
         rootURL: (workspace?.rootPath).map { URL(fileURLWithPath: $0.pathString) },
         initializationOptions: InitializationOptions(),

Sources/SourceKit/SourceKitServer.swift

@@ -42,7 +42,11 @@ final class SwiftSourceKitFramework {
 
   init(dylib path: AbsolutePath) throws {
     self.path = path
+    #if os(Windows)
+    self.dylib = try dlopen(path.pathString, mode: [])
+    #else
     self.dylib = try dlopen(path.pathString, mode: [.lazy, .local, .first, .deepBind])
+    #endif
 
     func dlsym_required<T>(_ handle: DLHandle, symbol: String) throws -> T {
       guard let sym: T = dlsym(handle, symbol: symbol) else {
@@ -54,7 +58,7 @@ final class SwiftSourceKitFramework {
     // Workaround rdar://problem/43656704 by not constructing the value directly.
     // self.api = sourcekitd_functions_t(
     let ptr = UnsafeMutablePointer<sourcekitd_functions_t>.allocate(capacity: 1)
-    bzero(UnsafeMutableRawPointer(ptr), MemoryLayout<sourcekitd_functions_t>.stride)
+    memset(UnsafeMutableRawPointer(ptr), 0, MemoryLayout<sourcekitd_functions_t>.stride)
     var api = ptr.pointee
     ptr.deallocate()
 

Sources/SourceKit/sourcekitd/SwiftSourceKitFramework.swift

@@ -14,7 +14,6 @@
 import Basic
 import SPMUtility
 import XCTest
-import POSIX
 
 final class ToolchainRegistryTests: XCTestCase {
   func testDefaultBasic() {
@@ -186,8 +185,8 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("SOURCEKIT_PATH", value: "/bogus:\(binPath):/bogus2")
-    defer { try! setenv("SOURCEKIT_PATH", value: "") }
+    try! ProcessEnv.setVar("SOURCEKIT_PATH", value: "/bogus:\(binPath):/bogus2")
+    defer { try! ProcessEnv.setVar("SOURCEKIT_PATH", value: "") }
 
     tr.scanForToolchains(fs)
 
@@ -205,7 +204,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tc.libIndexStore)
 
     let binPath2 = AbsolutePath("/other/my_toolchain/bin")
-    try! setenv("SOME_TEST_ENV_PATH", value: "/bogus:\(binPath2):/bogus2")
+    try! ProcessEnv.setVar("SOME_TEST_ENV_PATH", value: "/bogus:\(binPath2):/bogus2")
     makeToolchain(binPath: binPath2, fs, sourcekitd: true)
     tr.scanForToolchains(pathVariables: ["NOPE", "SOME_TEST_ENV_PATH", "MORE_NOPE"], fs)
 
@@ -228,7 +227,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("TEST_SOURCEKIT_TOOLCHAIN_PATH_1", value: binPath.parentDirectory.pathString)
+    try! ProcessEnv.setVar("TEST_SOURCEKIT_TOOLCHAIN_PATH_1", value: binPath.parentDirectory.pathString)
 
     tr.scanForToolchains(environmentVariables: ["TEST_SOURCEKIT_TOOLCHAIN_PATH_1"], fs)
 
@@ -255,7 +254,7 @@ final class ToolchainRegistryTests: XCTestCase {
     XCTAssertNil(tr.default)
     XCTAssert(tr.toolchains.isEmpty)
 
-    try! setenv("TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2", value: binPath.parentDirectory.pathString)
+    try! ProcessEnv.setVar("TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2", value: binPath.parentDirectory.pathString)
 
     tr.scanForToolchains(
       environmentVariables: ["TEST_ENV_SOURCEKIT_TOOLCHAIN_PATH_2"],

Tests/SKCoreTests/ToolchainRegistryTests.swift

@@ -13,7 +13,6 @@
 @testable import SKSupport
 import SKTestSupport
 import Basic
-import POSIX
 import XCTest
 
 final class SupportPerfTests: PerfTestCase {
@@ -24,8 +23,7 @@ final class SupportPerfTests: PerfTestCase {
       "\t", "\n"
     ] + (32...126).map { Character(UnicodeScalar($0)) }
 
-    // The debug performance is shockingly bad.
-    #if DEBUG
+    #if DEBUG || !ENABLE_PERF_TESTS
     let iterations = 1_000
     #else
     let iterations = 10_000
@@ -49,13 +47,11 @@ final class SupportPerfTests: PerfTestCase {
   }
 
   func testLineTableSingleCharEditPerf() {
-
     let characters: [Character] = [
       "\t", "\n"
       ] + (32...126).map { Character(UnicodeScalar($0)) }
 
-    // The debug performance is shockingly bad.
-    #if DEBUG
+    #if DEBUG || !ENABLE_PERF_TESTS
     let iterations = 1_000
     let size = 1_000
     #else
@@ -75,9 +71,9 @@ final class SupportPerfTests: PerfTestCase {
       self.startMeasuring()
 
       for _ in 1...iterations {
-        let line = (0..<(t.count-1)).randomElement(using: &lcg)!
-        let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg)!
-        let len = Bool.random() ? 1 : 0
+        let line = (0 ..< (t.count-1)).randomElement(using: &lcg) ?? 0
+        let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg) ?? 0
+        let len = t[line].isEmpty ? 0 : Bool.random() ? 1 : 0
         var newText = String(characters.randomElement(using: &lcg)!)
         if len == 1 && Bool.random(using: &lcg) {
           newText = "" // deletion

Tests/SKSupportTests/SupportPerfTests.swift

@@ -13,7 +13,6 @@
 import XCTest
 @testable import SKSupport
 import Basic
-import POSIX
 
 final class SupportTests: XCTestCase {
 
@@ -184,8 +183,8 @@ final class SupportTests: XCTestCase {
 
     testLogger.currentLevel = .default
 
-    try! setenv("TEST_ENV_LOGGGING_1", value: "1")
-    try! setenv("TEST_ENV_LOGGGING_0", value: "0")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_1", value: "1")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_0", value: "0")
     // .warning
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_1")
 
@@ -221,7 +220,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // invalid - no change
-    try! setenv("TEST_ENV_LOGGGING_err", value: "")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -233,7 +232,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // invalid - no change
-    try! setenv("TEST_ENV_LOGGGING_err", value: "a3")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "a3")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -245,7 +244,7 @@ final class SupportTests: XCTestCase {
       ])
 
     // too high - max out at .debug
-    try! setenv("TEST_ENV_LOGGGING_err", value: "1000")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_err", value: "1000")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_err")
 
     log("d", level: .error)
@@ -260,16 +259,16 @@ final class SupportTests: XCTestCase {
       ])
 
     // By string.
-    try! setenv("TEST_ENV_LOGGGING_string", value: "error")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "error")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .error)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "warning")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "warning")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .warning)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "info")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "info")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .info)
-    try! setenv("TEST_ENV_LOGGGING_string", value: "debug")
+    try! ProcessEnv.setVar("TEST_ENV_LOGGGING_string", value: "debug")
     testLogger.setLogLevel(environmentVariable: "TEST_ENV_LOGGGING_string")
     XCTAssertEqual(testLogger.currentLevel, .debug)
 

Tests/SKSupportTests/SupportTests.swift

@@ -20,7 +20,7 @@ import XCTest
 final class LocalClangTests: XCTestCase {
 
   /// Whether to fail tests if clangd cannot be found.
-  static let requireClangd: Bool = false
+  static let requireClangd: Bool = false // Note: Swift CI doesn't build clangd on all jobs
 
   /// Whether clangd exists in the toolchain.
   var haveClangd: Bool = false