Merge pull request #4 from apple/master

merge upstream

Author: sjavora

README.md

@@ -64,6 +64,28 @@ $ SWIFT_DRIVER_ENABLE_INTEGRATION_TESTS=1 \
   swift test --parallel
 ```
 
+#### Preparing a Linux docker for debug
+
+When developing on macOS without quick access to a Linux machine, using a Linux Docker is often helpful when debugging.
+
+To get a docker up and running to the following:
+- Install Docker for Mac.
+- Get the newest swift docker image `docker pull swift`.
+- Run the following command to start a docker
+```
+$ docker run -v /path/to/swift-driver:/home/swift-driver \
+  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
+  --security-opt apparmor=unconfined -it swift:latest bash
+```
+- Install dependencies by running
+```
+$ apt-get update
+$ apt-get install libsqlite3-dev
+$ apt-get install libncurses-dev
+```
+- You can now go to `/home/swift-driver` and run `swift test --parallel` to run your tests.
+
+
 ### Rebuilding `Options.swift`
 
 `Options.swift`, which contains the complete set of options that can be parsed by the driver, is automatically generated from the [option tables in the Swift compiler](https://github.com/apple/swift/tree/master/include/swift/Option). If you need to regenerate `Options.swift`, you will need to [build the Swift compiler](https://github.com/apple/swift#building-swift) and then build `makeOptions` program with a `-I` that allows the generated `Options.inc` to
@@ -100,7 +122,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Implement proper tokenization for response files
 * Compilation modes
   * [x] Batch mode
-  * [ ] Whole-module-optimization mode
+  * [x] Whole-module-optimization mode
   * [ ] REPL mode
   * [ ] Immediate mode
 * Features
@@ -110,6 +132,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Parseable output, as used by SwiftPM
   * [x] Response files
   * [ ] Input and primary input file lists
+  * [ ] Complete `OutputFileMap` implementation to handle all file types uniformly
 * Testing
   * [ ] Build stuff with SwiftPM or Xcode or your favorite build system, using `swift-driver`. Were the results identical? What changed?
   * [x] Shim in `swift-driver` so it can run the Swift repository's [driver test suite](https://github.com/apple/swift/tree/master/test/Driver).

Sources/SwiftDriver/Driver/CompilerMode.swift

@@ -47,6 +47,17 @@ extension CompilerMode {
       return true
     }
   }
+
+  /// Whether this compilation mode compiles the whole target in one job.
+  public var isSingleCompilation: Bool {
+    switch self {
+    case .immediate, .repl, .standardCompile, .batchCompile:
+      return false
+
+    case .singleCompile:
+      return true
+    }
+  }
 }
 
 extension CompilerMode: CustomStringConvertible {

Sources/SwiftDriver/Driver/Driver.swift

@@ -284,16 +284,70 @@ public struct Driver {
     self.enabledSanitizers = try Self.parseSanitizerArgValues(&parsedOptions, diagnosticEngine: diagnosticEngine, toolchain: toolchain, targetTriple: targetTriple)
 
     // Supplemental outputs.
-    self.dependenciesFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .dependencies, isOutput: .emitDependencies, outputPath: .emitDependenciesPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.referenceDependenciesFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .swiftDeps, isOutput: .emitReferenceDependencies, outputPath: .emitReferenceDependenciesPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .diagnostics, isOutput: .serializeDiagnostics, outputPath: .serializeDiagnosticsPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
+    self.dependenciesFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .dependencies, isOutput: .emitDependencies,
+        outputPath: .emitDependenciesPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.referenceDependenciesFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .swiftDeps, isOutput: .emitReferenceDependencies,
+        outputPath: .emitReferenceDependenciesPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .diagnostics, isOutput: .serializeDiagnostics,
+        outputPath: .serializeDiagnosticsPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
     // FIXME: -fixits-output-path
-    self.objcGeneratedHeaderPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .objcHeader, isOutput: .emitObjcHeader, outputPath: .emitObjcHeaderPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .moduleTrace, isOutput: .emitLoadedModuleTrace, outputPath: .emitLoadedModuleTracePath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.tbdPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .tbd, isOutput: .emitTbd, outputPath: .emitTbdPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.moduleDocOutputPath = try Self.computeModuleDocOutputPath(&parsedOptions, moduleOutputPath: self.moduleOutput?.outputPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.swiftInterfacePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .swiftInterface, isOutput: .emitModuleInterface, outputPath: .emitModuleInterfacePath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.optimizationRecordPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .optimizationRecord, isOutput: .saveOptimizationRecord, outputPath: .saveOptimizationRecordPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
+    self.objcGeneratedHeaderPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .objcHeader, isOutput: .emitObjcHeader,
+        outputPath: .emitObjcHeaderPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .moduleTrace, isOutput: .emitLoadedModuleTrace,
+        outputPath: .emitLoadedModuleTracePath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.tbdPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .tbd, isOutput: .emitTbd,
+        outputPath: .emitTbdPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.moduleDocOutputPath = try Self.computeModuleDocOutputPath(
+        &parsedOptions, moduleOutputPath: self.moduleOutput?.outputPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.swiftInterfacePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .swiftInterface, isOutput: .emitModuleInterface,
+        outputPath: .emitModuleInterfacePath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.optimizationRecordPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .optimizationRecord,
+        isOutput: .saveOptimizationRecord,
+        outputPath: .saveOptimizationRecordPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
   }
 }
 
@@ -1249,11 +1303,11 @@ extension Driver {
     isOutput: Option?,
     outputPath: Option,
     compilerOutputType: FileType?,
+    compilerMode: CompilerMode,
+    outputFileMap: OutputFileMap?,
     moduleName: String,
     patternOutputFile: VirtualPath? = nil
   ) throws -> VirtualPath? {
-    // FIXME: Do we need to check the output file map?
-
     // If there is an explicit argument for the output path, use that
     if let outputPathArg = parsedOptions.getLastArgument(outputPath) {
       // Consume the isOutput argument
@@ -1268,6 +1322,14 @@ extension Driver {
       return nil
     }
 
+    // If this is a single-file compile and there is an entry in the
+    // output file map, use that.
+    if compilerMode.isSingleCompilation,
+        let singleOutputPath = outputFileMap?.existingOutputForSingleInput(
+            outputType: type) {
+      return singleOutputPath
+    }
+
     // If there is an output argument, derive the name from there.
     if let outputPathArg = parsedOptions.getLastArgument(.o) {
       let path = try VirtualPath(path: outputPathArg.asSingle)
@@ -1288,10 +1350,10 @@ extension Driver {
     _ parsedOptions: inout ParsedOptions,
     moduleOutputPath: VirtualPath?,
     compilerOutputType: FileType?,
+    compilerMode: CompilerMode,
+    outputFileMap: OutputFileMap?,
     moduleName: String
   ) throws -> VirtualPath? {
-    // FIXME: Do we need to check the output file map?
-
     // If there is an explicit argument for the output path, use that
     if let outputPathArg = parsedOptions.getLastArgument(.emitModuleDocPath) {
       // Consume -emit-module-doc if it's there.
@@ -1300,6 +1362,14 @@ extension Driver {
       return try VirtualPath(path: outputPathArg.asSingle)
     }
 
+    // If this is a single-file compile and there is an entry in the
+    // output file map, use that.
+    if compilerMode.isSingleCompilation,
+        let singleOutputPath = outputFileMap?.existingOutputForSingleInput(
+          outputType: .swiftDocumentation) {
+      return singleOutputPath
+    }
+
     // If there's a known module output path, put the .swiftdoc file next
     // to it.
     if let moduleOutputPath = moduleOutputPath {

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -113,13 +113,17 @@ fileprivate struct OutputFileMapJSON: Codable {
       case dependencies
       case object
       case swiftmodule
+      case swiftinterface
       case swiftDependencies = "swift-dependencies"
+      case diagnostics
     }
 
     let dependencies: String?
     let object: String?
     let swiftmodule: String?
+    let swiftinterface: String?
     let swiftDependencies: String?
+    let diagnostics: String?
   }
 
   /// The parsed entires
@@ -146,7 +150,9 @@ fileprivate struct OutputFileMapJSON: Codable {
       map[.dependencies] = entry.dependencies
       map[.object] = entry.object
       map[.swiftModule] = entry.swiftmodule
+      map[.swiftInterface] = entry.swiftinterface
       map[.swiftDeps] = entry.swiftDependencies
+      map[.diagnostics] = entry.diagnostics
 
       result[input] = try map.mapValues(VirtualPath.init(path:))
     }
@@ -168,7 +174,9 @@ fileprivate struct OutputFileMapJSON: Codable {
         dependencies: outputs[.dependencies]?.name,
         object: outputs[.object]?.name,
         swiftmodule: outputs[.swiftModule]?.name,
-        swiftDependencies: outputs[.swiftDeps]?.name)
+        swiftinterface: outputs[.swiftInterface]?.name,
+        swiftDependencies: outputs[.swiftDeps]?.name,
+        diagnostics: outputs[.diagnostics]?.name)
     }
     return Self(entries: Dictionary(uniqueKeysWithValues: entries.map(convert(entry:))))
   }

Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift

@@ -32,7 +32,7 @@ public struct ToolExecutionDelegate: JobExecutorDelegate {
 
       // Compute the outputs for the message.
       let outputs: [BeganMessage.Output] = job.outputs.map {
-        .init(path: $0.file.name, type: $0.type.rawValue)
+        .init(path: $0.file.name, type: $0.type.description)
       }
 
       let beganMessage = BeganMessage(

Sources/SwiftDriver/Incremental Compilation/IncrementalCompilation.swift

@@ -160,7 +160,7 @@ fileprivate extension CompilerMode {
   }
 }
 
-public extension Diagnostic.Message {
+extension Diagnostic.Message {
   static var warning_incremental_requires_output_file_map: Diagnostic.Message {
     .warning("ignoring -incremental (currently requires an output file map)")
   }

Sources/SwiftDriver/Incremental Compilation/InputIInfoMap.swift

@@ -169,7 +169,7 @@ public extension InputInfoMap {
 }
 
 
-public extension Diagnostic.Message {
+extension Diagnostic.Message {
   static func remark_could_not_read_build_record(_ error: Error) -> Diagnostic.Message {
     .remark("Incremental compilation could not read build record: \(error.localizedDescription).")
   }

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -54,8 +54,10 @@ extension Driver {
       }
       commandLine.append(.path(input.file))
 
-      // If there is a primary output, add it.
-      if isPrimary, let compilerOutputType = compilerOutputType {
+      // If there is a primary output or we are doing multithreaded compiles,
+      // add an output for the input.
+      if isPrimary || numThreads > 0,
+          let compilerOutputType = compilerOutputType {
         let output = (outputFileMap ?? OutputFileMap()).getOutput(
           inputFile: input.file,
           outputType: compilerOutputType
@@ -64,6 +66,15 @@ extension Driver {
       }
     }
 
+    // When not using primary file inputs or multithreading, add a single output.
+    if !usesPrimaryFileInputs && numThreads == 0,
+        let outputType = compilerOutputType {
+      let existingOutputPath = outputFileMap?.existingOutputForSingleInput(
+          outputType: outputType)
+      let output = existingOutputPath ?? VirtualPath.temporary(.init(moduleName.appendingFileTypeExtension(outputType)))
+      primaryOutputs.append(TypedVirtualPath(file: output, type: outputType))
+    }
+
     return primaryOutputs
   }
 

Sources/SwiftDriver/Jobs/EmitModuleJob.swift

@@ -10,6 +10,37 @@
 //
 //===----------------------------------------------------------------------===//
 extension Driver {
+  /// Add options that are common to command lines that emit modules, e.g.,
+  /// options for the paths of various module files.
+  mutating func addCommonModuleOptions(
+      commandLine: inout [Job.ArgTemplate],
+      outputs: inout [TypedVirtualPath]
+  ) throws {
+    // Add supplemental outputs.
+    func addSupplementalOutput(path: VirtualPath?, flag: String, type: FileType) {
+      guard let path = path else { return }
+
+      commandLine.appendFlag(flag)
+      commandLine.appendPath(path)
+      outputs.append(.init(file: path, type: type))
+    }
+
+    addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation)
+    addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface)
+    addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics)
+    addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
+    addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)
+
+    if let dependenciesFilePath = dependenciesFilePath {
+      var path = dependenciesFilePath
+      // FIXME: Hack to workaround the fact that SwiftPM/Xcode don't pass this path right now.
+      if parsedOptions.getLastArgument(.emitDependenciesPath) == nil {
+        path = try moduleOutput!.outputPath.replacingExtension(with: .dependencies)
+      }
+      addSupplementalOutput(path: path, flag: "-emit-dependencies-path", type: .dependencies)
+    }
+  }
+
   /// Form a job that emits a single module
   mutating func emitModuleJob() throws -> Job {
     let moduleOutputPath = moduleOutput!.outputPath
@@ -32,29 +63,7 @@ extension Driver {
     try addCommonFrontendOptions(commandLine: &commandLine)
     // FIXME: Add MSVC runtime library flags
 
-    // Add suppplementable outputs.
-    func addSupplementalOutput(path: VirtualPath?, flag: String, type: FileType) {
-      guard let path = path else { return }
-
-      commandLine.appendFlag(flag)
-      commandLine.appendPath(path)
-      outputs.append(.init(file: path, type: type))
-    }
-
-    addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation)
-    addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface)
-    addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics)
-    addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
-    addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)
-
-    if let dependenciesFilePath = dependenciesFilePath {
-      var path = dependenciesFilePath
-      // FIXME: Hack to workaround the fact that SwiftPM/Xcode don't pass this path right now.
-      if parsedOptions.getLastArgument(.emitDependenciesPath) == nil {
-        path = try moduleOutputPath.replacingExtension(with: .dependencies)
-      }
-      addSupplementalOutput(path: path, flag: "-emit-dependencies-path", type: .dependencies)
-    }
+    try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
 
     commandLine.appendFlag(.o)
     commandLine.appendPath(moduleOutputPath)