Merge pull request #23 from apple/wmo

Whole module optimization mode

Author: DougGregor

README.md

@@ -122,7 +122,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Implement proper tokenization for response files
 * Compilation modes
   * [x] Batch mode
-  * [ ] Whole-module-optimization mode
+  * [x] Whole-module-optimization mode
   * [ ] REPL mode
   * [ ] Immediate mode
 * Features
@@ -132,6 +132,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Parseable output, as used by SwiftPM
   * [x] Response files
   * [ ] Input and primary input file lists
+  * [ ] Complete `OutputFileMap` implementation to handle all file types uniformly
 * Testing
   * [ ] Build stuff with SwiftPM or Xcode or your favorite build system, using `swift-driver`. Were the results identical? What changed?
   * [x] Shim in `swift-driver` so it can run the Swift repository's [driver test suite](https://github.com/apple/swift/tree/master/test/Driver).

Sources/SwiftDriver/Driver/CompilerMode.swift

@@ -47,6 +47,17 @@ extension CompilerMode {
       return true
     }
   }
+
+  /// Whether this compilation mode compiles the whole target in one job.
+  public var isSingleCompilation: Bool {
+    switch self {
+    case .immediate, .repl, .standardCompile, .batchCompile:
+      return false
+
+    case .singleCompile:
+      return true
+    }
+  }
 }
 
 extension CompilerMode: CustomStringConvertible {

Sources/SwiftDriver/Driver/Driver.swift

@@ -284,16 +284,70 @@ public struct Driver {
     self.enabledSanitizers = try Self.parseSanitizerArgValues(&parsedOptions, diagnosticEngine: diagnosticEngine, toolchain: toolchain, targetTriple: targetTriple)
 
     // Supplemental outputs.
-    self.dependenciesFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .dependencies, isOutput: .emitDependencies, outputPath: .emitDependenciesPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.referenceDependenciesFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .swiftDeps, isOutput: .emitReferenceDependencies, outputPath: .emitReferenceDependenciesPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .diagnostics, isOutput: .serializeDiagnostics, outputPath: .serializeDiagnosticsPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
+    self.dependenciesFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .dependencies, isOutput: .emitDependencies,
+        outputPath: .emitDependenciesPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.referenceDependenciesFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .swiftDeps, isOutput: .emitReferenceDependencies,
+        outputPath: .emitReferenceDependenciesPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .diagnostics, isOutput: .serializeDiagnostics,
+        outputPath: .serializeDiagnosticsPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
     // FIXME: -fixits-output-path
-    self.objcGeneratedHeaderPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .objcHeader, isOutput: .emitObjcHeader, outputPath: .emitObjcHeaderPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .moduleTrace, isOutput: .emitLoadedModuleTrace, outputPath: .emitLoadedModuleTracePath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.tbdPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .tbd, isOutput: .emitTbd, outputPath: .emitTbdPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.moduleDocOutputPath = try Self.computeModuleDocOutputPath(&parsedOptions, moduleOutputPath: self.moduleOutput?.outputPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.swiftInterfacePath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .swiftInterface, isOutput: .emitModuleInterface, outputPath: .emitModuleInterfacePath, compilerOutputType: compilerOutputType, moduleName: moduleName)
-    self.optimizationRecordPath = try Self.computeSupplementaryOutputPath(&parsedOptions, type: .optimizationRecord, isOutput: .saveOptimizationRecord, outputPath: .saveOptimizationRecordPath, compilerOutputType: compilerOutputType, moduleName: moduleName)
+    self.objcGeneratedHeaderPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .objcHeader, isOutput: .emitObjcHeader,
+        outputPath: .emitObjcHeaderPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .moduleTrace, isOutput: .emitLoadedModuleTrace,
+        outputPath: .emitLoadedModuleTracePath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.tbdPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .tbd, isOutput: .emitTbd,
+        outputPath: .emitTbdPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.moduleDocOutputPath = try Self.computeModuleDocOutputPath(
+        &parsedOptions, moduleOutputPath: self.moduleOutput?.outputPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.swiftInterfacePath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .swiftInterface, isOutput: .emitModuleInterface,
+        outputPath: .emitModuleInterfacePath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
+    self.optimizationRecordPath = try Self.computeSupplementaryOutputPath(
+        &parsedOptions, type: .optimizationRecord,
+        isOutput: .saveOptimizationRecord,
+        outputPath: .saveOptimizationRecordPath,
+        compilerOutputType: compilerOutputType,
+        compilerMode: compilerMode,
+        outputFileMap: self.outputFileMap,
+        moduleName: moduleName)
   }
 }
 
@@ -1249,11 +1303,11 @@ extension Driver {
     isOutput: Option?,
     outputPath: Option,
     compilerOutputType: FileType?,
+    compilerMode: CompilerMode,
+    outputFileMap: OutputFileMap?,
     moduleName: String,
     patternOutputFile: VirtualPath? = nil
   ) throws -> VirtualPath? {
-    // FIXME: Do we need to check the output file map?
-
     // If there is an explicit argument for the output path, use that
     if let outputPathArg = parsedOptions.getLastArgument(outputPath) {
       // Consume the isOutput argument
@@ -1268,6 +1322,14 @@ extension Driver {
       return nil
     }
 
+    // If this is a single-file compile and there is an entry in the
+    // output file map, use that.
+    if compilerMode.isSingleCompilation,
+        let singleOutputPath = outputFileMap?.existingOutputForSingleInput(
+            outputType: type) {
+      return singleOutputPath
+    }
+
     // If there is an output argument, derive the name from there.
     if let outputPathArg = parsedOptions.getLastArgument(.o) {
       let path = try VirtualPath(path: outputPathArg.asSingle)
@@ -1288,10 +1350,10 @@ extension Driver {
     _ parsedOptions: inout ParsedOptions,
     moduleOutputPath: VirtualPath?,
     compilerOutputType: FileType?,
+    compilerMode: CompilerMode,
+    outputFileMap: OutputFileMap?,
     moduleName: String
   ) throws -> VirtualPath? {
-    // FIXME: Do we need to check the output file map?
-
     // If there is an explicit argument for the output path, use that
     if let outputPathArg = parsedOptions.getLastArgument(.emitModuleDocPath) {
       // Consume -emit-module-doc if it's there.
@@ -1300,6 +1362,14 @@ extension Driver {
       return try VirtualPath(path: outputPathArg.asSingle)
     }
 
+    // If this is a single-file compile and there is an entry in the
+    // output file map, use that.
+    if compilerMode.isSingleCompilation,
+        let singleOutputPath = outputFileMap?.existingOutputForSingleInput(
+          outputType: .swiftDocumentation) {
+      return singleOutputPath
+    }
+
     // If there's a known module output path, put the .swiftdoc file next
     // to it.
     if let moduleOutputPath = moduleOutputPath {

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -113,13 +113,17 @@ fileprivate struct OutputFileMapJSON: Codable {
       case dependencies
       case object
       case swiftmodule
+      case swiftinterface
       case swiftDependencies = "swift-dependencies"
+      case diagnostics
     }
 
     let dependencies: String?
     let object: String?
     let swiftmodule: String?
+    let swiftinterface: String?
     let swiftDependencies: String?
+    let diagnostics: String?
   }
 
   /// The parsed entires
@@ -146,7 +150,9 @@ fileprivate struct OutputFileMapJSON: Codable {
       map[.dependencies] = entry.dependencies
       map[.object] = entry.object
       map[.swiftModule] = entry.swiftmodule
+      map[.swiftInterface] = entry.swiftinterface
       map[.swiftDeps] = entry.swiftDependencies
+      map[.diagnostics] = entry.diagnostics
 
       result[input] = try map.mapValues(VirtualPath.init(path:))
     }
@@ -168,7 +174,9 @@ fileprivate struct OutputFileMapJSON: Codable {
         dependencies: outputs[.dependencies]?.name,
         object: outputs[.object]?.name,
         swiftmodule: outputs[.swiftModule]?.name,
-        swiftDependencies: outputs[.swiftDeps]?.name)
+        swiftinterface: outputs[.swiftInterface]?.name,
+        swiftDependencies: outputs[.swiftDeps]?.name,
+        diagnostics: outputs[.diagnostics]?.name)
     }
     return Self(entries: Dictionary(uniqueKeysWithValues: entries.map(convert(entry:))))
   }

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -54,8 +54,10 @@ extension Driver {
       }
       commandLine.append(.path(input.file))
 
-      // If there is a primary output, add it.
-      if isPrimary, let compilerOutputType = compilerOutputType {
+      // If there is a primary output or we are doing multithreaded compiles,
+      // add an output for the input.
+      if isPrimary || numThreads > 0,
+          let compilerOutputType = compilerOutputType {
         let output = (outputFileMap ?? OutputFileMap()).getOutput(
           inputFile: input.file,
           outputType: compilerOutputType
@@ -64,6 +66,15 @@ extension Driver {
       }
     }
 
+    // When not using primary file inputs or multithreading, add a single output.
+    if !usesPrimaryFileInputs && numThreads == 0,
+        let outputType = compilerOutputType {
+      let existingOutputPath = outputFileMap?.existingOutputForSingleInput(
+          outputType: outputType)
+      let output = existingOutputPath ?? VirtualPath.temporary(.init(moduleName.appendingFileTypeExtension(outputType)))
+      primaryOutputs.append(TypedVirtualPath(file: output, type: outputType))
+    }
+
     return primaryOutputs
   }
 

Sources/SwiftDriver/Jobs/EmitModuleJob.swift

@@ -10,6 +10,37 @@
 //
 //===----------------------------------------------------------------------===//
 extension Driver {
+  /// Add options that are common to command lines that emit modules, e.g.,
+  /// options for the paths of various module files.
+  mutating func addCommonModuleOptions(
+      commandLine: inout [Job.ArgTemplate],
+      outputs: inout [TypedVirtualPath]
+  ) throws {
+    // Add supplemental outputs.
+    func addSupplementalOutput(path: VirtualPath?, flag: String, type: FileType) {
+      guard let path = path else { return }
+
+      commandLine.appendFlag(flag)
+      commandLine.appendPath(path)
+      outputs.append(.init(file: path, type: type))
+    }
+
+    addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation)
+    addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface)
+    addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics)
+    addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
+    addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)
+
+    if let dependenciesFilePath = dependenciesFilePath {
+      var path = dependenciesFilePath
+      // FIXME: Hack to workaround the fact that SwiftPM/Xcode don't pass this path right now.
+      if parsedOptions.getLastArgument(.emitDependenciesPath) == nil {
+        path = try moduleOutput!.outputPath.replacingExtension(with: .dependencies)
+      }
+      addSupplementalOutput(path: path, flag: "-emit-dependencies-path", type: .dependencies)
+    }
+  }
+
   /// Form a job that emits a single module
   mutating func emitModuleJob() throws -> Job {
     let moduleOutputPath = moduleOutput!.outputPath
@@ -32,29 +63,7 @@ extension Driver {
     try addCommonFrontendOptions(commandLine: &commandLine)
     // FIXME: Add MSVC runtime library flags
 
-    // Add suppplementable outputs.
-    func addSupplementalOutput(path: VirtualPath?, flag: String, type: FileType) {
-      guard let path = path else { return }
-
-      commandLine.appendFlag(flag)
-      commandLine.appendPath(path)
-      outputs.append(.init(file: path, type: type))
-    }
-
-    addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation)
-    addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface)
-    addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics)
-    addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
-    addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)
-
-    if let dependenciesFilePath = dependenciesFilePath {
-      var path = dependenciesFilePath
-      // FIXME: Hack to workaround the fact that SwiftPM/Xcode don't pass this path right now.
-      if parsedOptions.getLastArgument(.emitDependenciesPath) == nil {
-        path = try moduleOutputPath.replacingExtension(with: .dependencies)
-      }
-      addSupplementalOutput(path: path, flag: "-emit-dependencies-path", type: .dependencies)
-    }
+    try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
 
     commandLine.appendFlag(.o)
     commandLine.appendPath(moduleOutputPath)