[5.5] Cherry-pick changes about ABI checking integration

Sources/SwiftDriver/CMakeLists.txt

@@ -55,6 +55,7 @@ add_library(SwiftDriver
   "IncrementalCompilation/SourceFileDependencyGraph.swift"
   "IncrementalCompilation/TwoDMap.swift"
 
+  Jobs/APIDigesterJobs.swift
   Jobs/AutolinkExtractJob.swift
   Jobs/BackendJob.swift
   Jobs/CommandLineArguments.swift

Sources/SwiftDriver/Driver/Driver.swift

@@ -35,6 +35,8 @@ public struct Driver {
     case missingProfilingData(String)
     case conditionalCompilationFlagHasRedundantPrefix(String)
     case conditionalCompilationFlagIsNotValidIdentifier(String)
+    case baselineGenerationRequiresTopLevelModule(String)
+    case optionRequiresAnother(String, String)
     // Explicit Module Build Failures
     case malformedModuleDependency(String, String)
     case missingPCMArguments(String)
@@ -103,6 +105,10 @@ public struct Driver {
         return "unable to load output file map '\(path)': no such file or directory"
       case .missingExternalDependency(let moduleName):
         return "Missing External dependency info for module: \(moduleName)"
+      case .baselineGenerationRequiresTopLevelModule(let arg):
+        return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'"
+      case .optionRequiresAnother(let first, let second):
+        return "'\(first)' cannot be specified if '\(second)' is not present"
       }
     }
   }
@@ -291,6 +297,12 @@ public struct Driver {
   /// Path to the Swift module source information file.
   let moduleSourceInfoPath: VirtualPath.Handle?
 
+  /// Path to the module's digester baseline file.
+  let digesterBaselinePath: VirtualPath.Handle?
+
+  /// The mode the API digester should run in.
+  let digesterMode: DigesterMode
+
   /// Force the driver to emit the module first and then run compile jobs. This could be used to unblock
   /// dependencies in parallel builds.
   var forceEmitModuleBeforeCompile: Bool = false
@@ -539,6 +551,16 @@ public struct Driver {
     self.numThreads = Self.determineNumThreads(&parsedOptions, compilerMode: compilerMode, diagnosticsEngine: diagnosticEngine)
     self.numParallelJobs = Self.determineNumParallelJobs(&parsedOptions, diagnosticsEngine: diagnosticEngine, env: env)
 
+    var mode = DigesterMode.api
+    if let modeArg = parsedOptions.getLastArgument(.digesterMode)?.asSingle {
+      if let digesterMode = DigesterMode(rawValue: modeArg) {
+        mode = digesterMode
+      } else {
+        diagnosticsEngine.emit(Error.invalidArgumentValue(Option.digesterMode.spelling, modeArg))
+      }
+    }
+    self.digesterMode = mode
+
     Self.validateWarningControlArgs(&parsedOptions, diagnosticEngine: diagnosticEngine)
     Self.validateProfilingArgs(&parsedOptions,
                                fileSystem: fileSystem,
@@ -591,7 +613,7 @@ public struct Driver {
       diagnosticEngine: diagnosticEngine,
       toolchain: toolchain,
       targetInfo: frontendTargetInfo)
-    
+
     Self.validateSanitizerAddressUseOdrIndicatorFlag(&parsedOptions, diagnosticEngine: diagnosticsEngine, addressSanitizerEnabled: enabledSanitizers.contains(.address))
 
     Self.validateSanitizerRecoverArgValues(&parsedOptions, diagnosticEngine: diagnosticsEngine, enabledSanitizers: enabledSanitizers)
@@ -667,6 +689,15 @@ public struct Driver {
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name,
         projectDirectory: projectDirectory)
+    self.digesterBaselinePath = try Self.computeDigesterBaselineOutputPath(
+      &parsedOptions,
+      moduleOutputPath: self.moduleOutputInfo.output?.outputPath,
+      mode: self.digesterMode,
+      compilerOutputType: compilerOutputType,
+      compilerMode: compilerMode,
+      outputFileMap: self.outputFileMap,
+      moduleName: moduleOutputInfo.name,
+      projectDirectory: projectDirectory)
     self.swiftInterfacePath = try Self.computeSupplementaryOutputPath(
         &parsedOptions, type: .swiftInterface, isOutputOptions: [.emitModuleInterface],
         outputPath: .emitModuleInterfacePath,
@@ -704,6 +735,12 @@ public struct Driver {
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name)
 
+    Self.validateDigesterArgs(&parsedOptions,
+                              moduleOutputInfo: moduleOutputInfo,
+                              digesterMode: self.digesterMode,
+                              swiftInterfacePath: self.swiftInterfacePath,
+                              diagnosticEngine: diagnosticsEngine)
+
     try verifyOutputOptions()
   }
 
@@ -2182,6 +2219,36 @@ extension Driver {
     }
   }
 
+  static func validateDigesterArgs(_ parsedOptions: inout ParsedOptions,
+                                   moduleOutputInfo: ModuleOutputInfo,
+                                   digesterMode: DigesterMode,
+                                   swiftInterfacePath: VirtualPath.Handle?,
+                                   diagnosticEngine: DiagnosticsEngine) {
+    if moduleOutputInfo.output?.isTopLevel != true {
+      for arg in parsedOptions.arguments(for: .emitDigesterBaseline, .emitDigesterBaselinePath, .compareToBaselinePath) {
+        diagnosticEngine.emit(Error.baselineGenerationRequiresTopLevelModule(arg.option.spelling))
+      }
+    }
+
+    if parsedOptions.hasArgument(.serializeBreakingChangesPath) && !parsedOptions.hasArgument(.compareToBaselinePath) {
+      diagnosticEngine.emit(Error.optionRequiresAnother(Option.serializeBreakingChangesPath.spelling,
+                                                        Option.compareToBaselinePath.spelling))
+    }
+    if parsedOptions.hasArgument(.digesterBreakageAllowlistPath) && !parsedOptions.hasArgument(.compareToBaselinePath) {
+      diagnosticEngine.emit(Error.optionRequiresAnother(Option.digesterBreakageAllowlistPath.spelling,
+                                                        Option.compareToBaselinePath.spelling))
+    }
+    if digesterMode == .abi && !parsedOptions.hasArgument(.enableLibraryEvolution) {
+      diagnosticEngine.emit(Error.optionRequiresAnother("\(Option.digesterMode.spelling) abi",
+                                                        Option.enableLibraryEvolution.spelling))
+    }
+    if digesterMode == .abi && swiftInterfacePath == nil {
+      diagnosticEngine.emit(Error.optionRequiresAnother("\(Option.digesterMode.spelling) abi",
+                                                        Option.emitModuleInterface.spelling))
+    }
+  }
+
+
   static func validateProfilingArgs(_ parsedOptions: inout ParsedOptions,
                                     fileSystem: FileSystem,
                                     workingDirectory: AbsolutePath?,
@@ -2634,6 +2701,31 @@ extension Driver {
                                                 projectDirectory: projectDirectory)
   }
 
+  static func computeDigesterBaselineOutputPath(
+    _ parsedOptions: inout ParsedOptions,
+    moduleOutputPath: VirtualPath.Handle?,
+    mode: DigesterMode,
+    compilerOutputType: FileType?,
+    compilerMode: CompilerMode,
+    outputFileMap: OutputFileMap?,
+    moduleName: String,
+    projectDirectory: VirtualPath.Handle?
+  ) throws -> VirtualPath.Handle? {
+    // Only emit a baseline if at least of the arguments was provided.
+    guard parsedOptions.hasArgument(.emitDigesterBaseline, .emitDigesterBaselinePath) else { return nil }
+    return try computeModuleAuxiliaryOutputPath(&parsedOptions,
+                                                moduleOutputPath: moduleOutputPath,
+                                                type: mode.baselineFileType,
+                                                isOutput: .emitDigesterBaseline,
+                                                outputPath: .emitDigesterBaselinePath,
+                                                compilerOutputType: compilerOutputType,
+                                                compilerMode: compilerMode,
+                                                outputFileMap: outputFileMap,
+                                                moduleName: moduleName,
+                                                projectDirectory: projectDirectory)
+  }
+
+
 
   /// Determine the output path for a module auxiliary output.
   static func computeModuleAuxiliaryOutputPath(

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -54,6 +54,13 @@ public struct OutputFileMap: Hashable, Codable {
       }
       return VirtualPath.lookup(path).replacingExtension(with: outputType).intern()
 
+    case .jsonAPIBaseline, .jsonABIBaseline:
+      // Infer paths for these entities using .swiftsourceinfo path.
+      guard let path = entries[inputFile]?[.swiftSourceInfoFile] else {
+        return nil
+      }
+      return VirtualPath.lookup(path).replacingExtension(with: outputType).intern()
+
     case .object:
       // We may generate .o files from bitcode .bc files, but the output file map
       // uses .swift file as the key for .o file paths. So we need to dig further.

Sources/SwiftDriver/Jobs/APIDigesterJobs.swift

@@ -0,0 +1,152 @@
+//===- APIDigesterJobs.swift - Baseline Generation and API/ABI Comparison -===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import TSCBasic
+
+enum DigesterMode: String {
+  case api, abi
+
+  var baselineFileType: FileType {
+    switch self {
+    case .api:
+      return .jsonAPIBaseline
+    case .abi:
+      return .jsonABIBaseline
+    }
+  }
+
+  var baselineGenerationJobKind: Job.Kind {
+    switch self {
+    case .api:
+      return .generateAPIBaseline
+    case .abi:
+      return .generateABIBaseline
+    }
+  }
+
+  var baselineComparisonJobKind: Job.Kind {
+    switch self {
+    case .api:
+      return .compareAPIBaseline
+    case .abi:
+      return .compareABIBaseline
+    }
+  }
+}
+
+extension Driver {
+  mutating func digesterBaselineGenerationJob(modulePath: VirtualPath.Handle, outputPath: VirtualPath.Handle, mode: DigesterMode) throws -> Job {
+    var commandLine = [Job.ArgTemplate]()
+    commandLine.appendFlag("-dump-sdk")
+
+    try addCommonDigesterOptions(&commandLine, modulePath: modulePath, mode: mode)
+
+    commandLine.appendFlag(.o)
+    commandLine.appendPath(VirtualPath.lookup(outputPath))
+
+    return Job(
+      moduleName: moduleOutputInfo.name,
+      kind: mode.baselineGenerationJobKind,
+      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      commandLine: commandLine,
+      inputs: [.init(file: modulePath, type: .swiftModule)],
+      primaryInputs: [],
+      outputs: [.init(file: outputPath, type: mode.baselineFileType)],
+      supportsResponseFiles: true
+    )
+  }
+
+  mutating func digesterCompareToBaselineJob(modulePath: VirtualPath.Handle, baselinePath: VirtualPath.Handle, mode: DigesterMode) throws -> Job {
+    var commandLine = [Job.ArgTemplate]()
+    commandLine.appendFlag("-diagnose-sdk")
+    commandLine.appendFlag("-disable-fail-on-error")
+    commandLine.appendFlag("-baseline-path")
+    commandLine.appendPath(VirtualPath.lookup(baselinePath))
+
+    try addCommonDigesterOptions(&commandLine, modulePath: modulePath, mode: mode)
+
+    var serializedDiagnosticsPath: VirtualPath.Handle?
+    if let arg = parsedOptions.getLastArgument(.serializeBreakingChangesPath)?.asSingle {
+      let path = try VirtualPath.intern(path: arg)
+      commandLine.appendFlag("-serialize-diagnostics-path")
+      commandLine.appendPath(VirtualPath.lookup(path))
+      serializedDiagnosticsPath = path
+    }
+    if let arg = parsedOptions.getLastArgument(.digesterBreakageAllowlistPath)?.asSingle {
+      let path = try VirtualPath(path: arg)
+      commandLine.appendFlag("-breakage-allowlist-path")
+      commandLine.appendPath(path)
+    }
+
+    var inputs: [TypedVirtualPath] = [.init(file: modulePath, type: .swiftModule),
+                                      .init(file: baselinePath, type: mode.baselineFileType)]
+    // If a module interface was emitted, treat it as an input in ABI mode.
+    if let interfacePath = self.swiftInterfacePath, mode == .abi {
+      inputs.append(.init(file: interfacePath, type: .swiftInterface))
+    }
+
+    return Job(
+      moduleName: moduleOutputInfo.name,
+      kind: mode.baselineComparisonJobKind,
+      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      commandLine: commandLine,
+      inputs: inputs,
+      primaryInputs: [],
+      outputs: [.init(file: serializedDiagnosticsPath ?? VirtualPath.Handle.standardOutput, type: .diagnostics)],
+      supportsResponseFiles: true
+    )
+  }
+
+  private mutating func addCommonDigesterOptions(_ commandLine: inout [Job.ArgTemplate],
+                                                 modulePath: VirtualPath.Handle,
+                                                 mode: DigesterMode) throws {
+    commandLine.appendFlag("-module")
+    commandLine.appendFlag(moduleOutputInfo.name)
+    if mode == .abi {
+      commandLine.appendFlag("-abi")
+      commandLine.appendFlag("-use-interface-for-module")
+      commandLine.appendFlag(moduleOutputInfo.name)
+    }
+
+    // Add a search path for the emitted module, and its module interface if there is one.
+    let searchPath = VirtualPath.lookup(modulePath).parentDirectory
+    commandLine.appendFlag(.I)
+    commandLine.appendPath(searchPath)
+    if let interfacePath = self.swiftInterfacePath  {
+      let interfaceSearchPath = VirtualPath.lookup(interfacePath).parentDirectory
+      if interfaceSearchPath != searchPath {
+        commandLine.appendFlag(.I)
+        commandLine.appendPath(interfaceSearchPath)
+      }
+    }
+
+    commandLine.appendFlag(.target)
+    commandLine.appendFlag(targetTriple.triple)
+
+    if let sdkPath = frontendTargetInfo.sdkPath?.path {
+      commandLine.appendFlag(.sdk)
+      commandLine.append(.path(VirtualPath.lookup(sdkPath)))
+    }
+
+    commandLine.appendFlag(.resourceDir)
+    commandLine.appendPath(VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path))
+
+    try commandLine.appendAll(.I, from: &parsedOptions)
+    try commandLine.appendAll(.F, from: &parsedOptions)
+    for systemFramework in parsedOptions.arguments(for: .Fsystem) {
+      commandLine.appendFlag(.iframework)
+      commandLine.appendFlag(systemFramework.argument.asSingle)
+    }
+
+    try commandLine.appendLast(.swiftVersion, from: &parsedOptions)
+  }
+}

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -93,7 +93,7 @@ extension Driver {
          .privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
          .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
          .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
-         .indexUnitOutputPath, .modDepCache, nil:
+         .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline, nil:
       return false
     }
   }
@@ -444,7 +444,7 @@ extension FileType {
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
          .yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
          .privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
-         .indexUnitOutputPath, .modDepCache:
+         .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline:
       fatalError("Output type can never be a primary output")
     }
   }