Add initial experimental support for combined documentation for multiple targets (#84)

* Add a minimal build graph for documentation tasks

rdar://116698361

* Build documentation for targets in reverse dependency order

rdar://116698361

* Fix unrelated warning about a deprecated (renamed) DocC flag

* Combine nested conditionals into one if-statement

* Decode the supported features for a given DocC executable

* List all the generated documentation archvies

* Add flag to enable combined documentation support

This flag allows the targets to link to each other
and creates an additional combined archive.

rdar://116698361

* Warn if the DocC executable doesn't support combined documentation

* Update integration tests to more explicitly check for archive paths in console output

* Update check-source to include 2024 as a supported year

* Address code review feedback:

- Check for errors after queue has run the build operations
- Avoid repeat-visiting targets when constructing the build graph
- Update internal-only documentation comment

* Add a type to encapsulate performing work for each build graph item

* Remove extra blank line before license comment which cause a false-positive source validation error

Author: d-ronnqvist

IntegrationTests/Tests/Utility/XCTestCase+swiftPackage.swift

@@ -144,14 +144,20 @@ struct SwiftInvocationResult {
 
     var referencedDocCArchives: [URL] {
         return standardOutput
-            .components(separatedBy: .whitespacesAndNewlines)
-            .map { component in
-                return component.trimmingCharacters(in: CharacterSet(charactersIn: "'."))
+            .components(separatedBy: .newlines)
+            .filter { line in
+                line.hasPrefix("Generated DocC archive at")
             }
-            .filter { component in
-                return component.hasSuffix(".doccarchive")
+            .flatMap { line in
+                line.components(separatedBy: .whitespaces)
+                    .map { component in
+                        return component.trimmingCharacters(in: CharacterSet(charactersIn: "'."))
+                    }
+                    .filter { component in
+                        return component.hasSuffix(".doccarchive")
+                    }
+                    .compactMap(URL.init(fileURLWithPath:))
             }
-            .compactMap(URL.init(fileURLWithPath:))
     }
 
     var pluginOutputsDirectory: URL {

Plugins/Swift-DocC Convert/SwiftDocCConvert.swift

@@ -1,6 +1,6 @@
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Copyright (c) 2022-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -30,6 +30,19 @@ import PackagePlugin
         }
 
         let verbose = argumentExtractor.extractFlag(named: "verbose") > 0
+        let isCombinedDocumentationEnabled = argumentExtractor.extractFlag(named: PluginFlag.enableCombinedDocumentationSupportFlagName) > 0
+        
+        if isCombinedDocumentationEnabled {
+            let doccFeatures = try? DocCFeatures(doccExecutable: doccExecutableURL)
+            guard doccFeatures?.contains(.linkDependencies) == true else {
+                // The developer uses the combined documentation plugin flag with a DocC version that doesn't support combined documentation.
+                Diagnostics.error("""
+                Unsupported use of '--\(PluginFlag.enableCombinedDocumentationSupportFlagName)'. \
+                DocC version at '\(doccExecutableURL.path)' doesn't support combined documentation.
+                """)
+                return
+            }
+        }
 
         // Parse the given command-line arguments
         let parsedArguments = ParsedArguments(argumentExtractor.remainingArguments)
@@ -56,14 +69,9 @@ import PackagePlugin
         let snippetExtractor: SnippetExtractor? = nil
 #endif
 
-        
-        // Iterate over the Swift source module targets we were given.
-        for (index, target) in swiftSourceModuleTargets.enumerated() {
-            if index != 0 {
-                // Emit a line break if this is not the first target being built.
-                print()
-            }
-            
+        // An inner function that defines the work to build documentation for a given target.
+        func performBuildTask(_ task: DocumentationBuildGraph<SwiftSourceModuleTarget>.Task) throws -> URL? {
+            let target = task.target
             print("Generating documentation for '\(target.name)'...")
 
             let symbolGraphs = try packageManager.doccSymbolGraphs(
@@ -74,27 +82,24 @@ import PackagePlugin
                 customSymbolGraphOptions: parsedArguments.symbolGraphArguments
             )
 
-            if try FileManager.default.contentsOfDirectory(atPath: symbolGraphs.targetSymbolGraphsDirectory.path).isEmpty {
-                // This target did not produce any symbol graphs. Let's check if it has a
-                // DocC catalog.
+            if target.doccCatalogPath == nil,
+               try FileManager.default.contentsOfDirectory(atPath: symbolGraphs.targetSymbolGraphsDirectory.path).isEmpty
+            {
+                // This target did not produce any symbol graphs and has no DocC catalog.
+                let message = """
+                    '\(target.name)' does not contain any documentable symbols or a \
+                    DocC catalog and will not produce documentation
+                    """
 
-                guard target.doccCatalogPath != nil else {
-                    let message = """
-                        '\(target.name)' does not contain any documentable symbols or a \
-                        DocC catalog and will not produce documentation
-                        """
-                    
-                    if swiftSourceModuleTargets.count > 1 {
-                        // We're building multiple targets, just throw a warning for this
-                        // one target that does not produce documentation.
-                        Diagnostics.warning(message)
-                        continue
-                    } else {
-                        // This is the only target being built so throw an error
-                        Diagnostics.error(message)
-                        return
-                    }
+                if swiftSourceModuleTargets.count > 1 {
+                    // We're building multiple targets, just emit a warning for this
+                    // one target that does not produce documentation.
+                    Diagnostics.warning(message)
+                } else {
+                    // This is the only target being built so emit an error
+                    Diagnostics.error(message)
                 }
+                return nil
             }
 
             // Construct the output path for the generated DocC archive
@@ -108,14 +113,22 @@ import PackagePlugin
             // arguments to pass to `docc`. ParsedArguments will merge the flags provided
             // by the user with default fallback values for required flags that were not
             // provided.
-            let doccArguments = parsedArguments.doccArguments(
+            var doccArguments = parsedArguments.doccArguments(
                 action: .convert,
                 targetKind: target.kind == .executable ? .executable : .library,
                 doccCatalogPath: target.doccCatalogPath,
                 targetName: target.name,
                 symbolGraphDirectoryPath: symbolGraphs.unifiedSymbolGraphsDirectory.path,
                 outputPath: doccArchiveOutputPath
             )
+            if isCombinedDocumentationEnabled {
+                doccArguments.append(CommandLineOption.enableExternalLinkSupport.defaultName)
+                
+                for taskDependency in task.dependencies {
+                    let dependencyArchivePath = taskDependency.target.doccArchiveOutputPath(in: context)
+                    doccArguments.append(contentsOf: [CommandLineOption.externalLinkDependency.defaultName, dependencyArchivePath])
+                }
+            }
 
             if verbose {
                 let arguments = doccArguments.joined(separator: " ")
@@ -138,15 +151,57 @@ import PackagePlugin
                 let describedOutputPath = doccArguments.outputPath ?? "unknown location"
                 print("Generated DocC archive at '\(describedOutputPath)'")
             } else {
-                Diagnostics.error("""
-                    'docc convert' invocation failed with a nonzero exit code: '\(process.terminationStatus)'
-                    """
-                )
+                Diagnostics.error("'docc convert' invocation failed with a nonzero exit code: '\(process.terminationStatus)'")
             }
+            
+            return URL(fileURLWithPath: doccArchiveOutputPath)
         }
 
-        if swiftSourceModuleTargets.count > 1 {
-            print("\nMultiple DocC archives generated at '\(context.pluginWorkDirectory.string)'")
+        let buildGraphRunner = DocumentationBuildGraphRunner(buildGraph: .init(targets: swiftSourceModuleTargets))
+        var documentationArchives = try buildGraphRunner.perform(performBuildTask)
+            .compactMap { $0 }
+        
+        if documentationArchives.count > 1 {
+            documentationArchives = documentationArchives.sorted(by: { $0.lastPathComponent < $1.lastPathComponent })
+            
+            if isCombinedDocumentationEnabled {
+                // Merge the archives into a combined archive
+                let combinedArchiveName = "Combined \(context.package.displayName) Documentation.doccarchive"
+                let combinedArchiveOutput = URL(fileURLWithPath: context.pluginWorkDirectory.appending(combinedArchiveName).string)
+                
+                var mergeCommandArguments = ["merge"]
+                mergeCommandArguments.append(contentsOf: documentationArchives.map(\.standardizedFileURL.path))
+                mergeCommandArguments.append(contentsOf: ["--output-path", combinedArchiveOutput.path])
+                
+                // Remove the combined archive if it already exists
+                try? FileManager.default.removeItem(at: combinedArchiveOutput)
+                
+                // Create a new combined archive
+                let process = try Process.run(doccExecutableURL, arguments: mergeCommandArguments)
+                process.waitUntilExit()
+                
+                // Display the combined archive before the other generated archives
+                documentationArchives.insert(combinedArchiveOutput, at: 0)
+            }
+            
+            print("""
+            Generated \(documentationArchives.count) DocC archives in '\(context.pluginWorkDirectory.string)':
+              \(documentationArchives.map(\.lastPathComponent).joined(separator: "\n  "))
+            """)
+        }
+    }
+}
+
+// We add the conformance here so that 'DocumentationBuildGraphTarget' doesn't need to know about 'SwiftSourceModuleTarget' or import 'PackagePlugin'.
+extension SwiftSourceModuleTarget: DocumentationBuildGraphTarget {
+    var dependencyIDs: [String] {
+        // List all the target dependencies in a flat list.
+        dependencies.flatMap {
+            switch $0 {
+            case .target(let target):   return [target.id]
+            case .product(let product): return product.targets.map { $0.id }
+            @unknown default:           return []
+            }
         }
     }
 }

Sources/SwiftDocCPluginUtilities/BuildGraph/DocumentationBuildGraph.swift

@@ -0,0 +1,112 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+import Foundation
+
+/// A target that can have a documentation task in the build graph
+protocol DocumentationBuildGraphTarget {
+    typealias ID = String
+    /// The unique identifier of this target
+    var id: ID { get }
+    /// The unique identifiers of this target's direct dependencies (non-transitive).
+    var dependencyIDs: [ID] { get }
+}
+
+/// A build graph of documentation tasks.
+struct DocumentationBuildGraph<Target: DocumentationBuildGraphTarget> {
+    fileprivate typealias ID = Target.ID
+    /// All the documentation tasks
+    let tasks: [Task]
+    
+    /// Creates a new documentation build graph for a series of targets with dependencies.
+    init(targets: some Sequence<Target>) {
+        // Create tasks
+        let taskLookup: [ID: Task] = targets.reduce(into: [:]) { acc, target in
+            acc[target.id] = Task(target: target)
+        }
+        // Add dependency information to each task
+        for task in taskLookup.values {
+            task.dependencies = task.target.dependencyIDs.compactMap { taskLookup[$0] }
+        }
+        
+        tasks = Array(taskLookup.values)
+    }
+    
+    /// Creates a list of dependent operations to perform the given work for each task in the build graph.
+    ///
+    /// You can add these operations to an `OperationQueue` to perform them in dependency order
+    /// (dependencies before dependents). The queue can run these operations concurrently.
+    ///
+    /// - Parameter work: The work to perform for each task in the build graph.
+    /// - Returns: A list of dependent operations that performs `work` for each documentation task task.
+    func makeOperations(performing work: @escaping (Task) -> Void) -> [Operation] {
+        var builder = OperationBuilder(work: work)
+        for task in tasks {
+            builder.buildOperationHierarchy(for: task)
+        }
+        
+        return Array(builder.operationsByID.values)
+    }
+}
+
+extension DocumentationBuildGraph {
+    /// A documentation task in the build graph
+    final class Task {
+        /// The target to build documentation for
+        let target: Target
+        /// The unique identifier of the task
+        fileprivate var id: ID { target.id }
+        /// The other documentation tasks that this task depends on.
+        fileprivate(set) var dependencies: [Task]
+        
+        init(target: Target) {
+            self.target = target
+            self.dependencies = []
+        }
+    }
+}
+
+extension DocumentationBuildGraph {
+    /// A type that builds a hierarchy of dependent operations
+    private struct OperationBuilder {
+        /// The work that each operation should perform
+        let work: (Task) -> Void
+        /// A lookup of operations by their ID
+        private(set) var operationsByID: [ID: Operation] = [:]
+        
+        /// Adds new dependent operations to the builder.
+        ///
+        /// You can access the created dependent operations using `operationsByID.values`.
+        mutating func buildOperationHierarchy(for task: Task) {
+            let operation = makeOperation(for: task)
+            for dependency in task.dependencies {
+                let hasAlreadyVisitedTask = operationsByID[dependency.id] != nil
+                
+                let dependentOperation = makeOperation(for: dependency)
+                operation.addDependency(dependentOperation)
+                
+                if !hasAlreadyVisitedTask {
+                    buildOperationHierarchy(for: dependency)
+                }
+            }
+        }
+        
+        /// Returns the existing operation for the given task or creates a new operation if the builder didn't already have an operation for this task.
+        private mutating func makeOperation(for task: Task) -> Operation {
+            if let existing = operationsByID[task.id] {
+                return existing
+            }
+            // Copy the closure and the target into a block operation object
+            let new = BlockOperation { [work, task] in
+                work(task)
+            }
+            operationsByID[task.id] = new
+            return new
+        }
+    }
+}

Sources/SwiftDocCPluginUtilities/BuildGraph/DocumentationBuildGraphRunner.swift

@@ -0,0 +1,54 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+
+import Foundation
+
+/// A type that runs tasks for each target in a build graph in dependency order.
+struct DocumentationBuildGraphRunner<Target: DocumentationBuildGraphTarget> {
+    
+    let buildGraph: DocumentationBuildGraph<Target>
+    
+    typealias Work<Result> = (DocumentationBuildGraph<Target>.Task) throws -> Result
+    
+    func perform<Result>(_ work: @escaping Work<Result>) throws -> [Result] {
+        // Create a serial queue to perform each documentation build task
+        let queue = OperationQueue()
+        queue.maxConcurrentOperationCount = 1
+        
+        // Operations can't raise errors. Instead we catch the error from 'performBuildTask(_:)'
+        // and cancel the remaining tasks.
+        let resultLock = NSLock()
+        var caughtError: Error?
+        var results: [Result] = []
+        
+        let operations = buildGraph.makeOperations { [work] task in
+            do {
+                let result = try work(task)
+                resultLock.withLock {
+                    results.append(result)
+                }
+            } catch {
+                resultLock.withLock {
+                    caughtError = error
+                    queue.cancelAllOperations()
+                }
+            }
+        }
+        
+        // Run all the documentation build tasks in dependency order (dependencies before dependents).
+        queue.addOperations(operations, waitUntilFinished: true)
+        
+        // If any of the build tasks raised an error. Re-throw that error.
+        if let caughtError {
+            throw caughtError
+        }
+        
+        return results
+    }
+}

Sources/SwiftDocCPluginUtilities/CommandLineOptions/CommandLineOption.swift

@@ -59,4 +59,15 @@ extension CommandLineOption {
     static let fallbackDefaultModuleKind = CommandLineOption(
         defaultName: "--fallback-default-module-kind"
     )
+    
+    /// A DocC flag that enables support for linking to other DocC archives and enables 
+    /// other documentation builds to link to the generated DocC archive.
+    static let enableExternalLinkSupport = CommandLineOption(
+        defaultName: "--enable-experimental-external-link-support"
+    )
+    
+    /// A DocC flag that specifies a dependency DocC archive that the current build can link to.
+    static let externalLinkDependency = CommandLineOption(
+        defaultName: "--dependency"
+    )
 }