Minimal viable merge command (#821)

* Move assets into subdirectory to avoid collisions when merging archives

* Add basic merge command to combine documentation archives

rdar://114730477

* Publish docs for SwiftDocC and SwiftDocCUtilities as a combined archive

* Pass flags to generate source links for published framework docs

* Raise an error if the merge output directory already exists

Raise an error if the archives to merge has overlapping data

Use a temporary directory while merging the archives

* Consider all (future) subdirectories in the data directory

* Fix to docc merge command: Copy data/documentation and data/tutorials,
creating intermediate directories as necessary

---------

Co-authored-by: Pat Shaughnessy <[email protected]>

Author: d-ronnqvist

Sources/SwiftDocC/Converter/RenderNode+Coding.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -27,6 +27,9 @@ extension CodingUserInfoKey {
     static let variantOverrides = CodingUserInfoKey(rawValue: "variantOverrides")!
 
     static let baseEncodingPath = CodingUserInfoKey(rawValue: "baseEncodingPath")!
+    
+    /// A user info key to indicate a base path for local asset URLs.
+    static let assetPrefixComponent = CodingUserInfoKey(rawValue: "assetPrefixComponent")!
 }
 
 extension Encoder {
@@ -45,12 +48,12 @@ extension Encoder {
     ///
     /// These references will then be encoded at a later stage by `TopicRenderReferenceEncoder`.
     var skipsEncodingReferences: Bool {
-        guard let userInfoValue = userInfo[.skipsEncodingReferences] as? Bool else {
-            // The value doesn't exist so we should encode reference. Return false.
-            return false
-        }
-        
-        return userInfoValue
+        userInfo[.skipsEncodingReferences] as? Bool ?? false
+    }
+    
+    /// A base path to use when creating destination URLs for local assets (images, videos, downloads, etc.)
+    var assetPrefixComponent: String? {
+        userInfo[.assetPrefixComponent] as? String
     }
 }
 
@@ -81,12 +84,7 @@ extension JSONEncoder {
     /// These references will then be encoded at a later stage by `TopicRenderReferenceEncoder`.
     var skipsEncodingReferences: Bool {
         get {
-            guard let userInfoValue = userInfo[.skipsEncodingReferences] as? Bool else {
-                // The value doesn't exist so we should encode reference. Return false.
-                return false
-            }
-            
-            return userInfoValue
+            userInfo[.skipsEncodingReferences] as? Bool ?? false
         }
         set {
             userInfo[.skipsEncodingReferences] = newValue
@@ -104,13 +102,14 @@ public enum RenderJSONEncoder {
     /// process which should not be shared in other encoding units. Instead, call this API to create a new encoder for each render node you want to encode.
     ///
     /// - Parameters:
-    ///     - prettyPrint: If `true`, the encoder formats its output to make it easy to read; if `false`, the output is compact.
-    ///     - emitVariantOverrides: Whether the encoder should emit the top-level ``RenderNode/variantOverrides`` property that holds language-
-    ///     specific documentation data.
+    ///   - prettyPrint: If `true`, the encoder formats its output to make it easy to read; if `false`, the output is compact.
+    ///   - emitVariantOverrides: Whether the encoder should emit the top-level ``RenderNode/variantOverrides`` property that holds language-specific documentation data.
+    ///   - assetPrefixComponent: A path component to include in destination URLs for local assets (images, videos, downloads, etc.)
     /// - Returns: The new JSON encoder.
     public static func makeEncoder(
         prettyPrint: Bool = shouldPrettyPrintOutputJSON,
-        emitVariantOverrides: Bool = true
+        emitVariantOverrides: Bool = true,
+        assetPrefixComponent: String? = nil
     ) -> JSONEncoder {
         let encoder = JSONEncoder()
 
@@ -125,6 +124,9 @@ public enum RenderJSONEncoder {
         if emitVariantOverrides {
             encoder.userInfo[.variantOverrides] = VariantOverrides()
         }
+        if let bundleIdentifier = assetPrefixComponent {
+            encoder.userInfo[.assetPrefixComponent] = bundleIdentifier
+        }
 
         return encoder
     }

Sources/SwiftDocC/Indexing/RenderIndexJSON/RenderIndex.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2022 Apple Inc. and the Swift project authors
+ Copyright (c) 2022-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -23,45 +23,74 @@ import SymbolKit
 /// `Sources/SwiftDocC/SwiftDocC.docc/Resources/RenderIndex.spec.json`.
 public struct RenderIndex: Codable, Equatable {
     /// The current schema version of the Index JSON spec.
-    public static let currentSchemaVersion = SemanticVersion(major: 0, minor: 1, patch: 1)
+    public static let currentSchemaVersion = SemanticVersion(major: 0, minor: 1, patch: 2)
 
     /// The version of the RenderIndex spec that was followed when creating this index.
     public let schemaVersion: SemanticVersion
 
     /// A mapping of interface languages to the index nodes they contain.
-    public let interfaceLanguages: [String: [Node]]
+    public private(set) var interfaceLanguages: [String: [Node]]
 
     /// The values of the image references used in the documentation index.
     public private(set) var references: [String: ImageReference]
 
+    /// The unique identifiers of the archives that are included in the documentation index.
+    public private(set) var includedArchiveIdentifiers: [String]
+    
     enum CodingKeys: CodingKey {
         case schemaVersion
         case interfaceLanguages
         case references
+        case includedArchiveIdentifiers
     }
 
     /// Creates a new render index with the given interface language to node mapping.
     public init(
         interfaceLanguages: [String: [Node]],
-        references: [String: ImageReference] = [:]
+        references: [String: ImageReference] = [:],
+        includedArchiveIdentifiers: [String]
     ) {
         self.schemaVersion = Self.currentSchemaVersion
         self.interfaceLanguages = interfaceLanguages
         self.references = references
+        self.includedArchiveIdentifiers = includedArchiveIdentifiers
     }
 
     public func encode(to encoder: Encoder) throws {
         var container = encoder.container(keyedBy: CodingKeys.self)
         try container.encode(self.schemaVersion, forKey: .schemaVersion)
         try container.encode(self.interfaceLanguages, forKey: .interfaceLanguages)
         try container.encodeIfNotEmpty(self.references, forKey: .references)
+        try container.encodeIfNotEmpty(self.includedArchiveIdentifiers, forKey: .includedArchiveIdentifiers)
     }
 
     public init(from decoder: Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         self.schemaVersion = try container.decode(SemanticVersion.self, forKey: .schemaVersion)
         self.interfaceLanguages = try container.decode([String : [RenderIndex.Node]].self, forKey: .interfaceLanguages)
         self.references = try container.decodeIfPresent([String : ImageReference].self, forKey: .references) ?? [:]
+        self.includedArchiveIdentifiers = try container.decodeIfPresent([String].self.self, forKey: .includedArchiveIdentifiers) ?? []
+    }
+    
+    public mutating func merge(_ other: RenderIndex) throws {
+        for (languageID, nodes) in other.interfaceLanguages {
+            interfaceLanguages[languageID, default: []].append(contentsOf: nodes)
+        }
+        
+        try references.merge(other.references) { _, new in throw MergeError.referenceCollision(new.identifier.identifier) }
+        
+        includedArchiveIdentifiers.append(contentsOf: other.includedArchiveIdentifiers)
+    }
+    
+    enum MergeError: DescribedError {
+        case referenceCollision(String)
+        
+        var errorDescription: String {
+            switch self {
+            case .referenceCollision(let reference):
+                return "Collision merging image references. Reference \(reference.singleQuoted) exists in more than one input archive."
+            }
+        }
     }
 }
 
@@ -254,7 +283,8 @@ extension RenderIndex {
                 },
                 uniquingKeysWith: +
             ),
-            references: builder.iconReferences
+            references: builder.iconReferences,
+            includedArchiveIdentifiers: [builder.bundleIdentifier]
         )
     }
 }

Sources/SwiftDocC/Model/Rendering/References/ImageReference.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -74,7 +74,7 @@ public struct ImageReference: MediaReference, URLReference, Equatable {
         var result = [VariantProxy]()
         // sort assets by URL path for deterministic sorting of images
         asset.variants.sorted(by: \.value.path).forEach { (key, value) in
-            let url = value.isAbsoluteWebURL ? value : destinationURL(for: value.lastPathComponent)
+            let url = value.isAbsoluteWebURL ? value : destinationURL(for: value.lastPathComponent, prefixComponent: encoder.assetPrefixComponent)
             result.append(VariantProxy(url: url, traits: key, svgID: asset.metadata[value]?.svgID))
         }
         try container.encode(result, forKey: .variants)

Sources/SwiftDocC/Model/Rendering/References/RenderReference.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -69,10 +69,17 @@ extension URLReference {
     ///
     /// The converter that writes the built documentation to the file system is responsible for copying the referenced file to this destination.
     /// 
-    /// - Parameter path: The path of the file.
+    /// - Parameters:
+    ///   - path: The path of the file.
+    ///   - prefixComponent: An optional path component to add before the path of the file.
     /// - Returns: The destination URL for the given file path.
-    func destinationURL(for path: String) -> URL {
-        return Self.baseURL.appendingPathComponent(path, isDirectory: false)
+    func destinationURL(for path: String, prefixComponent: String?) -> URL {
+        var url = Self.baseURL
+        if let bundleName = prefixComponent {
+            url.appendPathComponent(bundleName, isDirectory: true)
+        }
+        url.appendPathComponent(path, isDirectory: false)
+        return url
     }
 }
 

Sources/SwiftDocC/Model/Rendering/References/VideoReference.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -81,7 +81,7 @@ public struct VideoReference: MediaReference, URLReference, Equatable {
         // convert the data asset to a serializable object
         var result = [VariantProxy]()
         asset.variants.sorted(by: \.value.path).forEach { (key, value) in
-            let url = value.isAbsoluteWebURL ? value : destinationURL(for: value.lastPathComponent)
+            let url = value.isAbsoluteWebURL ? value : destinationURL(for: value.lastPathComponent, prefixComponent: encoder.assetPrefixComponent)
             result.append(VariantProxy(url: url, traits: key))
         }
         try container.encode(result, forKey: .variants)

Sources/SwiftDocC/Model/Rendering/Tutorial/References/DownloadReference.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -86,7 +86,7 @@ public struct DownloadReference: RenderReference, URLReference, Equatable {
 
         // Render URL
         if !encodeUrlVerbatim {
-            try container.encode(renderURL(for: url), forKey: .url)
+            try container.encode(renderURL(for: url, prefixComponent: encoder.assetPrefixComponent), forKey: .url)
         } else {
             try container.encode(url, forKey: .url)
         }
@@ -100,8 +100,8 @@ public struct DownloadReference: RenderReference, URLReference, Equatable {
 }
 
 extension DownloadReference {
-    private func renderURL(for url: URL) -> URL {
-        url.isAbsoluteWebURL ? url : destinationURL(for: url.lastPathComponent)
+    private func renderURL(for url: URL, prefixComponent: String?) -> URL {
+        url.isAbsoluteWebURL ? url : destinationURL(for: url.lastPathComponent, prefixComponent: prefixComponent)
     }
 }
 

Sources/SwiftDocC/SwiftDocC.docc/Resources/RenderIndex.spec.json

@@ -2,7 +2,7 @@
     "openapi": "3.0.0",
     "info": {
         "description": "Specification of the Swift-DocC Index.json file.",
-        "version": "0.1.0",
+        "version": "0.1.2",
         "title": "RenderIndex"
     },
     "paths": {},
@@ -33,6 +33,12 @@
                             "$ref": "#/components/schemas/ImageRenderReference"
                         }
                     },
+                    "includedArchiveIdentifiers": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
                 }
             },
             "Node": {

Sources/SwiftDocC/Utility/FileManagerProtocol.swift

@@ -51,6 +51,9 @@ public protocol FileManagerProtocol {
     func contentsOfDirectory(atPath path: String) throws -> [String]
     func contentsOfDirectory(at url: URL, includingPropertiesForKeys keys: [URLResourceKey]?, options mask: FileManager.DirectoryEnumerationOptions) throws -> [URL]
 
+    /// The temporary directory for the current user.
+    var temporaryDirectory: URL { get }
+    
     /// Creates a file with the specified `contents` at the specified location.
     ///
     /// - Parameters: