Add Static Hosting Support (#44)

Relevant forums discussion: https://forums.swift.org/t/support-hosting-docc-archives-in-static-hosting-environments/53572.

Resolves rdar://70800606.

Co-authored-by: Ethan Kusters <[email protected]>

Author: macdevnet

Sources/SwiftDocC/Infrastructure/NodeURLGenerator.swift

@@ -41,6 +41,7 @@ public struct NodeURLGenerator {
     public enum Path {
         public static let tutorialsFolderName = "tutorials"
         public static let documentationFolderName = "documentation"
+        public static let dataFolderName = "data"
 
         public static let tutorialsFolder = "/\(tutorialsFolderName)"
         public static let documentationFolder = "/\(documentationFolderName)"

Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift

@@ -38,7 +38,11 @@ public struct ConvertAction: Action, RecreatingContext {
     let documentationCoverageOptions: DocumentationCoverageOptions
     let diagnosticLevel: DiagnosticSeverity
     let diagnosticEngine: DiagnosticEngine
-
+    
+    let transformForStaticHosting: Bool
+    let hostingBasePath: String?
+    
+    
     private(set) var context: DocumentationContext {
         didSet {
             // current platforms?
@@ -88,7 +92,10 @@ public struct ConvertAction: Action, RecreatingContext {
         diagnosticEngine: DiagnosticEngine? = nil,
         emitFixits: Bool = false,
         inheritDocs: Bool = false,
-        experimentalEnableCustomTemplates: Bool = false) throws
+        experimentalEnableCustomTemplates: Bool = false,
+        transformForStaticHosting: Bool = false,
+        hostingBasePath: String? = nil
+    ) throws
     {
         self.rootURL = documentationBundleURL
         self.outOfProcessResolver = outOfProcessResolver
@@ -101,7 +108,9 @@ public struct ConvertAction: Action, RecreatingContext {
         self.injectedDataProvider = dataProvider
         self.fileManager = fileManager
         self.documentationCoverageOptions = documentationCoverageOptions
-
+        self.transformForStaticHosting = transformForStaticHosting
+        self.hostingBasePath = hostingBasePath
+        
         let filterLevel: DiagnosticSeverity
         if analyze {
             filterLevel = .information
@@ -189,7 +198,9 @@ public struct ConvertAction: Action, RecreatingContext {
         diagnosticEngine: DiagnosticEngine? = nil,
         emitFixits: Bool = false,
         inheritDocs: Bool = false,
-        experimentalEnableCustomTemplates: Bool = false
+        experimentalEnableCustomTemplates: Bool = false,
+        transformForStaticHosting: Bool,
+        hostingBasePath: String?
     ) throws {
         // Note: This public initializer exists separately from the above internal one
         // because the FileManagerProtocol type we use to enable mocking in tests
@@ -217,7 +228,9 @@ public struct ConvertAction: Action, RecreatingContext {
             diagnosticEngine: diagnosticEngine,
             emitFixits: emitFixits,
             inheritDocs: inheritDocs,
-            experimentalEnableCustomTemplates: experimentalEnableCustomTemplates
+            experimentalEnableCustomTemplates: experimentalEnableCustomTemplates,
+            transformForStaticHosting: transformForStaticHosting,
+            hostingBasePath: hostingBasePath
         )
     }
 
@@ -240,7 +253,7 @@ public struct ConvertAction: Action, RecreatingContext {
     mutating func cancel() throws {
         /// If the action is not running, there is nothing to cancel
         guard isPerforming.sync({ $0 }) == true else { return }
-
+        
         /// If the action is already cancelled throw `cancelPending`.
         if isCancelled.sync({ $0 }) == true {
             throw Error.cancelPending
@@ -278,6 +291,28 @@ public struct ConvertAction: Action, RecreatingContext {
         let temporaryFolder = try createTempFolder(
             with: htmlTemplateDirectory)
 
+        var indexHTMLData: Data?
+
+        // The `template-index.html` is a duplicate version of `index.html` with extra template
+        // tokens that allow for customizing the base-path.
+        // If a base bath is provided we will transform the template using the base path
+        // to produce a replacement index.html file.
+        // After any required transforming has been done the template file will be removed.
+        let templateURL: URL = temporaryFolder.appendingPathComponent(HTMLTemplate.templateFileName.rawValue)
+        if fileManager.fileExists(atPath: templateURL.path) {
+            // If the `transformForStaticHosting` is not set but there is a `hostingBasePath`
+            // then transform the index template
+            if !transformForStaticHosting,
+               let hostingBasePath = hostingBasePath,
+               !hostingBasePath.isEmpty  {
+                indexHTMLData = try StaticHostableTransformer.transformHTMLTemplate(htmlTemplate: temporaryFolder, hostingBasePath: hostingBasePath)
+                let indexURL = temporaryFolder.appendingPathComponent(HTMLTemplate.indexFileName.rawValue)
+                try indexHTMLData!.write(to: indexURL)
+            }
+            
+            try fileManager.removeItem(at: templateURL)
+        }
+        
         defer {
             try? fileManager.removeItem(at: temporaryFolder)
         }
@@ -330,15 +365,26 @@ public struct ConvertAction: Action, RecreatingContext {
             allProblems.append(contentsOf: indexerProblems)
         }
 
+        // Process Static Hosting as needed.
+        if transformForStaticHosting, let templateDirectory = htmlTemplateDirectory {
+            if indexHTMLData == nil {
+                indexHTMLData = try StaticHostableTransformer.transformHTMLTemplate(htmlTemplate: templateDirectory, hostingBasePath: hostingBasePath)
+            }
+            
+            let dataProvider = try LocalFileSystemDataProvider(rootURL: temporaryFolder.appendingPathComponent(NodeURLGenerator.Path.dataFolderName))
+            let transformer = StaticHostableTransformer(dataProvider: dataProvider, fileManager: fileManager, outputURL: temporaryFolder, indexHTMLData: indexHTMLData!)
+            try transformer.transform()
+        }
+        
         // We should generally only replace the current build output if we didn't encounter errors
         // during conversion. However, if the `emitDigest` flag is true,
         // we should replace the current output with our digest of problems.
         if !allProblems.containsErrors || emitDigest {
             try moveOutput(from: temporaryFolder, to: targetDirectory)
         }
-                
+
         // Log the output size.
-        benchmark(add: Benchmark.OutputSize(dataURL: targetDirectory.appendingPathComponent("data")))
+        benchmark(add: Benchmark.OutputSize(dataURL: targetDirectory.appendingPathComponent(NodeURLGenerator.Path.dataFolderName)))
 
         if Benchmark.main.isEnabled {
             // Write the benchmark files directly in the target directory.
@@ -363,6 +409,7 @@ public struct ConvertAction: Action, RecreatingContext {
     }
 
     func createTempFolder(with templateURL: URL?) throws -> URL {
+        
         let targetURL = URL(fileURLWithPath: NSTemporaryDirectory())
             .appendingPathComponent(ProcessInfo.processInfo.globallyUniqueString)
 

Sources/SwiftDocCUtilities/Action/Actions/TransformForStaticHostingAction.swift

@@ -0,0 +1,114 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+import SwiftDocC
+
+/// An action that emits a static hostable website from a DocC Archive.
+struct TransformForStaticHostingAction: Action {
+    
+    let rootURL: URL
+    let outputURL: URL
+    let hostingBasePath: String?
+    let outputIsExternal: Bool
+    let htmlTemplateDirectory: URL
+
+    let fileManager: FileManagerProtocol
+    
+    var diagnosticEngine: DiagnosticEngine
+    
+    /// Initializes the action with the given validated options, creates or uses the given action workspace & context.
+    init(documentationBundleURL: URL,
+         outputURL:URL?,
+         hostingBasePath: String?,
+         htmlTemplateDirectory: URL,
+         fileManager: FileManagerProtocol = FileManager.default,
+         diagnosticEngine: DiagnosticEngine = .init()) throws
+    {
+        // Initialize the action context.
+        self.rootURL = documentationBundleURL
+        self.outputURL = outputURL ?? documentationBundleURL
+        self.outputIsExternal = outputURL != nil
+        self.hostingBasePath = hostingBasePath
+        self.htmlTemplateDirectory = htmlTemplateDirectory
+        self.fileManager = fileManager
+        self.diagnosticEngine = diagnosticEngine
+        self.diagnosticEngine.add(DiagnosticConsoleWriter(formattingOptions: []))
+    }
+    
+    /// Converts each eligible file from the source archive and
+    /// saves the results in the given output folder.
+    mutating func perform(logHandle: LogHandle) throws -> ActionResult {
+        try emit()
+        return ActionResult(didEncounterError: false, outputs: [outputURL])
+    }
+    
+    mutating private func emit() throws  {
+        
+
+        // If the emit is to create the static hostable content outside of the source archive
+        // then the output folder needs to be set up and the archive data copied
+        // to the new folder.
+        if outputIsExternal {
+            
+            try setupOutputDirectory(outputURL: outputURL)
+
+            // Copy the appropriate folders from the archive.
+            // We will copy individual items from the folder rather then just copy the folder
+            // as we want to preserve anything intentionally left in the output URL by `setupOutputDirectory`
+            for sourceItem in try fileManager.contentsOfDirectory(at: rootURL, includingPropertiesForKeys: [], options:[.skipsHiddenFiles]) {
+                let targetItem = outputURL.appendingPathComponent(sourceItem.lastPathComponent)
+                try fileManager.copyItem(at: sourceItem, to: targetItem)
+            }
+        }
+
+        // Copy the HTML template to the output folder.
+        var excludedFiles = [HTMLTemplate.templateFileName.rawValue]
+
+        if outputIsExternal {
+            excludedFiles.append(HTMLTemplate.indexFileName.rawValue)
+        }
+
+        for content in try fileManager.contentsOfDirectory(atPath: htmlTemplateDirectory.path) {
+
+            guard !excludedFiles.contains(content) else { continue }
+
+            let source = htmlTemplateDirectory.appendingPathComponent(content)
+            let target = outputURL.appendingPathComponent(content)
+            if fileManager.fileExists(atPath: target.path){
+                try fileManager.removeItem(at: target)
+            }
+            try fileManager.copyItem(at: source, to: target)
+        }
+        
+        // Transform the indexHTML if needed.
+        let indexHTMLData = try StaticHostableTransformer.transformHTMLTemplate(htmlTemplate: htmlTemplateDirectory, hostingBasePath: hostingBasePath)
+
+        // Create a StaticHostableTransformer targeted at the archive data folder
+        let dataProvider = try LocalFileSystemDataProvider(rootURL: rootURL.appendingPathComponent(NodeURLGenerator.Path.dataFolderName))
+        let transformer = StaticHostableTransformer(dataProvider: dataProvider, fileManager: fileManager, outputURL: outputURL, indexHTMLData: indexHTMLData)
+        try transformer.transform()
+        
+    }
+    
+    /// Create output directory or empty its contents if it already exists.
+    private func setupOutputDirectory(outputURL: URL) throws {
+        
+        var isDirectory: ObjCBool = false
+        if fileManager.fileExists(atPath: outputURL.path, isDirectory: &isDirectory), isDirectory.boolValue {
+            let contents = try fileManager.contentsOfDirectory(at: outputURL, includingPropertiesForKeys: [], options: [.skipsHiddenFiles])
+            for content in contents {
+                try fileManager.removeItem(at: content)
+            }
+        } else {
+            try fileManager.createDirectory(at: outputURL, withIntermediateDirectories: false, attributes: [:])
+        }
+    }
+}

Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift

@@ -79,7 +79,9 @@ extension ConvertAction {
             diagnosticLevel: convert.diagnosticLevel,
             emitFixits: convert.emitFixits,
             inheritDocs: convert.enableInheritedDocs,
-            experimentalEnableCustomTemplates: convert.experimentalEnableCustomTemplates
+            experimentalEnableCustomTemplates: convert.experimentalEnableCustomTemplates,
+            transformForStaticHosting: convert.transformForStaticHosting,
+            hostingBasePath: convert.hostingBasePath
         )
     }
 }

Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/TransformForStaticHostingAction+CommandInitialization.swift

@@ -0,0 +1,34 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+import ArgumentParser
+
+
+extension TransformForStaticHostingAction {
+    /// Initializes ``TransformForStaticHostingAction`` from the options in the ``TransformForStaticHosting`` command.
+    /// - Parameters:
+    ///   - cmd: The emit command this `TransformForStaticHostingAction` will be based on.
+    init(fromCommand cmd: Docc.ProcessArchive.TransformForStaticHosting, withFallbackTemplate fallbackTemplateURL: URL? = nil) throws {
+        // Initialize the `TransformForStaticHostingAction` from the options provided by the `EmitStaticHostable` command
+        
+        guard let htmlTemplateFolder = cmd.templateOption.templateURL ?? fallbackTemplateURL else {
+            throw TemplateOption.missingHTMLTemplateError(
+                path: cmd.templateOption.defaultTemplateURL.path
+            )
+        }
+        
+        try self.init(
+            documentationBundleURL: cmd.documentationArchive.urlOrFallback,
+            outputURL: cmd.outputURL,
+            hostingBasePath: cmd.hostingBasePath,
+            htmlTemplateDirectory: htmlTemplateFolder )
+    }
+}

Sources/SwiftDocCUtilities/ArgumentParsing/Options/DocumentationArchiveOption.swift

@@ -12,21 +12,45 @@ import ArgumentParser
 import Foundation
 
 /// Resolves and validates a URL value that provides the path to a documentation archive.
-///
-/// This option is used by the ``Docc/Index`` subcommand.
-public struct DocumentationArchiveOption: DirectoryPathOption {
+public struct DocCArchiveOption: DirectoryPathOption {
 
-    public init() {}
+    public init(){}
 
     /// The name of the command line argument used to specify a source archive path.
     static let argumentValueName = "source-archive-path"
+    static let expectedContent: Set<String> = ["data"]
 
-    /// The path to an archive to be indexed by DocC.
+    /// The path to an archive to be used by DocC.
     @Argument(
         help: ArgumentHelp(
-            "Path to a documentation archive data directory of JSON files.",
-            discussion: "The '.doccarchive' bundle docc will index.",
+            "Path to the DocC Archive ('.doccarchive') that should be processed.",
             valueName: argumentValueName),
         transform: URL.init(fileURLWithPath:))
     public var url: URL?
+
+    public mutating func validate() throws {
+
+        // Validate that the URL represents a directory
+        guard urlOrFallback.hasDirectoryPath else {
+            throw ValidationError("'\(urlOrFallback.path)' is not a valid DocC Archive. Expected a directory but a path to a file was provided")
+        }
+        
+        var archiveContents: [String]
+        do {
+            archiveContents = try FileManager.default.contentsOfDirectory(atPath: urlOrFallback.path)
+        } catch {
+            throw ValidationError("'\(urlOrFallback.path)' is not a valid DocC Archive: \(error)")
+        }
+        
+        let missingContents = Array(Set(DocCArchiveOption.expectedContent).subtracting(archiveContents))
+        guard missingContents.isEmpty else {
+            throw ValidationError(
+                """
+                '\(urlOrFallback.path)' is not a valid DocC Archive.
+                Expected a 'data' directory at the root of the archive.
+                """
+            )
+        }
+        
+    }
 }