Treat external content as pre-rendered (#806)

* Treat all externally resolved content as "pre-rendered"

rdar://78718811

* Collect external links grouped by bundle ID to avoid need to assert

* Use new cache struct for local content as well

* Remove configuration that's never used

* Remove test resolver that's never used

* Remove private types that are never used

* Remove unused argument to problem factory method

* Fix a performance regression

The cause of the regression was accidentally copying the local cache while mutating it.

* Fix a bug where external resolved references with updated paths wasn't found

* Update test to expect relative URL

* Add more documentation for ContentCache struct

* Remove reserveSymbolIDCapacity parameters that was always passed true

* Remove argument label for `value` parameter

* Rename references to allReferences to make it explicit

* Document when the resolved reference doesn't match the authored link

* More documentation for how the convert service fallback resolver works

* More documentation for how the local and external content caches are built up

* Update new test to lookup nodes by symbol ID directly in the cache

* Consistently use `---` instead of em dashes in documentation comments

* Properly test automatic curation for extended symbols

Author: d-ronnqvist

Sources/SwiftDocC/Benchmark/Metrics/ExternalTopicsHash.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -11,19 +11,18 @@
 import Foundation
 
 extension Benchmark {
-    /// A hash metric produced off the externally resolved links and symbols.
+    /// A hash metric produced off the externally resolved links.
     ///
-    /// Use this metric to verify that your code changes
-    /// did not affect external resolving.
+    /// Use this metric to verify that your code changes did not affect external link resolution.
     public class ExternalTopicsHash: BenchmarkMetric {
         public static let identifier = "external-topics-hash"
         public static let displayName = "External Topics Checksum"
 
-        /// Creates a new metric and stores the checksum of the given documentation context external topics.
-        /// - Parameter context: A documentation context.
+        /// Creates a new metric that stores the checksum of the successfully externally resolved links.
+        /// - Parameter context: A documentation context that the external links were resolved in.
         public init(context: DocumentationContext) {
             // If there are no externally resolved topics return quickly.
-            guard !context.externallyResolvedLinks.isEmpty || !context.externallyResolvedSymbols.isEmpty else {
+            guard !context.externallyResolvedLinks.isEmpty else {
                 return
             }
 
@@ -37,7 +36,6 @@ extension Benchmark {
                     return nil
                 }
             }).sorted().joined()
-                + context.externallyResolvedSymbols.map({ $0.absoluteString }).sorted().joined()
 
             result = .checksum(Checksum.md5(of: Data(sourceString.utf8)))
         }

Sources/SwiftDocC/Catalog Processing/GeneratedCurationWriter.swift

@@ -137,7 +137,7 @@ public struct GeneratedCurationWriter {
         }
 
         var contentsToWrite = [URL: String]()
-        for (usr, reference) in context.symbolIndex {
+        for (usr, reference) in context.documentationCache.referencesBySymbolID {
             // Filter out symbols that aren't in the specified sub hierarchy.
             if symbolLink != nil || depthLimit != nil {
                 guard reference == curationCrawlRoot || context.pathsTo(reference).contains(where: { path in path.suffix(depthLimit ?? .max).contains(curationCrawlRoot)}) else {

Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -155,7 +155,6 @@ public struct ConvertService: DocumentationService {
 
             // Enable support for generating documentation for standalone articles and tutorials.
             context.allowsRegisteringArticlesWithoutTechnologyRoot = true
-            context.allowsRegisteringUncuratedTutorials = true
             context.considerDocumentationExtensionsThatDoNotMatchSymbolsAsResolved = true
 
             context.configureSymbolGraph = { symbolGraph in
@@ -173,9 +172,8 @@ public struct ConvertService: DocumentationService {
                     convertRequestIdentifier: messageIdentifier
                 )
 
-                context.fallbackReferenceResolvers[request.bundleInfo.identifier] = resolver
-                context.fallbackAssetResolvers[request.bundleInfo.identifier] = resolver
-                context.externalSymbolResolver = resolver
+                context.convertServiceFallbackResolver = resolver
+                context.globalExternalSymbolResolver = resolver
             }
 
             var converter = try self.converter ?? DocumentationConverter(

Sources/SwiftDocC/DocumentationService/Convert/Fallback Link Resolution/ConvertServiceFallbackResolver.swift

@@ -0,0 +1,47 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2024 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+
+/// A resolver that attempts to resolve local references to content that wasn't included in the catalog or symbol input.
+///
+/// The ``ConvertService`` builds documentation for a single page, which doesn't have to be a top-level page. If this page's content contains references to other local
+/// symbols, pages, or assets that aren't included in the ``ConvertRequest``, this fallback resolver resolves those references to on-demand fill in the missing local content.
+///
+/// For example, when building documentation for `someFunction()` that's a member of `SomeClass` in `SomeModule`, the ``ConvertService`` can pass a
+/// "partial" symbol graph file that only contains `someFunction()` and its relationships but not `SomeClass` or any other symbols. If `someFunction()` has a local
+/// documentation link or symbol link to another symbol or page, DocC won't be able to find the page that the link refers to and will ask the fallback resolver to attempt to resolve it.
+///
+/// > Note: The ``ConvertService`` only renders the one page that it provided inputs for. Because of this, the content that this fallback resolver returns is considered
+/// "external" content, even if it represents pages that would be "local" if the full project was built together.
+protocol ConvertServiceFallbackResolver {
+    
+    // MARK: References
+    
+    /// Attempts to resolve an unresolved reference for a page that couldn't be resolved locally.
+    ///
+    /// - Parameter reference: The unresolved local reference.
+    /// - Returns: The resolved reference, or information about why the resolver failed to resolve the reference.
+    func resolve(_ reference: TopicReference) -> TopicReferenceResolutionResult
+    
+    /// Returns an external entity with the documentation content for a local resolved reference if the reference was previously resolved by this resolver.
+    ///
+    /// - Parameter reference: The local reference that this resolver may have previously resolved.
+    /// - Returns: An entity with the documentation content for the referenced page or landmark, or `nil` if the reference wasn't previously resolved by this resolver.
+    func entityIfPreviouslyResolved(with reference: ResolvedTopicReference) -> LinkResolver.ExternalEntity?
+    
+    // MARK:  Assets
+    
+    /// Attempts to resolve an asset that couldn't be resolved locally.
+    ///
+    /// - Parameter assetName: The name of the local asset to resolve.
+    /// - Returns: The local asset with the given name if found; otherwise `nil`.
+    func resolve(assetNamed assetName: String) -> DataAsset?
+}

Sources/SwiftDocC/Infrastructure/External Data/ExternalReferenceResolverServiceClient.swift renamed to Sources/SwiftDocC/DocumentationService/ExternalReferenceResolverServiceClient.swift

@@ -0,0 +1,108 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2024 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+extension DocumentationContext {
+    /// A cache for symbol and page content.
+    ///
+    /// The context uses this cache type with different values for both local content (``DocumentationContext/LocalCache``) and external content (``DocumentationContext/ExternalCache``).
+    ///
+    /// > Note:
+    /// > The cache is not thread-safe. It's safe to read from the cache concurrently but writing needs to happen with exclusive access. It is the callers responsibility to synchronize write access.
+    struct ContentCache<Value> {
+        /// The main storage of cached values.
+        private(set) var valuesByReference = [ResolvedTopicReference: Value]()
+        /// A supplementary lookup of references by their symbol ID.
+        ///
+        /// If a reference is found, ``valuesByReference``  will also have a value for that reference because ``add(_:reference:symbolID:)`` is the only place that writes to this lookup and it always adds the reference-value pair to ``valuesByReference``.
+        private(set) var referencesBySymbolID = [String: ResolvedTopicReference]()
+        
+        /// Accesses the value for a given reference.
+        /// - Parameter reference: The reference to find in the cache.
+        subscript(reference: ResolvedTopicReference) -> Value? {
+            // Avoid copying the values if possible
+            _read { yield valuesByReference[reference] }
+            _modify { yield &valuesByReference[reference] }
+        }
+        
+        /// Adds a value to the cache for a given reference _and_ symbol ID.
+        /// - Parameters:
+        ///   - value: The value to add to the cache.
+        ///   - reference: The reference associated with that value.
+        ///   - symbolID: The symbol ID associated with that value.
+        mutating func add(_ value: Value, reference: ResolvedTopicReference, symbolID: String) {
+            referencesBySymbolID[symbolID] = reference
+            valuesByReference[reference] = value
+        }
+        
+        /// Accesses the reference for a given symbol ID.
+        /// - Parameter symbolID: The symbol ID to find in the cache.
+        func reference(symbolID: String) -> ResolvedTopicReference? {
+            referencesBySymbolID[symbolID]
+        }
+        
+        /// Accesses the value for a given symbol ID.
+        /// - Parameter symbolID: The symbol ID to find in the cache.
+        subscript(symbolID: String) -> Value? {
+            // Avoid copying the values if possible
+            _read { yield referencesBySymbolID[symbolID].map { valuesByReference[$0]! } }
+        }
+        
+        /// Reserves enough space to store the specified number of values and symbol IDs.
+        ///
+        /// If you are adding a known number of values pairs to a cache, use this method to avoid multiple reallocations.
+        ///
+        /// > Note: The cache reserves the specified capacity for both values and symbol IDs.
+        ///
+        /// - Parameter minimumCapacity: The requested number of key-value pairs to store.
+        mutating func reserveCapacity(_ minimumCapacity: Int) {
+            valuesByReference.reserveCapacity(minimumCapacity)
+            // The only place that currently calls expects reserve the same capacity for both stored properties.
+            // This is because symbols are 
+            referencesBySymbolID.reserveCapacity(minimumCapacity)
+        }
+        
+        /// Returns a list of all the references in the cache.
+        var allReferences: [ResolvedTopicReference] {
+            return Array(valuesByReference.keys)
+        }
+        
+        /// Returns a list of all the references in the cache.
+        var symbolReferences: [ResolvedTopicReference] {
+            return Array(referencesBySymbolID.values)
+        }
+    }
+}
+
+// Support iterating over the cached values, checking the number of cached values, and other collection operations.
+extension DocumentationContext.ContentCache: Collection {
+    typealias Wrapped = [ResolvedTopicReference: Value]
+    typealias Index = Wrapped.Index
+    typealias Element = Wrapped.Element
+    
+    func makeIterator() -> Wrapped.Iterator {
+        valuesByReference.makeIterator()
+    }
+    
+    var startIndex: Wrapped.Index {
+        valuesByReference.startIndex
+    }
+    
+    var endIndex: Wrapped.Index {
+        valuesByReference.endIndex
+    }
+    
+    func index(after i: Wrapped.Index) -> Wrapped.Index {
+        valuesByReference.index(after: i)
+    }
+    
+    subscript(position: Wrapped.Index) -> Wrapped.Element {
+        _read { yield valuesByReference[position] }
+    }
+}

Sources/SwiftDocC/Infrastructure/ContentCache.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -371,7 +371,7 @@ extension CoverageDataEntry {
                 )
                 let total = children.count
                 let documented = children.filter {
-                    (context.nodeWithSymbolIdentifier($0.reference.description)?.semantic as? Symbol)?.abstractSection != nil
+                    (context.documentationCache[$0.reference.description]?.semantic as? Symbol)?.abstractSection != nil
                 }.count
 
                 if total == 0 {