[ASTGen/Macros] Introduce a Swift-side SourceManager into ASTGen.

Add SourceManager that can keep track of multiple source file syntax
nodes along with their external representations. The source manager can
emit diagnostics into any of those files, including tracking any
explicitly "detached" syntax nodes used for macro expansion.

Make sure we detach syntax nodes before passing them to macro
implementations, so they cannot see more of the source file than they
are permitted. We hadn't been doing this before (by accident), and
doing so motivated the introduction of the SourceManager.

Additionally, perform operator folding on macro arguments as part of
detaching them. Macro clients shouldn't have to do this, and moreover,
when clients do this, they lose the ability to easily emit diagnostics
on the now-folded nodes.

Author: DougGregor

lib/ASTGen/CMakeLists.txt

@@ -28,6 +28,7 @@ if (SWIFT_SWIFT_PARSER)
     Sources/ASTGen/Macros.swift
     Sources/ASTGen/Misc.swift
     Sources/ASTGen/SourceFile.swift
+    Sources/ASTGen/SourceManager.swift
     Sources/ASTGen/Stmts.swift
     Sources/ASTGen/Types.swift
   )

lib/ASTGen/Package.swift

@@ -26,6 +26,7 @@ let package = Package(
       dependencies: [
         .product(name: "SwiftDiagnostics", package: "swift-syntax"),
         .product(name: "SwiftSyntax", package: "swift-syntax"),
+        .product(name: "SwiftOperators", package: "swift-syntax"),
         .product(name: "SwiftParser", package: "swift-syntax"),
         .product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
       ],

lib/ASTGen/Sources/ASTGen/Diagnostics.swift

@@ -123,6 +123,124 @@ func emitDiagnostic(
   }
 }
 
+extension SourceManager {
+  private func diagnoseSingle(
+    message: String,
+    severity: DiagnosticSeverity,
+    node: some SyntaxProtocol,
+    position: AbsolutePosition,
+    highlights: [Syntax] = [],
+    fixItChanges: [FixIt.Change] = []
+  ) {
+    // Map severity
+    let bridgedSeverity: BridgedDiagnosticSeverity
+    switch severity {
+      case .error: bridgedSeverity = .error
+      case .note: bridgedSeverity = .note
+      case .warning: bridgedSeverity = .warning
+    }
+
+    // Emit the diagnostic
+    var mutableMessage = message
+    let diag = mutableMessage.withUTF8 { messageBuffer in
+      SwiftDiagnostic_create(
+        cxxDiagnosticEngine, bridgedSeverity,
+        cxxSourceLocation(for: node, at: position),
+        messageBuffer.baseAddress, messageBuffer.count
+      )
+    }
+
+    // Emit highlights
+    for highlight in highlights {
+      SwiftDiagnostic_highlight(
+        diag,
+        cxxSourceLocation(for: highlight),
+        cxxSourceLocation(for: highlight, at: highlight.endPosition)
+      )
+    }
+
+    // Emit changes for a Fix-It.
+    for change in fixItChanges {
+      let replaceStartLoc: CxxSourceLoc?
+      let replaceEndLoc: CxxSourceLoc?
+      var newText: String
+
+      switch change {
+      case .replace(let oldNode, let newNode):
+        replaceStartLoc = cxxSourceLocation(for: oldNode)
+        replaceEndLoc = cxxSourceLocation(
+          for: oldNode,
+          at: oldNode.endPosition
+        )
+        newText = newNode.description
+
+      case .replaceLeadingTrivia(let oldToken, let newTrivia):
+        replaceStartLoc = cxxSourceLocation(for: oldToken)
+        replaceEndLoc = cxxSourceLocation(
+          for: oldToken,
+          at: oldToken.positionAfterSkippingLeadingTrivia
+        )
+        newText = newTrivia.description
+
+      case .replaceTrailingTrivia(let oldToken, let newTrivia):
+        replaceStartLoc = cxxSourceLocation(
+          for: oldToken,
+          at: oldToken.endPositionBeforeTrailingTrivia)
+        replaceEndLoc = cxxSourceLocation(
+          for: oldToken,
+          at: oldToken.endPosition
+        )
+        newText = newTrivia.description
+      }
+
+      newText.withUTF8 { textBuffer in
+        SwiftDiagnostic_fixItReplace(
+          diag, replaceStartLoc, replaceEndLoc,
+          textBuffer.baseAddress, textBuffer.count
+        )
+      }
+    }
+
+    SwiftDiagnostic_finish(diag);
+  }
+
+  /// Emit a diagnostic via the C++ diagnostic engine.
+  func diagnose(
+    diagnostic: Diagnostic,
+    messageSuffix: String? = nil
+  ) {
+    // Emit the main diagnostic.
+    diagnoseSingle(
+      message: diagnostic.diagMessage.message + (messageSuffix ?? ""),
+      severity: diagnostic.diagMessage.severity,
+      node: diagnostic.node,
+      position: diagnostic.position,
+      highlights: diagnostic.highlights
+    )
+
+    // Emit Fix-Its.
+    for fixIt in diagnostic.fixIts {
+      diagnoseSingle(
+          message: fixIt.message.message,
+          severity: .note,
+          node: diagnostic.node,
+          position: diagnostic.position,
+          fixItChanges: fixIt.changes.changes
+      )
+    }
+
+    // Emit any notes as follow-ons.
+    for note in diagnostic.notes {
+      diagnoseSingle(
+        message: note.message,
+        severity: .note,
+        node: note.node,
+        position: note.position
+      )
+    }
+  }
+}
+
 /// A set of queued diagnostics created by the C++ compiler and rendered
 /// via the swift-syntax renderer.
 struct QueuedDiagnostics {

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -1,4 +1,5 @@
 import SwiftDiagnostics
+import SwiftOperators
 import SwiftParser
 import SwiftSyntax
 import SwiftSyntaxMacros
@@ -94,18 +95,6 @@ func allocateUTF8String(
   }
 }
 
-extension String {
-  /// Drop everything up to and including the last '/' from the string.
-  fileprivate func withoutPath() -> String {
-    // Only keep everything after the last slash.
-    if let lastSlash = lastIndex(of: "/") {
-      return String(self[index(after: lastSlash)...])
-    }
-
-    return self
-  }
-}
-
 /// Diagnostic message used for thrown errors.
 fileprivate struct ThrownErrorDiagnostic: DiagnosticMessage {
   let message: String
@@ -151,14 +140,11 @@ func evaluateMacro(
     return -1
   }
 
-  let context = BasicMacroExpansionContext(
-    sourceFiles: [
-      sourceFilePtr.pointee.syntax : .init(
-        moduleName: sourceFilePtr.pointee.moduleName,
-        fullFilePath: sourceFilePtr.pointee.fileName
-      )
-    ]
-  )
+  // Create a source manager. This should probably persist and be given to us.
+  let sourceManager = SourceManager(cxxDiagnosticEngine: diagEnginePtr)
+  sourceManager.insert(sourceFilePtr)
+
+  let context = sourceManager.createMacroExpansionContext()
 
   guard let parentSyntax = token.parent else {
     print("not on a macro expansion node: \(token.recursiveDescription)")
@@ -183,7 +169,10 @@ func evaluateMacro(
       macroName = parentExpansion.macro.text
       evaluatedSyntax = Syntax(
         try exprMacro.expansion(
-          of: context.detach(parentExpansion),
+          of: sourceManager.detach(
+            parentExpansion, in: context,
+            foldingWith: OperatorTable.standardOperators
+          ),
           in: context
         )
       )
@@ -195,7 +184,7 @@ func evaluateMacro(
         return -1
       }
       let decls = try declMacro.expansion(
-        of: context.detach(parentExpansion),
+        of: sourceManager.detach(parentExpansion, in: context),
         in: context
       )
       macroName = parentExpansion.macro.text
@@ -219,9 +208,7 @@ func evaluateMacro(
 
   // Emit diagnostics accumulated in the context.
   for diag in context.diagnostics {
-    emitDiagnostic(
-      diagEnginePtr: diagEnginePtr,
-      sourceFileBuffer: .init(mutating: sourceFilePtr.pointee.buffer),
+    sourceManager.diagnose(
       diagnostic: diag,
       messageSuffix: " (from macro '\(macroName)')"
     )
@@ -330,19 +317,15 @@ func expandAttachedMacro(
     to: ExportedSourceFile.self, capacity: 1
   )
 
-  // Record the source file(s).
-  var sourceFiles: [SourceFileSyntax : BasicMacroExpansionContext.KnownSourceFile] = [:]
-  sourceFiles[attributeSourceFile.pointee.syntax] = .init(
-    moduleName: attributeSourceFile.pointee.moduleName,
-    fullFilePath: attributeSourceFile.pointee.fileName
-  )
-  sourceFiles[declarationSourceFilePtr.pointee.syntax] = .init(
-    moduleName: declarationSourceFilePtr.pointee.moduleName,
-    fullFilePath: declarationSourceFilePtr.pointee.fileName
-  )
+  // Create a source manager covering the files we know about.
+  let sourceManager = SourceManager(cxxDiagnosticEngine: diagEnginePtr)
+  sourceManager.insert(attributeSourceFile)
+  sourceManager.insert(declarationSourceFilePtr)
 
-  let context = BasicMacroExpansionContext(sourceFiles: sourceFiles)
+  // Create an expansion context
+  let context = sourceManager.createMacroExpansionContext()
 
+  let macroName = customAttrNode.attributeName.description
   var evaluatedSyntaxStr: String
   do {
     switch (macro, macroRole) {
@@ -417,8 +400,13 @@ func expandAttachedMacro(
     return 1
   }
 
-  // FIXME: Emit diagnostics, but how do we figure out which source file to
-  // use?
+  // Emit diagnostics accumulated in the context.
+  for diag in context.diagnostics {
+    sourceManager.diagnose(
+      diagnostic: diag,
+      messageSuffix: " (from macro '\(macroName)')"
+    )
+  }
 
   // Form the result buffer for our caller.
   evaluatedSyntaxStr.withUTF8 { utf8 in