[SourceKit] Include generated macro buffers in diagnostic responses

Introduce a new key `generated_buffers`, which
stores an array of generated buffers. These
include the buffer text, as well as its original
location and any parent buffers.

While here, also fix rdar://107281079 such that
only apply the filename fallback logic to the
pretty-printed Decl case. We ought to remove this
fallback once the editor can handle it though.

rdar://107281079
rdar://107952288

Author: hamishknight

lib/Frontend/SerializedDiagnosticConsumer.cpp

@@ -231,6 +231,8 @@ unsigned SerializedDiagnosticConsumer::getEmitFile(
   // NOTE: Using Filename.data() here relies on SourceMgr using
   // const char* as buffer identifiers.  This is fast, but may
   // be brittle.  We can always switch over to using a StringMap.
+  // Note that the logic in EditorDiagConsumer::getBufferInfo
+  // will also need changing.
   unsigned &existingEntry = State->Files[Filename.data()];
   if (existingEntry)
     return existingEntry;

test/Macros/Inputs/syntax_macro_definitions.swift

@@ -897,6 +897,15 @@ public struct InvalidMacro: PeerMacro, DeclarationMacro {
   }
 }
 
+public struct CoerceToIntMacro: ExpressionMacro {
+  public static func expansion(
+    of node: some FreestandingMacroExpansionSyntax,
+    in context: some MacroExpansionContext
+  ) -> ExprSyntax {
+    "\(node.argumentList.first!.expression) as Int"
+  }
+}
+
 public struct WrapInType: PeerMacro {
   public static func expansion(
     of node: AttributeSyntax,

test/SourceKit/Diagnostics/rdar107281079.swift

@@ -0,0 +1,32 @@
+// RUN: %sourcekitd-test -req=diags %s -- %s -I %S/../Inputs/header_with_macro | %FileCheck %s
+
+import HeaderWithMacro
+
+_ = FOO(5)
+
+// CHECK:      key.diagnostics: [
+// CHECK-NEXT:   {
+// CHECK-NEXT:     key.line: 5,
+// CHECK-NEXT:     key.column: 5,
+// CHECK-NEXT:     key.filepath: "{{.*}}/rdar107281079.swift",
+// CHECK-NEXT:     key.severity: source.diagnostic.severity.error,
+// CHECK-NEXT:     key.id: "cannot_find_in_scope",
+// CHECK-NEXT:     key.description: "cannot find 'FOO' in scope",
+// CHECK-NEXT:     key.ranges: [
+// CHECK-NEXT:       {
+// CHECK-NEXT:         key.offset: 124,
+// CHECK-NEXT:         key.length: 3
+// CHECK-NEXT:       }
+// CHECK-NEXT:     ],
+// CHECK-NEXT:     key.diagnostics: [
+// CHECK-NEXT:       {
+// CHECK-NEXT:         key.line: 4,
+// CHECK-NEXT:         key.column: 9,
+// CHECK-NEXT:         key.filepath: "{{.*}}/Inputs/header_with_macro/Header.h",
+// CHECK-NEXT:         key.severity: source.diagnostic.severity.note,
+// CHECK-NEXT:         key.id: "macro_not_imported_function_like",
+// CHECK-NEXT:         key.description: "macro 'FOO' unavailable: function like macros not supported"
+// CHECK-NEXT:       }
+// CHECK-NEXT:     ]
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]

test/SourceKit/Inputs/header_with_macro/Header.h

@@ -0,0 +1,6 @@
+#ifndef Header_h
+#define Header_h
+
+#define FOO(x) #x
+
+#endif /* Header_h */

test/SourceKit/Inputs/header_with_macro/module.modulemap

@@ -0,0 +1,3 @@
+module HeaderWithMacro {
+    header "Header.h"
+}

test/SourceKit/Inputs/sourcekitd_path_sanitize.py

@@ -12,21 +12,35 @@
 import re
 import sys
 
-SWIFTMODULE_BUNDLE_RE = re.compile(
-    r'key.filepath: ".*[/\\](.*)\.swiftmodule[/\\].*\.swiftmodule"')
-SWIFTMODULE_RE = re.compile(r'key.filepath: ".*[/\\](.*)\.swiftmodule"')
-SWIFT_RE = re.compile(r'key.filepath: ".*[/\\](.*)\.swift"')
-PCM_RE = re.compile(r'key.filepath: ".*[/\\](.*)-[0-9A-Z]*\.pcm"')
-HEADER_RE = re.compile(r' file=\\".*[/\\](.*)\.h\\"')
+# I'm sorry dear reader, unfortunately my knowledge of regex trumps my knowledge of
+# Python. This pattern allows us to clean up file paths by stripping them down to
+# just the relevant file name.
+RE = re.compile(
+    # The key can either be 'filepath' or 'buffer_name'. Also apply this logic to
+    # the file name in XML, which is written 'file=\"...\"'.
+    r'(key\.(?:filepath|buffer_name): |file=)'
+
+    # Open delimiter with optional escape.
+    r'\\?"'
+
+    # Lazily match characters until we hit a slash, then match any non-slash that
+    # ends in the right file extension, capturing the result.
+    r'.*?[/\\]+([^/\\]*\.(?:swiftmodule|swift|pcm|h))'
+
+    # For swiftmodule bundles, we want to match against the directory name, so
+    # optionally match the .swiftmodule filename here. The lazy matching of the
+    # previous logic means we'll prefer to match the previous '.swiftmodule' as the
+    # directory.
+    r'(?:[/\\]+[^/\\]*\.swiftmodule)?'
+
+    # Close delimiter with optional escape.
+    r'\\?"'
+)
 
 try:
     for line in sys.stdin.readlines():
-        line = re.sub(SWIFTMODULE_BUNDLE_RE,
-                      r'key.filepath: \1.swiftmodule', line)
-        line = re.sub(SWIFTMODULE_RE, r'key.filepath: \1.swiftmodule', line)
-        line = re.sub(SWIFT_RE, r'key.filepath: \1.swift', line)
-        line = re.sub(PCM_RE, r'key.filepath: \1.pcm', line)
-        line = re.sub(HEADER_RE, r' file=\1.h', line)
+        # We substitute in both the key and the matched filename.
+        line = re.sub(RE, r'\1\2', line)
         sys.stdout.write(line)
 except KeyboardInterrupt:
     sys.stdout.flush()

test/SourceKit/Macros/diags.swift

@@ -0,0 +1,31 @@
+@freestanding(expression)
+macro coerceToInt<T>(_: T) -> Int = #externalMacro(module: "MacroDefinition", type: "CoerceToIntMacro")
+
+@freestanding(expression)
+macro stringify<T>(_: T) -> (T, String) = #externalMacro(module: "MacroDefinition", type: "StringifyMacro")
+
+@attached(peer)
+macro Invalid() = #externalMacro(module: "MacroDefinition", type: "InvalidMacro")
+
+func foo() {
+  let _ = #coerceToInt("a")
+  let _ = #coerceToInt("b")
+
+  // FIXME: There's currently a duplicate diagnostic issue with this,
+  // so the response lists the same diagnostic twice, and has a spurious
+  // buffer containing the expansion of `#coerceToInt` on its own (rdar://108622244).
+  let _ = #stringify(#coerceToInt("c"))
+}
+
+@Invalid
+struct Bad {}
+
+// REQUIRES: swift_swift_parser
+
+// RUN: %empty-directory(%t)
+
+//##-- Prepare the macro plugin.
+// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroDefinition) -module-name=MacroDefinition %S/../../Macros/Inputs/syntax_macro_definitions.swift -g -no-toolchain-stdlib-rpath
+
+// RUN: %sourcekitd-test -req=diags %s -- -swift-version 5 -load-plugin-library %t/%target-library-name(MacroDefinition) -module-name MacroUser %s | %sed_clean > %t.response
+// RUN: %diff -u %s.response %t.response