[Serialized diagnostics] Record generated buffer contents in diagnostics files

Extend the format for serialized diagnostic files (`.dia`) to allow for
the contents of source files to be included inside the diagnostic files
themselves. This enables producers of diagnostics files that generate
source code as part of their build (e.g., a compiler that performs
macro expansion) to provide that generated source code to consumers of
diagnostics, treating it like a normal file (e.g., with diagnostics
pointing into it) that isn't otherwise recorded anywhere on disk.

Source files with contents can, optionally, provide a source range
that describes the "original" source range from which those contents
are derived. This information is optional, but if present it can refer
back to the source locations that triggered the creation of the
generated source code, i.e., the use of a macro that caused macro
expansion.

This change extends the libclang C APIs with access to the source file
contents (via `clang_getDiagnosticFileContents`) and the original
source range (via `clang_getDiagnosticFileOriginalSourceRange`).

The diagnostics file format, which has been stable for roughly a
decade, is extended in a backward-compatible way by adding a new
record, which existing clients (i.e., older libclang versions) will
safely ignore. As such, we do not bump the diagnostics file format
version at all.

At present, the Swift compiler is the only producer of source file
content records, which uses them to provide macro-expansion buffers.
The Swift repository contains tests that produce these records and
verify their correctness via `c-index-test`.

We could consider introducing command-line flags to Clang to make it
start producing these records, e.g., to capture macro-expansion
buffers (as we're doing in Swift) or even to package up the contents
of all source files that are referenced by serialized diagnostics,
making the serialized-diagnostics file self-contained.

Author: DougGregor

clang/include/clang-c/CXDiagnostic.h

@@ -148,6 +148,30 @@ CINDEX_LINKAGE void clang_disposeDiagnosticSet(CXDiagnosticSet Diags);
  */
 CINDEX_LINKAGE CXDiagnosticSet clang_getChildDiagnostics(CXDiagnostic D);
 
+/**
+ * Get the contents if the given file that was provided via diagnostics.
+ *
+ * \param diags the diagnostics set to query for the contents of the file.
+ * \param file the file to get the contents of.
+ * \param outFileSize if non-null, set to the file size on success.
+ * \returns on success, a pointer to the file contents. Otherwise, NULL.
+ */
+CINDEX_LINKAGE const char *clang_getDiagnosticFileContents(
+    CXDiagnosticSet diags, CXFile file, size_t *outFileSize);
+
+/**
+ * Retrieve the original source range if the given file was provided via
+ * diagnostics and is conceptually a replacement for the original source range.
+ *
+ * \param diags the diagnostics set to query for the contents of the file.
+ * \param file the file to get the contents of.
+ * \returns on success, the source range (into another file) that is
+ * conceptually replaced by the contents of the given file (available via
+ * \c clang_getDiagnosticFileContents).
+ */
+CINDEX_LINKAGE CXSourceRange clang_getDiagnosticFileOriginalSourceRange(
+    CXDiagnosticSet diags, CXFile file);
+
 /**
  * Destroy a diagnostic.
  */

clang/include/clang/Frontend/SerializedDiagnosticReader.h

@@ -109,6 +109,16 @@ class SerializedDiagnosticReader {
     return {};
   }
 
+  /// Visit file contents. This associates the file's \c ID with the
+  /// contents of
+  virtual std::error_code visitSourceFileContentsRecord(
+      unsigned ID,
+      const Location &OriginalStartLoc,
+      const Location &OriginalEndLoc,
+      StringRef Contents) {
+    return {};
+  }
+
   /// Visit a fixit hint.
   virtual std::error_code
   visitFixitRecord(const Location &Start, const Location &End, StringRef Text) {

clang/include/clang/Frontend/SerializedDiagnostics.h

@@ -32,8 +32,9 @@ enum RecordIDs {
   RECORD_CATEGORY,
   RECORD_FILENAME,
   RECORD_FIXIT,
+  RECORD_SOURCE_FILE_CONTENTS,
   RECORD_FIRST = RECORD_VERSION,
-  RECORD_LAST = RECORD_FIXIT
+  RECORD_LAST = RECORD_SOURCE_FILE_CONTENTS
 };
 
 /// A stable version of DiagnosticIDs::Level.

clang/lib/Frontend/SerializedDiagnosticPrinter.cpp

@@ -93,6 +93,7 @@ class SDiagsMerger : SerializedDiagnosticReader {
   AbbrevLookup FileLookup;
   AbbrevLookup CategoryLookup;
   AbbrevLookup DiagFlagLookup;
+  llvm::DenseSet<unsigned> ContentsWritten;
 
 public:
   SDiagsMerger(SDiagsWriter &Writer) : Writer(Writer) {}
@@ -112,6 +113,12 @@ class SDiagsMerger : SerializedDiagnosticReader {
   std::error_code visitFilenameRecord(unsigned ID, unsigned Size,
                                       unsigned Timestamp,
                                       StringRef Name) override;
+  std::error_code visitSourceFileContentsRecord(
+      unsigned ID,
+      const Location &OriginalStartLoc,
+      const Location &OriginalEndLoc,
+      StringRef Contents) override;
+
   std::error_code visitFixitRecord(const serialized_diags::Location &Start,
                                    const serialized_diags::Location &End,
                                    StringRef CodeToInsert) override;
@@ -462,6 +469,8 @@ void SDiagsWriter::EmitBlockInfoBlock() {
   EmitRecordID(RECORD_DIAG_FLAG, "DiagFlag", Stream, Record);
   EmitRecordID(RECORD_FILENAME, "FileName", Stream, Record);
   EmitRecordID(RECORD_FIXIT, "FixIt", Stream, Record);
+  EmitRecordID(
+      RECORD_SOURCE_FILE_CONTENTS, "SourceFileContents", Stream, Record);
 
   // Emit abbreviation for RECORD_DIAG.
   Abbrev = std::make_shared<BitCodeAbbrev>();
@@ -518,6 +527,16 @@ void SDiagsWriter::EmitBlockInfoBlock() {
   Abbrevs.set(RECORD_FIXIT, Stream.EmitBlockInfoAbbrev(BLOCK_DIAG,
                                                        Abbrev));
 
+  // Emit the abbreviation for RECORD_SOURCE_FILE_CONTENTS.
+  Abbrev = std::make_shared<BitCodeAbbrev>();
+  Abbrev->Add(BitCodeAbbrevOp(RECORD_SOURCE_FILE_CONTENTS));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); // File ID.
+  AddRangeLocationAbbrev(*Abbrev);
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // File size.
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File contents.
+  Abbrevs.set(RECORD_SOURCE_FILE_CONTENTS,
+              Stream.EmitBlockInfoAbbrev(BLOCK_DIAG, Abbrev));
+
   Stream.ExitBlock();
 }
 
@@ -890,6 +909,28 @@ std::error_code SDiagsMerger::visitFilenameRecord(unsigned ID, unsigned Size,
   return std::error_code();
 }
 
+std::error_code SDiagsMerger::visitSourceFileContentsRecord(
+    unsigned ID,
+    const Location &OriginalStartLoc,
+    const Location &OriginalEndLoc,
+    StringRef Contents) {
+  unsigned MappedID = FileLookup[ID];
+  if (!ContentsWritten.insert(MappedID).second)
+    return std::error_code();
+
+  RecordData::value_type Record[] = {
+      RECORD_SOURCE_FILE_CONTENTS, MappedID,
+      FileLookup[OriginalStartLoc.FileID],
+      OriginalStartLoc.Line, OriginalStartLoc.Col, OriginalStartLoc.Offset,
+      FileLookup[OriginalEndLoc.FileID], OriginalEndLoc.Line,
+      OriginalEndLoc.Col, OriginalEndLoc.Offset,
+    Contents.size()};
+
+  Writer.State->Stream.EmitRecordWithBlob(
+      Writer.State->Abbrevs.get(RECORD_SOURCE_FILE_CONTENTS), Record, Contents);
+  return std::error_code();
+}
+
 std::error_code SDiagsMerger::visitCategoryRecord(unsigned ID, StringRef Name) {
   CategoryLookup[ID] = Writer.getEmitCategory(ID);
   return std::error_code();

clang/lib/Frontend/SerializedDiagnosticReader.cpp

@@ -302,6 +302,16 @@ SerializedDiagnosticReader::readDiagnosticBlock(llvm::BitstreamCursor &Stream) {
                Location(Record[4], Record[5], Record[6], Record[7]), Blob)))
         return EC;
       continue;
+    case RECORD_SOURCE_FILE_CONTENTS:
+      if (Record.size() != 10 || Record[9] != Blob.size())
+        return SDError::MalformedDiagnosticRecord;
+      if ((EC = visitSourceFileContentsRecord(
+               Record[0],
+               Location(Record[1], Record[2], Record[3], Record[4]),
+               Location(Record[5], Record[6], Record[7], Record[8]),
+               Blob)))
+        return EC;
+      continue;
     case RECORD_SOURCE_RANGE:
       // A source range is two locations (4 each).
       if (Record.size() != 8)

clang/tools/c-index-test/c-index-test.c

@@ -4730,7 +4730,8 @@ static void printDiagnosticSet(CXDiagnosticSet Diags, unsigned indent) {
     CXString FileName, DiagSpelling, DiagOption, DiagCat;
     unsigned line, column, offset;
     const char *FileNameStr = 0, *DiagOptionStr = 0, *DiagCatStr = 0;
-    
+    const char *FileContents = 0;
+
     D = clang_getDiagnosticInSet(Diags, i);
     DiagLoc = clang_getDiagnosticLocation(D);
     clang_getExpansionLocation(DiagLoc, &File, &line, &column, &offset);
@@ -4763,15 +4764,37 @@ static void printDiagnosticSet(CXDiagnosticSet Diags, unsigned indent) {
 
     printRanges(D, indent);
     printFixIts(D, indent);
-    
+
+    // If we have the source file contents for this file, print them now.
+    FileContents = clang_getDiagnosticFileContents(Diags, File, 0);
+    if (FileContents) {
+      CXSourceRange OriginalSourceRange;
+
+      fprintf(stderr, "CONTENTS OF FILE %s:\n",
+              FileNameStr ? FileNameStr : "(null)");
+
+      OriginalSourceRange = clang_getDiagnosticFileOriginalSourceRange(
+          Diags, File);
+      if (!clang_equalRanges(clang_getNullRange(), OriginalSourceRange)) {
+        printIndent(indent);
+        fprintf(stderr, "Original source range: ");
+        printLocation(clang_getRangeStart(OriginalSourceRange));
+        fprintf(stderr, " - ");
+        printLocation(clang_getRangeEnd(OriginalSourceRange));
+        fprintf(stderr, "\n");
+      }
+
+      fprintf(stderr, "%s\nEND CONTENTS OF FILE\n", FileContents);
+    }
+
     /* Print subdiagnostics. */
     printDiagnosticSet(clang_getChildDiagnostics(D), indent+2);
 
     clang_disposeString(FileName);
     clang_disposeString(DiagSpelling);
     clang_disposeString(DiagOption);
     clang_disposeString(DiagCat);
-  }  
+  }
 }
 
 static int read_diagnostics(const char *filename) {

clang/tools/libclang/CIndexDiagnostic.cpp

@@ -34,6 +34,11 @@ CXDiagnosticSetImpl::appendDiagnostic(std::unique_ptr<CXDiagnosticImpl> D) {
   Diagnostics.push_back(std::move(D));
 }
 
+void CXDiagnosticSetImpl::recordSourceFileContents(
+    CXFile file, StringRef contents, CXSourceRange originalSourceRange) {
+  FileContents[file] = CXSourceFileContents{contents, originalSourceRange};
+}
+
 CXDiagnosticImpl::~CXDiagnosticImpl() {}
 
 namespace {
@@ -479,6 +484,30 @@ CXDiagnosticSet clang_getChildDiagnostics(CXDiagnostic Diag) {
   return nullptr;
 }
 
+const char *clang_getDiagnosticFileContents(
+    CXDiagnosticSet diags, CXFile file, size_t *outFileSize) {
+  if (CXDiagnosticSetImpl *D = static_cast<CXDiagnosticSetImpl *>(diags)) {
+    CXSourceRange originalSourceRange;
+    if (auto contents = D->getSourceFileContents(file, originalSourceRange)) {
+      if (outFileSize)
+        *outFileSize = contents->size();
+      return contents->data();
+    }
+  }
+  return nullptr;
+}
+
+CXSourceRange clang_getDiagnosticFileOriginalSourceRange(
+    CXDiagnosticSet diags, CXFile file) {
+  if (CXDiagnosticSetImpl *D = static_cast<CXDiagnosticSetImpl *>(diags)) {
+    CXSourceRange originalSourceRange;
+    if (auto contents = D->getSourceFileContents(file, originalSourceRange)) {
+      return originalSourceRange;
+    }
+  }
+  return clang_getNullRange();
+}
+
 unsigned clang_getNumDiagnosticsInSet(CXDiagnosticSet Diags) {
   if (CXDiagnosticSetImpl *D = static_cast<CXDiagnosticSetImpl*>(Diags))
     return D->getNumDiagnostics();

clang/tools/libclang/CIndexDiagnostic.h

@@ -15,6 +15,9 @@
 
 #include "clang-c/Index.h"
 #include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
 #include <memory>
 #include <vector>
 #include <assert.h>
@@ -24,9 +27,15 @@ namespace clang {
 class LangOptions;
 class StoredDiagnostic;
 class CXDiagnosticImpl;
-  
+
 class CXDiagnosticSetImpl {
+  struct CXSourceFileContents {
+    StringRef Contents;
+    CXSourceRange OriginalSourceRange;
+  };
+
   std::vector<std::unique_ptr<CXDiagnosticImpl>> Diagnostics;
+  llvm::DenseMap<CXFile, CXSourceFileContents> FileContents;
   const bool IsExternallyManaged;
 public:
   CXDiagnosticSetImpl(bool isManaged = false)
@@ -45,6 +54,19 @@ class CXDiagnosticSetImpl {
 
   void appendDiagnostic(std::unique_ptr<CXDiagnosticImpl> D);
 
+  void recordSourceFileContents(
+      CXFile file, StringRef contents, CXSourceRange originalSourceRange);
+
+  Optional<StringRef> getSourceFileContents(
+      CXFile file, CXSourceRange &originalSourceRange) {
+    auto found = FileContents.find(file);
+    if (found == FileContents.end())
+      return None;
+
+    originalSourceRange = found->second.OriginalSourceRange;
+    return found->second.Contents;
+  }
+
   bool empty() const {
     return Diagnostics.empty();
   }

clang/tools/libclang/CXLoadedDiagnostic.cpp

@@ -225,6 +225,12 @@ class DiagLoader : serialized_diags::SerializedDiagnosticReader {
                                       unsigned Timestamp,
                                       StringRef Name) override;
 
+  std::error_code visitSourceFileContentsRecord(
+      unsigned ID,
+      const serialized_diags::Location &OriginalStartLoc,
+      const serialized_diags::Location &OriginalEndLoc,
+      StringRef Contents) override;
+
   std::error_code visitFixitRecord(const serialized_diags::Location &Start,
                                    const serialized_diags::Location &End,
                                    StringRef CodeToInsert) override;
@@ -347,6 +353,28 @@ std::error_code DiagLoader::visitFilenameRecord(unsigned ID, unsigned Size,
   return std::error_code();
 }
 
+std::error_code DiagLoader::visitSourceFileContentsRecord(
+    unsigned ID,
+    const serialized_diags::Location &OriginalStartLoc,
+    const serialized_diags::Location &OriginalEndLoc,
+    StringRef Contents
+) {
+  CXSourceRange OriginalSourceRange;
+  if (std::error_code EC = readRange(
+          OriginalStartLoc, OriginalEndLoc, OriginalSourceRange))
+    return EC;
+
+  auto file = const_cast<FileEntry *>(TopDiags->Files[ID]);
+  if (!file)
+    return reportInvalidFile("Source file contents for unknown file ID");
+
+  StringRef CopiedContents(TopDiags->copyString(Contents),
+                           Contents.size());
+
+  TopDiags->recordSourceFileContents(file, CopiedContents, OriginalSourceRange);
+  return std::error_code();
+}
+
 std::error_code
 DiagLoader::visitSourceRangeRecord(const serialized_diags::Location &Start,
                                    const serialized_diags::Location &End) {

clang/tools/libclang/libclang.map

@@ -484,6 +484,8 @@ LLVM_16 {
     clang_experimental_DependencyScannerServiceOptions_setObjectStore;
     clang_experimental_DependencyScannerWorker_getFileDependencies_v5;
     clang_createAPISet;
+    clang_getDiagnosticFileContents;
+    clang_getDiagnosticFileOriginalSourceRange;
     clang_disposeAPISet;
     clang_getSymbolGraphForCursor;
     clang_getSymbolGraphForUSR;