Add AST_SIGNATURE record to unhashed control block of PCM files

Summary:
This record is constructed by hashing the bytes of the AST block in a similiar
fashion to the SIGNATURE record. This new signature only means anything if the
AST block is fully relocatable, i.e. it does not embed absolute offsets within
the PCM file. This change ensure this does not happen by replacing these offsets
with offsets relative to the nearest relevant subblock of the AST block.

Reviewers: Bigcheese, dexonsmith

Subscribers: dexonsmith, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D80383

Author: daniel-grumberg

clang/include/clang/Serialization/ASTBitCodes.h

@@ -41,7 +41,7 @@ namespace serialization {
 /// Version 4 of AST files also requires that the version control branch and
 /// revision match exactly, since there is no backward compatibility of
 /// AST files at this time.
-const unsigned VERSION_MAJOR = 10;
+const unsigned VERSION_MAJOR = 11;
 
 /// AST file minor version number supported by this version of
 /// Clang.
@@ -242,14 +242,16 @@ class TypeIdx {
       /// Raw source location.
       unsigned Loc = 0;
 
-      /// Offset in the AST file. Keep structure alignment 32-bit and avoid
-      /// padding gap because undefined value in the padding affects AST hash.
+      /// Offset relative to the start of the DECLTYPES_BLOCK block. Keep
+      /// structure alignment 32-bit and avoid padding gap because undefined
+      /// value in the padding affects AST hash.
       UnderalignedInt64 BitOffset;
 
       DeclOffset() = default;
-      DeclOffset(SourceLocation Loc, uint64_t BitOffset) {
+      DeclOffset(SourceLocation Loc, uint64_t BitOffset,
+                 uint64_t DeclTypesBlockStartOffset) {
         setLocation(Loc);
-        setBitOffset(BitOffset);
+        setBitOffset(BitOffset, DeclTypesBlockStartOffset);
       }
 
       void setLocation(SourceLocation L) {
@@ -260,12 +262,13 @@ class TypeIdx {
         return SourceLocation::getFromRawEncoding(Loc);
       }
 
-      void setBitOffset(uint64_t Offset) {
-        BitOffset.setBitOffset(Offset);
+      void setBitOffset(uint64_t Offset,
+                        const uint64_t DeclTypesBlockStartOffset) {
+        BitOffset.setBitOffset(Offset - DeclTypesBlockStartOffset);
       }
 
-      uint64_t getBitOffset() const {
-        return BitOffset.getBitOffset();
+      uint64_t getBitOffset(const uint64_t DeclTypesBlockStartOffset) const {
+        return BitOffset.getBitOffset() + DeclTypesBlockStartOffset;
       }
     };
 
@@ -394,6 +397,9 @@ class TypeIdx {
       /// Record code for the signature that identifiers this AST file.
       SIGNATURE = 1,
 
+      /// Record code for the content hash of the AST block.
+      AST_BLOCK_HASH,
+
       /// Record code for the diagnostic options table.
       DIAGNOSTIC_OPTIONS,
 

clang/include/clang/Serialization/ASTReader.h

@@ -1890,7 +1890,8 @@ class ASTReader
   /// ReadBlockAbbrevs - Enter a subblock of the specified BlockID with the
   /// specified cursor.  Read the abbreviations that are at the top of the block
   /// and then leave the cursor pointing into the block.
-  static bool ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor, unsigned BlockID);
+  static bool ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor, unsigned BlockID,
+                               uint64_t *StartOfBlockOffset = nullptr);
 
   /// Finds all the visible declarations with a given name.
   /// The current implementation of this method just loads the entire

clang/include/clang/Serialization/ASTWriter.h

@@ -27,6 +27,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -137,6 +138,12 @@ class ASTWriter : public ASTDeserializationListener,
   /// The module we're currently writing, if any.
   Module *WritingModule = nullptr;
 
+  /// The offset of the first bit inside the AST_BLOCK.
+  uint64_t ASTBlockStartOffset = 0;
+
+  /// The range representing all the AST_BLOCK.
+  std::pair<uint64_t, uint64_t> ASTBlockRange;
+
   /// The base directory for any relative paths we emit.
   std::string BaseDirectory;
 
@@ -206,6 +213,10 @@ class ASTWriter : public ASTDeserializationListener,
   /// the declaration's ID.
   std::vector<serialization::DeclOffset> DeclOffsets;
 
+  /// The offset of the DECLTYPES_BLOCK. The offsets in DeclOffsets
+  /// are relative to this value.
+  uint64_t DeclTypesBlockStartOffset = 0;
+
   /// Sorted (by file offset) vector of pairs of file offset/DeclID.
   using LocDeclIDsTy =
       SmallVector<std::pair<unsigned, serialization::DeclID>, 64>;
@@ -441,7 +452,7 @@ class ASTWriter : public ASTDeserializationListener,
 
   /// A list of the module file extension writers.
   std::vector<std::unique_ptr<ModuleFileExtensionWriter>>
-    ModuleFileExtensionWriters;
+      ModuleFileExtensionWriters;
 
   /// Retrieve or create a submodule ID for this module.
   unsigned getSubmoduleID(Module *Mod);
@@ -458,7 +469,8 @@ class ASTWriter : public ASTDeserializationListener,
                                              ASTContext &Context);
 
   /// Calculate hash of the pcm content.
-  static ASTFileSignature createSignature(StringRef Bytes);
+  static std::pair<ASTFileSignature, ASTFileSignature>
+  createSignature(StringRef AllBytes, StringRef ASTBlockBytes);
 
   void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts,
                        bool Modules);

clang/include/clang/Serialization/ModuleFile.h

@@ -168,6 +168,10 @@ class ModuleFile {
   /// and modification time to identify this particular file.
   ASTFileSignature Signature;
 
+  /// The signature of the AST block of the module file, this can be used to
+  /// unique module files based on AST contents.
+  ASTFileSignature ASTBlockHash;
+
   /// Whether this module has been directly imported by the
   /// user.
   bool DirectlyImported = false;
@@ -185,6 +189,9 @@ class ModuleFile {
   /// The global bit offset (or base) of this module
   uint64_t GlobalBitOffset = 0;
 
+  /// The bit offset of the AST block of this module.
+  uint64_t ASTBlockStartOffset = 0;
+
   /// The serialized bitstream data for this file.
   StringRef Data;
 
@@ -242,6 +249,9 @@ class ModuleFile {
   /// Cursor used to read source location entries.
   llvm::BitstreamCursor SLocEntryCursor;
 
+  /// The bit offset to the start of the SOURCE_MANAGER_BLOCK.
+  uint64_t SourceManagerBlockStartOffset = 0;
+
   /// The number of source location entries in this AST file.
   unsigned LocalNumSLocEntries = 0;
 
@@ -409,11 +419,14 @@ class ModuleFile {
 
   // === Declarations ===
 
-  /// DeclsCursor - This is a cursor to the start of the DECLS_BLOCK block. It
-  /// has read all the abbreviations at the start of the block and is ready to
-  /// jump around with these in context.
+  /// DeclsCursor - This is a cursor to the start of the DECLTYPES_BLOCK block.
+  /// It has read all the abbreviations at the start of the block and is ready
+  /// to jump around with these in context.
   llvm::BitstreamCursor DeclsCursor;
 
+  /// The offset to the start of the DECLTYPES_BLOCK block.
+  uint64_t DeclsBlockStartOffset = 0;
+
   /// The number of declarations in this AST file.
   unsigned LocalNumDecls = 0;
 

clang/lib/Serialization/ASTReader.cpp

@@ -1336,6 +1336,7 @@ bool ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
     Error(std::move(Err));
     return true;
   }
+  F.SourceManagerBlockStartOffset = SLocEntryCursor.GetCurrentBitNo();
 
   RecordData Record;
   while (true) {
@@ -1628,13 +1629,17 @@ SourceLocation ASTReader::getImportLocation(ModuleFile *F) {
 /// Enter a subblock of the specified BlockID with the specified cursor. Read
 /// the abbreviations that are at the top of the block and then leave the cursor
 /// pointing into the block.
-bool ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, unsigned BlockID) {
+bool ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, unsigned BlockID,
+                                 uint64_t *StartOfBlockOffset) {
   if (llvm::Error Err = Cursor.EnterSubBlock(BlockID)) {
     // FIXME this drops errors on the floor.
     consumeError(std::move(Err));
     return true;
   }
 
+  if (StartOfBlockOffset)
+    *StartOfBlockOffset = Cursor.GetCurrentBitNo();
+
   while (true) {
     uint64_t Offset = Cursor.GetCurrentBitNo();
     Expected<unsigned> MaybeCode = Cursor.ReadCode();
@@ -2933,6 +2938,7 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
     Error(std::move(Err));
     return Failure;
   }
+  F.ASTBlockStartOffset = Stream.GetCurrentBitNo();
 
   // Read all of the records and blocks for the AST file.
   RecordData Record;
@@ -2973,7 +2979,8 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
           Error(std::move(Err));
           return Failure;
         }
-        if (ReadBlockAbbrevs(F.DeclsCursor, DECLTYPES_BLOCK_ID)) {
+        if (ReadBlockAbbrevs(F.DeclsCursor, DECLTYPES_BLOCK_ID,
+                             &F.DeclsBlockStartOffset)) {
           Error("malformed block record in AST file");
           return Failure;
         }
@@ -3377,7 +3384,7 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
       F.SLocEntryOffsets = (const uint32_t *)Blob.data();
       F.LocalNumSLocEntries = Record[0];
       unsigned SLocSpaceSize = Record[1];
-      F.SLocEntryOffsetsBase = Record[2];
+      F.SLocEntryOffsetsBase = Record[2] + F.SourceManagerBlockStartOffset;
       std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
           SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
                                               SLocSpaceSize);
@@ -3696,7 +3703,7 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
       F.MacroOffsets = (const uint32_t *)Blob.data();
       F.LocalNumMacros = Record[0];
       unsigned LocalBaseMacroID = Record[1];
-      F.MacroOffsetsBase = Record[2];
+      F.MacroOffsetsBase = Record[2] + F.ASTBlockStartOffset;
       F.BaseMacroID = getTotalNumMacros();
 
       if (F.LocalNumMacros > 0) {
@@ -3837,17 +3844,18 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
 
   while (Data < DataEnd) {
     // FIXME: Looking up dependency modules by filename is horrible. Let's
-    // start fixing this with prebuilt and explicit modules and see how it
-    // goes...
+    // start fixing this with prebuilt, explicit and implicit modules and see
+    // how it goes...
     using namespace llvm::support;
     ModuleKind Kind = static_cast<ModuleKind>(
       endian::readNext<uint8_t, little, unaligned>(Data));
     uint16_t Len = endian::readNext<uint16_t, little, unaligned>(Data);
     StringRef Name = StringRef((const char*)Data, Len);
     Data += Len;
-    ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule
-                      ? ModuleMgr.lookupByModuleName(Name)
-                      : ModuleMgr.lookupByFileName(Name));
+    ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule ||
+                              Kind == MK_ImplicitModule
+                          ? ModuleMgr.lookupByModuleName(Name)
+                          : ModuleMgr.lookupByFileName(Name));
     if (!OM) {
       std::string Msg =
           "SourceLocation remap refers to unknown module, cannot find ";
@@ -4736,6 +4744,11 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
       if (F)
         F->Signature = ASTFileSignature::create(Record.begin(), Record.end());
       break;
+    case AST_BLOCK_HASH:
+      if (F)
+        F->ASTBlockHash =
+            ASTFileSignature::create(Record.begin(), Record.end());
+      break;
     case DIAGNOSTIC_OPTIONS: {
       bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
       if (Listener && ValidateDiagnosticOptions &&
@@ -6350,7 +6363,8 @@ ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
   assert(I != GlobalTypeMap.end() && "Corrupted global type map");
   ModuleFile *M = I->second;
   return RecordLocation(
-      M, M->TypeOffsets[Index - M->BaseTypeIndex].getBitOffset());
+      M, M->TypeOffsets[Index - M->BaseTypeIndex].getBitOffset() +
+             M->DeclsBlockStartOffset);
 }
 
 static llvm::Optional<Type::TypeClass> getTypeClassForCode(TypeCode code) {

clang/lib/Serialization/ASTReaderDecl.cpp

@@ -2869,7 +2869,7 @@ ASTReader::DeclCursorForID(DeclID ID, SourceLocation &Loc) {
   const DeclOffset &DOffs =
       M->DeclOffsets[ID - M->BaseDeclID - NUM_PREDEF_DECL_IDS];
   Loc = TranslateSourceLocation(*M, DOffs.getLocation());
-  return RecordLocation(M, DOffs.getBitOffset());
+  return RecordLocation(M, DOffs.getBitOffset(M->DeclsBlockStartOffset));
 }
 
 ASTReader::RecordLocation ASTReader::getLocalBitOffset(uint64_t GlobalOffset) {