Add (unsupported experimental) feature allowing byte aligned bitcode.

This is the first step of allowing PNaCl bitcode files to generate and
accept byte aligned bitcode records. To write bitcode files with byte
alignment, one uses the command-line argument
'--align-bitcode-records'. When such bitcode files are written, a
corresponding flag is added to the bitcode header so that the reader
knows whether it should align bitcode records. This allows old PNaCl
bitcode files to continue to work.

BUG=https://code.google.com/p/nativeclient/issues/detail?id=4164
[email protected]

Review URL: https://codereview.chromium.org/1122423005

Author: KarlSchimpf

include/llvm/Bitcode/NaCl/NaClBitcodeHeader.h

@@ -38,13 +38,18 @@ class NaClBitcodeHeaderField {
   // Defines the ID associated with the value. Valid values are in
   // {0x0, ..., 0xFFF}
   typedef enum {
-    kInvalid = 0,     // KUnknownType.
-    kPNaClVersion = 1 // kUint32.
+    kInvalid = 0,             // KUnknownType.
+    kPNaClVersion = 1,        // kUint32Type.
+    kAlignBitcodeRecords = 2, // kFlagType.
+    kTag_MAX = kAlignBitcodeRecords
   } Tag;
   // Defines the type of value.
   typedef enum {
     kBufferType, // Buffer of form uint8_t[len].
-    kUInt32Type
+    kUInt32Type,
+    kFlagType,
+    kUnknownType,
+    kFieldType_MAX = kUnknownType
   } FieldType;
   // Defines the number of bytes in a (32-bit) word.
   static const int WordSize = 4;
@@ -55,6 +60,9 @@ class NaClBitcodeHeaderField {
   // Create an invalid header field.
   NaClBitcodeHeaderField();
 
+  // Creates a header field where MyID is a flag.
+  NaClBitcodeHeaderField(Tag MyID);
+
   // Create a header field with an uint32_t value.
   NaClBitcodeHeaderField(Tag MyID, uint32_t value);
 
@@ -78,6 +86,18 @@ class NaClBitcodeHeaderField {
   /// \brief Read field from Buf[BufLen].
   bool Read(const uint8_t *Buf, size_t BufLen);
 
+  /// \brief Returns string describing ID of field.
+  static const char *IDName(Tag ID);
+  const char *IDName() const {
+    return IDName(ID);
+  }
+
+  /// \brief Returns string describing type of field.
+  static const char *TypeName(FieldType FType);
+  const char *TypeName() const {
+    return TypeName(FType);
+  }
+
   /// \brief Returns string describing field.
   std::string Contents() const;
 
@@ -108,8 +128,11 @@ class NaClBitcodeHeaderField {
   FixedSubfield EncodeTypedID() const { return (ID << 4) | FType; }
   // Extract out ID and Type from a fixed subfield.
   void DecodeTypedID(FixedSubfield Subfield, Tag &ID, FieldType &FType) {
-    ID = static_cast<Tag>(Subfield >> 4);
-    FType = static_cast<FieldType>(Subfield & 0xF);
+    FixedSubfield PossibleID = Subfield >> 4;
+    ID = (PossibleID > kTag_MAX ? kInvalid : static_cast<Tag>(PossibleID));
+    FixedSubfield PossibleFType = Subfield & 0xF;
+    FType = (PossibleFType > kFieldType_MAX
+             ? kUnknownType : static_cast<FieldType>(PossibleFType));
   }
   // Combined size of the fixed subfields
   const static size_t kTagLenSize = 2 * sizeof(FixedSubfield);
@@ -144,6 +167,8 @@ class NaClBitcodeHeader {
   bool IsReadableFlag;
   // Defines the PNaCl version defined by the header file.
   uint32_t PNaClVersion;
+  // Byte align bitcode records when nonzero.
+  bool AlignBitcodeRecords = false;
 
 public:
   static const int WordSize = NaClBitcodeHeaderField::WordSize;
@@ -173,7 +198,7 @@ class NaClBitcodeHeader {
   /// field.
   ///
   /// Returns false if able to read (all of) the bitcode header.
-  bool Read(const unsigned char *&BufPtr, const unsigned char *&BufEnd);
+  bool Read(const unsigned char *BufPtr, const unsigned char *BufEnd);
 
   // \brief Read the PNaCl bitcode header, recording the fields found
   // in the header. Returns false if able to read (all of) the bitcode header.
@@ -209,6 +234,9 @@ class NaClBitcodeHeader {
   /// \brief Returns the PNaClVersion, as defined by the header.
   uint32_t GetPNaClVersion() const { return PNaClVersion; }
 
+  /// \brief Returns if one should byte align bitcode records.
+  bool getAlignBitcodeRecords() const { return AlignBitcodeRecords; }
+
 private:
   // Reads and verifies the first 8 bytes of the header, consisting
   // of the magic number 'PEXE', and the value defining the number

include/llvm/Bitcode/NaCl/NaClBitstreamReader.h

@@ -17,6 +17,7 @@
 #define LLVM_BITCODE_NACL_NACLBITSTREAMREADER_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
 #include "llvm/Bitcode/NaCl/NaClLLVMBitCodes.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/StreamingMemoryObject.h"
@@ -61,34 +62,48 @@ class NaClBitstreamReader {
     std::vector<NaClBitCodeAbbrev*> Abbrevs;
   };
 private:
+  friend class NaClBitstreamCursor;
+
   std::unique_ptr<MemoryObject> BitcodeBytes;
 
   std::vector<BlockInfo> BlockInfoRecords;
 
   /// \brief Holds the offset of the first byte after the header.
   size_t InitialAddress;
 
+  // True if filler should be added to byte align records.
+  bool AlignBitcodeRecords = false;
+
   NaClBitstreamReader(const NaClBitstreamReader&) LLVM_DELETED_FUNCTION;
   void operator=(const NaClBitstreamReader&) LLVM_DELETED_FUNCTION;
-public:
-  NaClBitstreamReader() : InitialAddress(0) {}
 
+  void initFromHeader(NaClBitcodeHeader &Header) {
+    InitialAddress = Header.getHeaderSize();
+    AlignBitcodeRecords = Header.getAlignBitcodeRecords();
+  }
+
+public:
+  /// Read stream from sequence of bytes [Start .. End) after parsing
+  /// the given bitcode header.
   NaClBitstreamReader(const unsigned char *Start, const unsigned char *End,
-                      size_t MyInitialAddress=0) {
-    InitialAddress = MyInitialAddress;
-    init(Start, End);
+                      NaClBitcodeHeader &Header)
+      : BitcodeBytes(getNonStreamedMemoryObject(Start, End)) {
+    initFromHeader(Header);
   }
 
-  NaClBitstreamReader(MemoryObject *Bytes, size_t MyInitialAddress=0)
-      : InitialAddress(MyInitialAddress) {
-    BitcodeBytes.reset(Bytes);
+  /// Read stream from Bytes, after parsing the given bitcode header.
+  NaClBitstreamReader(MemoryObject *Bytes, NaClBitcodeHeader &Header)
+      : BitcodeBytes(Bytes) {
+    initFromHeader(Header);
   }
 
-  void init(const unsigned char *Start, const unsigned char *End) {
-    assert(((End-Start) & 3) == 0 &&"Bitcode stream not a multiple of 4 bytes");
-    BitcodeBytes.reset(getNonStreamedMemoryObject(Start, End));
+  /// Read stream from bytes, starting at the given initial address.
+  /// Provides simple API for unit testing.
+  NaClBitstreamReader(MemoryObject *Bytes, size_t InitialAddress)
+      : BitcodeBytes(Bytes), InitialAddress(InitialAddress) {
   }
 
+  // Returns the memory object that is being read.
   MemoryObject &getBitcodeBytes() { return *BitcodeBytes; }
 
   ~NaClBitstreamReader() {
@@ -516,6 +531,19 @@ class NaClBitstreamCursor {
   }
 
 private:
+  void SkipToByteBoundary() {
+    unsigned BitsToSkip = BitsInCurWord % CHAR_BIT;
+    if (BitsToSkip) {
+      CurWord >>= BitsToSkip;
+      BitsInCurWord -= BitsToSkip;
+    }
+  }
+
+  void SkipToByteBoundaryIfAligned() {
+    if (BitStream->AlignBitcodeRecords)
+      SkipToByteBoundary();
+  }
+
   void SkipToFourByteBoundary() {
     // If word_t is 64-bits and if we've read less than 32 bits, just dump
     // the bits we have up to the next 32-bit boundary.

include/llvm/Bitcode/NaCl/NaClBitstreamWriter.h

@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitcode/NaCl/NaClBitCodes.h"
+#include "llvm/Bitcode/NaCl/NaClBitcodeHeader.h"
 #include <vector>
 
 namespace llvm {
@@ -61,6 +62,9 @@ class NaClBitstreamWriter {
   };
   std::vector<BlockInfo> BlockInfoRecords;
 
+  // True if filler should be added to byte align records.
+  bool AlignBitcodeRecords = false;
+
   /// AbbrevValues - Wrapper class that allows the bitstream writer to
   /// prefix a code to the set of values, associated with a record to
   /// emit, without having to destructively change the contents of
@@ -137,6 +141,10 @@ class NaClBitstreamWriter {
     }
   }
 
+  void initFromHeader(const NaClBitcodeHeader &Header) {
+    AlignBitcodeRecords = Header.getAlignBitcodeRecords();
+  }
+
   /// \brief Retrieve the current position in the stream, in bits.
   uint64_t GetCurrentBitNo() const { return GetBufferOffset() * 8 + CurBit; }
 
@@ -172,6 +180,17 @@ class NaClBitstreamWriter {
     }
   }
 
+  void flushToByte() {
+    unsigned BitsToFlush = (32 - CurBit) % CHAR_BIT;
+    if (BitsToFlush)
+      Emit(0, BitsToFlush);
+  }
+
+  void flushToByteIfAligned() {
+    if (AlignBitcodeRecords)
+      flushToByte();
+  }
+
   void FlushToWord() {
     if (CurBit) {
       WriteWord(CurValue);
@@ -416,12 +435,14 @@ class NaClBitstreamWriter {
       EmitVBR(static_cast<uint32_t>(Vals.size()), 6);
       for (unsigned i = 0, e = static_cast<unsigned>(Vals.size()); i != e; ++i)
         EmitVBR64(Vals[i], 6);
+      flushToByteIfAligned();
       return;
     }
 
     // combine code and values, and then emit.
     AbbrevValues<uintty> AbbrevVals(Code, Vals);
     EmitRecordWithAbbrevImpl(Abbrev, AbbrevVals);
+    flushToByteIfAligned();
   }
 
   //===--------------------------------------------------------------------===//
@@ -446,6 +467,7 @@ class NaClBitstreamWriter {
           EmitVBR64(Op.getValue(), 5);
       }
     }
+    flushToByteIfAligned();
   }
 public:
 

lib/Bitcode/NaCl/Analysis/NaClBitcodeAnalyzer.cpp

@@ -419,7 +419,7 @@ int AnalyzeBitcodeInBuffer(const std::unique_ptr<MemoryBuffer> &Buf,
   if (!Header.IsReadable())
     Error("Bitcode file is not readable");
 
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr);
+  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
   NaClBitstreamCursor Stream(StreamFile);
 
   unsigned NumTopBlocks = 0;

lib/Bitcode/NaCl/Analysis/NaClCompress.cpp

@@ -1161,11 +1161,16 @@ static bool analyzeBitcode(
 
   // First read header and verify it is good.
   NaClBitcodeHeader Header;
-  if (Header.Read(BufPtr, EndBufPtr) || !Header.IsSupported())
+  if (Header.Read(BufPtr, EndBufPtr))
     return Error("Invalid PNaCl bitcode header");
+  if (!Header.IsSupported()) {
+    errs() << Header.Unsupported();
+    if (!Header.IsReadable())
+      return Error("Invalid PNaCl bitcode header");
+  }
 
   // Create a bitstream reader to read the bitcode file.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr);
+  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
   NaClBitstreamCursor Stream(StreamFile);
 
   // Parse the the bitcode file.
@@ -1389,7 +1394,7 @@ static bool chooseAbbrevs(MemoryBuffer *MemBuf, BlockAbbrevsMapType &AbbrevsMap,
     return Error("Invalid PNaCl bitcode header");
 
   // Create the bitcode reader.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr);
+  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
   NaClBitstreamCursor Stream(StreamFile);
 
   // Set up the parser.
@@ -1587,7 +1592,7 @@ static bool copyBitcode(const NaClBitcodeCompressor::CompressFlags &Flags,
     return Error("Invalid PNaCl bitcode header");
 
   // Create the bitcode reader.
-  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr);
+  NaClBitstreamReader StreamFile(BufPtr, EndBufPtr, Header);
   NaClBitstreamCursor Stream(StreamFile);
 
   // Create the bitcode writer.

lib/Bitcode/NaCl/Analysis/NaClObjDump.cpp

@@ -3531,18 +3531,23 @@ bool NaClObjDump(MemoryBufferRef MemBuf, raw_ostream &Output,
 
   const unsigned char *BufPtr = (const unsigned char *)MemBuf.getBufferStart();
   const unsigned char *EndBufPtr = BufPtr+MemBuf.getBufferSize();
-  const unsigned char *HeaderPtr = BufPtr;
 
   // Read header and verify it is good.
   NaClBitcodeHeader Header;
-  if (Header.Read(HeaderPtr, EndBufPtr) || !Header.IsSupported()) {
+  if (Header.Read(BufPtr, EndBufPtr)) {
     ObjDump.Error() << "Invalid PNaCl bitcode header.\n";
     return true;
   }
+  if (!Header.IsSupported()) {
+    ObjDump.Warning() << Header.Unsupported();
+    if (!Header.IsReadable()) {
+      ObjDump.Error() << "Invalid PNaCl bitcode header.\n";
+      return true;
+    }
+  }
 
   // Create a bitstream reader to read the bitcode file.
-  NaClBitstreamReader InputStreamFile(BufPtr, EndBufPtr,
-                                      Header.getHeaderSize());
+  NaClBitstreamReader InputStreamFile(BufPtr, EndBufPtr, Header);
   NaClBitstreamCursor InputStream(InputStreamFile);
 
   // Parse the the bitcode file.