Merge pull request #21078 from linux-on-ibm-z/s390x-bytetreeserialize

[Basic] Serialize integers in the specified byte order

Author: mundaym

include/swift/Basic/ByteTreeSerialization.h

@@ -155,7 +155,7 @@ class ByteTreeWriter {
   llvm::BinaryStreamWriter &StreamWriter;
 
   /// The underlying stream of the StreamWriter. We need this reference so that
-  /// we can call \c ExponentialGrowthAppendingBinaryByteStream.writeRaw
+  /// we can call \c ExponentialGrowthAppendingBinaryByteStream.writeInteger
   /// which is more efficient than the generic \c writeBytes of
   /// \c llvm::BinaryStreamWriter since it avoids the arbitrary size memcopy.
   ExponentialGrowthAppendingBinaryByteStream &Stream;
@@ -179,13 +179,10 @@ class ByteTreeWriter {
                  llvm::BinaryStreamWriter &StreamWriter, UserInfoMap &UserInfo)
       : StreamWriter(StreamWriter), Stream(Stream), UserInfo(UserInfo) {}
 
-  /// Write the given value to the ByteTree in the same form in which it is
-  /// represented on the serializing machine.
+  /// Write the given value to the ByteTree in little-endian byte order.
   template <typename T>
-  llvm::Error writeRaw(T Value) {
-    // FIXME: We implicitly inherit the endianess of the serializing machine.
-    // Since we're currently only supporting macOS that's not a problem for now.
-    auto Error = Stream.writeRaw(StreamWriter.getOffset(), Value);
+  llvm::Error writeInteger(T Value) {
+    auto Error = Stream.writeInteger(StreamWriter.getOffset(), Value);
     StreamWriter.setOffset(StreamWriter.getOffset() + sizeof(T));
     return Error;
   }
@@ -205,7 +202,7 @@ class ByteTreeWriter {
     // Set the most significant bit to indicate that the next construct is an
     // object and not a scalar.
     uint32_t ToWrite = NumFields | (1 << 31);
-    auto Error = writeRaw(ToWrite);
+    auto Error = writeInteger(ToWrite);
     (void)Error;
     assert(!Error);
 
@@ -241,7 +238,7 @@ class ByteTreeWriter {
     llvm::BinaryStreamWriter StreamWriter(Stream);
     ByteTreeWriter Writer(Stream, StreamWriter, UserInfo);
 
-    auto Error = Writer.writeRaw(ProtocolVersion);
+    auto Error = Writer.writeInteger(ProtocolVersion);
     (void)Error;
     assert(!Error);
 
@@ -272,7 +269,7 @@ class ByteTreeWriter {
     // bitflag that indicates if the next construct in the tree is an object
     // or a scalar.
     assert((ValueSize & ((uint32_t)1 << 31)) == 0 && "Value size too large");
-    auto SizeError = writeRaw(ValueSize);
+    auto SizeError = writeInteger(ValueSize);
     (void)SizeError;
     assert(!SizeError);
 
@@ -292,11 +289,11 @@ class ByteTreeWriter {
     validateAndIncreaseFieldIndex(Index);
 
     uint32_t ValueSize = sizeof(T);
-    auto SizeError = writeRaw(ValueSize);
+    auto SizeError = writeInteger(ValueSize);
     (void)SizeError;
     assert(!SizeError);
 
-    auto ContentError = writeRaw(Value);
+    auto ContentError = writeInteger(Value);
     (void)ContentError;
     assert(!ContentError);
   }

include/swift/Basic/ExponentialGrowthAppendingBinaryByteStream.h

@@ -33,13 +33,10 @@ class ExponentialGrowthAppendingBinaryByteStream
   /// The buffer holding the data.
   SmallVector<uint8_t, 0> Data;
 
-  llvm::support::endianness Endian;
+  /// Data in the stream is always encoded in little-endian byte order.
+  const llvm::support::endianness Endian = llvm::support::endianness::little;
 public:
-  ExponentialGrowthAppendingBinaryByteStream()
-      : ExponentialGrowthAppendingBinaryByteStream(
-            llvm::support::endianness::little) {}
-  ExponentialGrowthAppendingBinaryByteStream(llvm::support::endianness Endian)
-      : Endian(Endian) {}
+  ExponentialGrowthAppendingBinaryByteStream() = default;
 
   void reserve(size_t Size);
 
@@ -57,16 +54,11 @@ class ExponentialGrowthAppendingBinaryByteStream
 
   llvm::Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> Buffer) override;
 
-  /// This is an optimized version of \c writeBytes that assumes we know the
-  /// size of \p Value at compile time (which in particular holds for integers).
-  /// It does so by exposing the memcpy to the optimizer along with the size 
-  /// of the value being assigned; the compiler can then optimize the memcpy
-  /// into a fixed set of instructions.
-  /// This assumes that the endianess of this steam is the same as the native
-  /// endianess on the executing machine. No endianess transformations are
-  /// performed.
+  /// This is an optimized version of \c writeBytes specifically for integers.
+  /// Integers are written in little-endian byte order.
   template<typename T>
-  llvm::Error writeRaw(uint32_t Offset, T Value) {
+  llvm::Error writeInteger(uint32_t Offset, T Value) {
+    static_assert(std::is_integral<T>::value, "Integer required.");
     if (auto Error = checkOffsetForWrite(Offset, sizeof(T))) {
       return Error;
     }
@@ -77,7 +69,8 @@ class ExponentialGrowthAppendingBinaryByteStream
       Data.resize(RequiredSize);
     }
 
-    ::memcpy(Data.data() + Offset, &Value, sizeof Value);
+    llvm::support::endian::write<T, llvm::support::unaligned>(
+      Data.data() + Offset, Value, Endian);
 
     return llvm::Error::success();
   }

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -2651,8 +2651,7 @@ void serializeSyntaxTreeAsByteTree(
     ResponseBuilder::Dictionary &Dict) {
   auto StartClock = clock();
   // Serialize the syntax tree as a ByteTree
-  swift::ExponentialGrowthAppendingBinaryByteStream Stream(
-      llvm::support::endianness::little);
+  auto Stream = swift::ExponentialGrowthAppendingBinaryByteStream();
   Stream.reserve(32 * 1024);
   std::map<void *, void *> UserInfo;
   UserInfo[swift::byteTree::UserInfoKeyReusedNodeIds] = &ReusedNodeIds;

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -735,8 +735,7 @@ int doSerializeRawTree(const char *MainExecutablePath,
         return EXIT_FAILURE;
       }
 
-      swift::ExponentialGrowthAppendingBinaryByteStream Stream(
-          llvm::support::endianness::little);
+      auto Stream = ExponentialGrowthAppendingBinaryByteStream();
       Stream.reserve(32 * 1024);
       std::map<void *, void *> UserInfo;
       UserInfo[swift::byteTree::UserInfoKeyReusedNodeIds] = &ReusedNodeIds;

unittests/Basic/ExponentialGrowthAppendingBinaryByteStreamTests.cpp

@@ -18,7 +18,6 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
-using namespace llvm::support;
 using namespace swift;
 
 class ExponentialGrowthAppendingBinaryByteStreamTest : public testing::Test {};
@@ -27,7 +26,7 @@ class ExponentialGrowthAppendingBinaryByteStreamTest : public testing::Test {};
 // unittests/BinaryStreamTests.cpp in the LLVM project
 TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, ReadAndWrite) {
   StringRef Strings[] = {"1", "2", "3", "4"};
-  ExponentialGrowthAppendingBinaryByteStream Stream(support::little);
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
 
   BinaryStreamWriter Writer(Stream);
   BinaryStreamReader Reader(Stream);
@@ -76,7 +75,7 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, ReadAndWrite) {
 }
 
 TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteAtInvalidOffset) {
-  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
   EXPECT_EQ(0U, Stream.getLength());
 
   std::vector<uint8_t> InputData = {'T', 'e', 's', 't', 'T', 'e', 's', 't'};
@@ -97,7 +96,7 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteAtInvalidOffset) {
 TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, InitialSizeZero) {
   // Test that the stream also works with an initial size of 0, which doesn't
   // grow when doubled.
-  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
 
   std::vector<uint8_t> InputData = {'T', 'e', 's', 't'};
   auto Test = makeArrayRef(InputData).take_front(4);
@@ -106,7 +105,7 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, InitialSizeZero) {
 }
 
 TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, GrowMultipleSteps) {
-  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
 
   // Test that the buffer can grow multiple steps at once, e.g. 1 -> 2 -> 4
   std::vector<uint8_t> InputData = {'T', 'e', 's', 't'};
@@ -116,7 +115,7 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, GrowMultipleSteps) {
 }
 
 TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteIntoMiddle) {
-  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
 
   // Test that the stream resizes correctly if we write into its middle
   std::vector<uint8_t> InitialData = {'T', 'e', 's', 't'};
@@ -143,16 +142,16 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteIntoMiddle) {
   EXPECT_EQ(6u, Stream.getLength());
 }
 
-TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteRaw) {
-  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteInteger) {
+  auto Stream = ExponentialGrowthAppendingBinaryByteStream();
 
-  // Test the writeRaw method
+  // Test the writeInteger method
   std::vector<uint8_t> InitialData = {'H', 'e', 'l', 'l', 'o'};
   auto InitialDataRef = makeArrayRef(InitialData);
   EXPECT_THAT_ERROR(Stream.writeBytes(0, InitialDataRef), Succeeded());
   EXPECT_EQ(InitialDataRef, Stream.data());
 
-  EXPECT_THAT_ERROR(Stream.writeRaw(5, (uint8_t)' '), Succeeded());
+  EXPECT_THAT_ERROR(Stream.writeInteger(5, (uint8_t)' '), Succeeded());
   std::vector<uint8_t> AfterFirstInsert = {'H', 'e', 'l', 'l', 'o', ' '};
   auto AfterFirstInsertRef = makeArrayRef(AfterFirstInsert);
   EXPECT_EQ(AfterFirstInsertRef, Stream.data());
@@ -162,7 +161,7 @@ TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteRaw) {
                       'o' << 8 |
                       'r' << 16 |
                       'l' << 24;
-  EXPECT_THAT_ERROR(Stream.writeRaw(6, ToInsert), Succeeded());
+  EXPECT_THAT_ERROR(Stream.writeInteger(6, ToInsert), Succeeded());
   std::vector<uint8_t> AfterSecondInsert = {'H', 'e', 'l', 'l', 'o', ' ',
                                             'w', 'o', 'r', 'l'};
   auto AfterSecondInsertRef = makeArrayRef(AfterSecondInsert);