Merge pull request #18887 from ahoppen/exp-growth-stream

[SourceKit] Use an exponentially growing appending binary stream to serialize the syntax tree

Author: ahoppen

include/swift/Basic/ExponentialGrowthAppendingBinaryByteStream.h

@@ -0,0 +1,69 @@
+//===--- ExponentialGrowthAppendingBinaryByteStream.h -----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+/// Defines an \c llvm::WritableBinaryStream whose internal buffer grows
+/// exponentially in size as data is written to it. It is thus more efficient
+/// than llvm::AppendingBinaryByteStream if a lot of small data gets appended to
+/// it.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_BASIC_EXPONENTIALGROWTHAPPENDINGBINARYBYTESTREAM_H
+#define SWIFT_BASIC_EXPONENTIALGROWTHAPPENDINGBINARYBYTESTREAM_H
+
+#include "swift/Basic/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/BinaryByteStream.h"
+
+namespace swift {
+
+/// \brief An implementation of WritableBinaryStream which can write at its end
+/// causing the underlying data to grow.  This class owns the underlying data.
+class ExponentialGrowthAppendingBinaryByteStream
+    : public llvm::WritableBinaryStream {
+  /// The buffer holding the data.
+  SmallVector<uint8_t, 0> Data;
+
+  llvm::support::endianness Endian;
+public:
+  ExponentialGrowthAppendingBinaryByteStream()
+      : ExponentialGrowthAppendingBinaryByteStream(
+            llvm::support::endianness::little) {}
+  ExponentialGrowthAppendingBinaryByteStream(llvm::support::endianness Endian)
+      : Endian(Endian) {}
+
+  void reserve(size_t Size);
+
+  llvm::support::endianness getEndian() const override { return Endian; }
+
+  llvm::Error readBytes(uint32_t Offset, uint32_t Size,
+                        ArrayRef<uint8_t> &Buffer) override;
+
+  llvm::Error readLongestContiguousChunk(uint32_t Offset,
+                                         ArrayRef<uint8_t> &Buffer) override;
+
+  MutableArrayRef<uint8_t> data() { return Data; }
+
+  uint32_t getLength() override { return Data.size(); }
+
+  llvm::Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> Buffer) override;
+
+  llvm::Error commit() override { return llvm::Error::success(); }
+
+  virtual llvm::BinaryStreamFlags getFlags() const override {
+    return llvm::BSF_Write | llvm::BSF_Append;
+  }
+};
+
+} // end namespace swift
+
+#endif /* SWIFT_BASIC_EXPONENTIALGROWTHAPPENDINGBINARYBYTESTREAM_H */

lib/Basic/CMakeLists.txt

@@ -71,6 +71,7 @@ add_swift_library(swiftBasic STATIC
   DiverseStack.cpp
   Edit.cpp
   EditorPlaceholder.cpp
+  ExponentialGrowthAppendingBinaryByteStream.cpp
   FileSystem.cpp
   FileTypes.cpp
   JSONSerialization.cpp

lib/Basic/ExponentialGrowthAppendingBinaryByteStream.cpp

@@ -0,0 +1,60 @@
+//===-------- ExponentialGrowthAppendingBinaryByteStream.cpp --------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/Basic/ExponentialGrowthAppendingBinaryByteStream.h"
+
+using namespace llvm;
+using namespace swift;
+
+Error ExponentialGrowthAppendingBinaryByteStream::readBytes(
+    uint32_t Offset, uint32_t Size, ArrayRef<uint8_t> &Buffer) {
+  if (auto Error = checkOffsetForRead(Offset, Size)) {
+    return Error;
+  }
+  
+  Buffer = ArrayRef<uint8_t>(Data.data() + Offset, Size);
+  return Error::success();
+}
+
+Error ExponentialGrowthAppendingBinaryByteStream::readLongestContiguousChunk(
+    uint32_t Offset, ArrayRef<uint8_t> &Buffer) {
+  if (auto Error = checkOffsetForRead(Offset, 0)) {
+    return Error;
+  }
+
+  Buffer = ArrayRef<uint8_t>(Data.data() + Offset, Data.size() - Offset);
+  return Error::success();
+}
+
+void ExponentialGrowthAppendingBinaryByteStream::reserve(size_t Size) {
+  Data.reserve(Size);
+}
+
+Error ExponentialGrowthAppendingBinaryByteStream::writeBytes(
+    uint32_t Offset, ArrayRef<uint8_t> Buffer) {
+  if (Buffer.empty())
+    return Error::success();
+
+  if (auto Error = checkOffsetForWrite(Offset, Buffer.size())) {
+    return Error;
+  }
+  
+  // Resize the internal buffer if needed.
+  uint32_t RequiredSize = Offset + Buffer.size();
+  if (RequiredSize > Data.size()) {
+    Data.resize(RequiredSize);
+  }
+
+  ::memcpy(Data.data() + Offset, Buffer.data(), Buffer.size());
+
+  return Error::success();
+}

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -26,6 +26,7 @@
 #include "SourceKit/Support/UIdent.h"
 #include "SourceKit/SwiftLang/Factory.h"
 
+#include "swift/Basic/ExponentialGrowthAppendingBinaryByteStream.h"
 #include "swift/Basic/Mangler.h"
 #include "swift/Demangling/Demangler.h"
 #include "swift/Demangling/ManglingMacros.h"
@@ -2442,7 +2443,9 @@ void serializeSyntaxTreeAsByteTree(
     ResponseBuilder::Dictionary &Dict) {
   auto StartClock = clock();
   // Serialize the syntax tree as a ByteTree
-  llvm::AppendingBinaryByteStream Stream(llvm::support::endianness::little);
+  swift::ExponentialGrowthAppendingBinaryByteStream Stream(
+      llvm::support::endianness::little);
+  Stream.reserve(32 * 1024);
   llvm::BinaryStreamWriter Writer(Stream);
   std::map<void *, void *> UserInfo;
   UserInfo[swift::byteTree::UserInfoKeyReusedNodeIds] = &ReusedNodeIds;

unittests/Basic/CMakeLists.txt

@@ -13,6 +13,7 @@ add_swift_unittest(SwiftBasicTests
   DiverseStackTest.cpp
   EditorPlaceholderTest.cpp
   EncodedSequenceTest.cpp
+  ExponentialGrowthAppendingBinaryByteStreamTests.cpp
   FileSystemTest.cpp
   ImmutablePointerSetTest.cpp
   JSONSerialization.cpp
@@ -38,4 +39,5 @@ target_link_libraries(SwiftBasicTests
   PRIVATE
   swiftBasic
   clangBasic
+  LLVMTestingSupport
   )

unittests/Basic/ExponentialGrowthAppendingBinaryByteStreamTests.cpp

@@ -0,0 +1,146 @@
+//===--- ExponentialGrowthAppendingBinaryByteStreamTests.cpp -------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/Basic/ExponentialGrowthAppendingBinaryByteStream.h"
+#include "llvm/Support/BinaryStreamReader.h"
+#include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Testing/Support/Error.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::support;
+using namespace swift;
+
+class ExponentialGrowthAppendingBinaryByteStreamTest : public testing::Test {};
+
+// This test case has been taken and adapted from
+// unittests/BinaryStreamTests.cpp in the LLVM project
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, ReadAndWrite) {
+  StringRef Strings[] = {"1", "2", "3", "4"};
+  ExponentialGrowthAppendingBinaryByteStream Stream(support::little);
+
+  BinaryStreamWriter Writer(Stream);
+  BinaryStreamReader Reader(Stream);
+  const uint8_t *Byte;
+  StringRef Str;
+
+  // When the stream is empty, it should report a 0 length and we should get an
+  // error trying to read even 1 byte from it.
+  BinaryStreamRef ConstRef(Stream);
+  EXPECT_EQ(0U, ConstRef.getLength());
+  EXPECT_THAT_ERROR(Reader.readObject(Byte), Failed());
+
+  // But if we write to it, its size should increase and we should be able to
+  // read not just a byte, but the string that was written.
+  EXPECT_THAT_ERROR(Writer.writeCString(Strings[0]), Succeeded());
+  EXPECT_EQ(2U, ConstRef.getLength());
+  EXPECT_THAT_ERROR(Reader.readObject(Byte), Succeeded());
+
+  Reader.setOffset(0);
+  EXPECT_THAT_ERROR(Reader.readCString(Str), Succeeded());
+  EXPECT_EQ(Str, Strings[0]);
+
+  // If we drop some bytes from the front, we should still track the length as
+  // the
+  // underlying stream grows.
+  BinaryStreamRef Dropped = ConstRef.drop_front(1);
+  EXPECT_EQ(1U, Dropped.getLength());
+
+  EXPECT_THAT_ERROR(Writer.writeCString(Strings[1]), Succeeded());
+  EXPECT_EQ(4U, ConstRef.getLength());
+  EXPECT_EQ(3U, Dropped.getLength());
+
+  // If we drop zero bytes from the back, we should continue tracking the
+  // length.
+  Dropped = Dropped.drop_back(0);
+  EXPECT_THAT_ERROR(Writer.writeCString(Strings[2]), Succeeded());
+  EXPECT_EQ(6U, ConstRef.getLength());
+  EXPECT_EQ(5U, Dropped.getLength());
+
+  // If we drop non-zero bytes from the back, we should stop tracking the
+  // length.
+  Dropped = Dropped.drop_back(1);
+  EXPECT_THAT_ERROR(Writer.writeCString(Strings[3]), Succeeded());
+  EXPECT_EQ(8U, ConstRef.getLength());
+  EXPECT_EQ(4U, Dropped.getLength());
+}
+
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteAtInvalidOffset) {
+  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+  EXPECT_EQ(0U, Stream.getLength());
+
+  std::vector<uint8_t> InputData = {'T', 'e', 's', 't', 'T', 'e', 's', 't'};
+  auto Test = makeArrayRef(InputData).take_front(4);
+  // Writing past the end of the stream is an error.
+  EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Failed());
+
+  // Writing exactly at the end of the stream is ok.
+  EXPECT_THAT_ERROR(Stream.writeBytes(0, Test), Succeeded());
+  EXPECT_EQ(Test, Stream.data());
+
+  // And now that the end of the stream is where we couldn't write before, now
+  // we can write.
+  EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Succeeded());
+  EXPECT_EQ(MutableArrayRef<uint8_t>(InputData), Stream.data());
+}
+
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, InitialSizeZero) {
+  // Test that the stream also works with an initial size of 0, which doesn't
+  // grow when doubled.
+  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+
+  std::vector<uint8_t> InputData = {'T', 'e', 's', 't'};
+  auto Test = makeArrayRef(InputData).take_front(4);
+  EXPECT_THAT_ERROR(Stream.writeBytes(0, Test), Succeeded());
+  EXPECT_EQ(Test, Stream.data());
+}
+
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, GrowMultipleSteps) {
+  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+
+  // Test that the buffer can grow multiple steps at once, e.g. 1 -> 2 -> 4
+  std::vector<uint8_t> InputData = {'T', 'e', 's', 't'};
+  auto Test = makeArrayRef(InputData).take_front(4);
+  EXPECT_THAT_ERROR(Stream.writeBytes(0, Test), Succeeded());
+  EXPECT_EQ(Test, Stream.data());
+}
+
+TEST_F(ExponentialGrowthAppendingBinaryByteStreamTest, WriteIntoMiddle) {
+  // Test that the stream resizes correctly if we write into its middle
+
+  ExponentialGrowthAppendingBinaryByteStream Stream(llvm::support::little);
+
+  // Test that the buffer can grow multiple steps at once, e.g. 1 -> 2 -> 4
+  std::vector<uint8_t> InitialData = {'T', 'e', 's', 't'};
+  auto InitialDataRef = makeArrayRef(InitialData);
+  EXPECT_THAT_ERROR(Stream.writeBytes(0, InitialDataRef), Succeeded());
+  EXPECT_EQ(InitialDataRef, Stream.data());
+
+  std::vector<uint8_t> UpdateData = {'u'};
+  auto UpdateDataRef = makeArrayRef(UpdateData);
+  EXPECT_THAT_ERROR(Stream.writeBytes(1, UpdateDataRef), Succeeded());
+
+  std::vector<uint8_t> DataAfterUpdate = {'T', 'u', 's', 't'};
+  auto DataAfterUpdateRef = makeArrayRef(DataAfterUpdate);
+  EXPECT_EQ(DataAfterUpdateRef, Stream.data());
+  EXPECT_EQ(4u, Stream.getLength());
+
+  std::vector<uint8_t> InsertData = {'e', 'r'};
+  auto InsertDataRef = makeArrayRef(InsertData);
+  EXPECT_THAT_ERROR(Stream.writeBytes(4, InsertDataRef), Succeeded());
+
+  std::vector<uint8_t> DataAfterInsert = {'T', 'u', 's', 't', 'e', 'r'};
+  auto DataAfterInsertRef = makeArrayRef(DataAfterInsert);
+  EXPECT_EQ(DataAfterInsertRef, Stream.data());
+  EXPECT_EQ(6u, Stream.getLength());
+}