Skip to content

Commit 7720f55

Browse files
ahoppenahoppen
authored
Merge pull request #18735 from ahoppen/incremental-serialization
[libSyntax] Add support for incremental (de)serialisation of ByteTree syntax trees
2 parents 096e6ad + da7cdbb commit 7720f55

24 files changed

+435
-129
lines changed

‎include/swift/Basic/ByteTreeSerialization.h

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "llvm/Support/BinaryStreamError.h"
2323
#include "llvm/Support/BinaryStreamWriter.h"
24+
#include <map>
2425

2526
namespace {
2627
// Only used by compiler if both template types are the same
@@ -32,19 +33,26 @@ namespace swift {
3233
namespace byteTree {
3334
class ByteTreeWriter;
3435

36+
using UserInfoMap = std::map<void *, void *>;
37+
3538
/// Add a template specialization of \c ObjectTraits for any that type
3639
/// serializes as an object consisting of multiple fields.
3740
template <class T>
3841
struct ObjectTraits {
3942
// Must provide:
4043

4144
/// Return the number of fields that will be written in \c write when
42-
/// \p Object gets serialized.
43-
// static unsigned numFields(const T &Object);
45+
/// \p Object gets serialized. \p UserInfo can contain arbitrary values that
46+
/// can modify the serialization behaviour and gets passed down from the
47+
/// serialization invocation.
48+
// static unsigned numFields(const T &Object, UserInfoMap &UserInfo);
4449

4550
/// Serialize \p Object by calling \c Writer.write for all the fields of
46-
/// \p Object.
47-
// static void write(BinaryTreeWriter &Writer, const T &Object);
51+
/// \p Object. \p UserInfo can contain arbitrary values that can modify the
52+
/// serialization behaviour and gets passed down from the serialization
53+
/// invocation.
54+
// static void write(ByteTreeWriter &Writer, const T &Object,
55+
// UserInfoMap &UserInfo);
4856
};
4957

5058
/// Add a template specialization of \c ScalarTraits for any that type
@@ -79,8 +87,9 @@ struct WrapperTypeTraits {
7987
// Test if ObjectTraits<T> is defined on type T.
8088
template <class T>
8189
struct has_ObjectTraits {
82-
using Signature_numFields = unsigned (*)(const T &);
83-
using Signature_write = void (*)(ByteTreeWriter &Writer, const T &Object);
90+
using Signature_numFields = unsigned (*)(const T &, UserInfoMap &UserInfo);
91+
using Signature_write = void (*)(ByteTreeWriter &Writer, const T &Object,
92+
UserInfoMap &UserInfo);
8493

8594
template <typename U>
8695
static char test(SameType<Signature_numFields, &U::numFields> *,
@@ -144,10 +153,12 @@ class ByteTreeWriter {
144153
/// expected number of fields.
145154
unsigned CurrentFieldIndex = 0;
146155

156+
UserInfoMap &UserInfo;
157+
147158
/// The \c ByteTreeWriter can only be constructed internally. Use
148159
/// \c ByteTreeWriter.write to serialize a new object.
149-
ByteTreeWriter(llvm::BinaryStreamWriter &StreamWriter)
150-
: StreamWriter(StreamWriter) {}
160+
ByteTreeWriter(llvm::BinaryStreamWriter &StreamWriter, UserInfoMap &UserInfo)
161+
: StreamWriter(StreamWriter), UserInfo(UserInfo) {}
151162

152163
/// Set the expected number of fields the object written by this writer is
153164
/// expected to have.
@@ -188,8 +199,8 @@ class ByteTreeWriter {
188199
template <typename T>
189200
typename std::enable_if<has_ObjectTraits<T>::value, void>::type
190201
static write(uint32_t ProtocolVersion, llvm::BinaryStreamWriter &StreamWriter,
191-
const T &Object) {
192-
ByteTreeWriter Writer(StreamWriter);
202+
const T &Object, UserInfoMap &UserInfo) {
203+
ByteTreeWriter Writer(StreamWriter, UserInfo);
193204

194205
auto Error = Writer.StreamWriter.writeInteger(ProtocolVersion);
195206
(void)Error;
@@ -206,10 +217,10 @@ class ByteTreeWriter {
206217
write(const T &Object, unsigned Index) {
207218
validateAndIncreaseFieldIndex(Index);
208219

209-
auto ObjectWriter = ByteTreeWriter(StreamWriter);
210-
ObjectWriter.setNumFields(ObjectTraits<T>::numFields(Object));
220+
auto ObjectWriter = ByteTreeWriter(StreamWriter, UserInfo);
221+
ObjectWriter.setNumFields(ObjectTraits<T>::numFields(Object, UserInfo));
211222

212-
ObjectTraits<T>::write(ObjectWriter, Object);
223+
ObjectTraits<T>::write(ObjectWriter, Object, UserInfo);
213224
}
214225

215226
template <typename T>

‎include/swift/Syntax/Serialization/SyntaxSerialization.h

Lines changed: 53 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,11 @@ struct NullableTraits<RC<syntax::RawSyntax>> {
203203

204204
namespace byteTree {
205205

206+
/// The key for a ByteTree serializion user info of type
207+
/// `std::unordered_set<unsigned> *`. Specifies the IDs of syntax nodes that
208+
/// shall be omitted when the syntax tree gets serialized.
209+
static void *UserInfoKeyReusedNodeIds = &UserInfoKeyReusedNodeIds;
210+
206211
template <>
207212
struct WrapperTypeTraits<tok> {
208213
static uint8_t numericValue(const tok &Value);
@@ -229,12 +234,14 @@ template <>
229234

230235
template <>
231236
struct ObjectTraits<ArrayRef<syntax::TriviaPiece>> {
232-
static unsigned numFields(const ArrayRef<syntax::TriviaPiece> &Trivia) {
237+
static unsigned numFields(const ArrayRef<syntax::TriviaPiece> &Trivia,
238+
UserInfoMap &UserInfo) {
233239
return Trivia.size();
234240
}
235241

236242
static void write(ByteTreeWriter &Writer,
237-
const ArrayRef<syntax::TriviaPiece> &Trivia) {
243+
const ArrayRef<syntax::TriviaPiece> &Trivia,
244+
UserInfoMap &UserInfo) {
238245
for (unsigned I = 0, E = Trivia.size(); I < E; ++I) {
239246
Writer.write(Trivia[I], /*Index=*/I);
240247
}
@@ -243,64 +250,91 @@ struct ObjectTraits<ArrayRef<syntax::TriviaPiece>> {
243250

244251
template <>
245252
struct ObjectTraits<ArrayRef<RC<syntax::RawSyntax>>> {
246-
static unsigned numFields(const ArrayRef<RC<syntax::RawSyntax>> &Layout) {
253+
static unsigned numFields(const ArrayRef<RC<syntax::RawSyntax>> &Layout,
254+
UserInfoMap &UserInfo) {
247255
return Layout.size();
248256
}
249257

250258
static void write(ByteTreeWriter &Writer,
251-
const ArrayRef<RC<syntax::RawSyntax>> &Layout);
259+
const ArrayRef<RC<syntax::RawSyntax>> &Layout,
260+
UserInfoMap &UserInfo);
252261
};
253262

254263
template <>
255264
struct ObjectTraits<std::pair<tok, StringRef>> {
256-
static unsigned numFields(const std::pair<tok, StringRef> &Pair) { return 2; }
265+
static unsigned numFields(const std::pair<tok, StringRef> &Pair,
266+
UserInfoMap &UserInfo) {
267+
return 2;
268+
}
257269

258270
static void write(ByteTreeWriter &Writer,
259-
const std::pair<tok, StringRef> &Pair) {
271+
const std::pair<tok, StringRef> &Pair,
272+
UserInfoMap &UserInfo) {
260273
Writer.write(Pair.first, /*Index=*/0);
261274
Writer.write(Pair.second, /*Index=*/1);
262275
}
263276
};
264277

265278
template <>
266279
struct ObjectTraits<syntax::RawSyntax> {
267-
enum NodeKind { Token = 0, Layout = 1 };
280+
enum NodeKind { Token = 0, Layout = 1, Omitted = 2 };
268281

269-
static unsigned numFields(const syntax::RawSyntax &Syntax) {
270-
switch (nodeKind(Syntax)) {
271-
case Token:
272-
return 6;
273-
case Layout:
274-
return 5;
282+
static bool shouldOmitNode(const syntax::RawSyntax &Syntax,
283+
UserInfoMap &UserInfo) {
284+
if (auto ReusedNodeIds = static_cast<std::unordered_set<unsigned> *>(
285+
UserInfo[UserInfoKeyReusedNodeIds])) {
286+
return ReusedNodeIds->count(Syntax.getId()) > 0;
287+
} else {
288+
return false;
275289
}
276290
}
277291

278-
static NodeKind nodeKind(const syntax::RawSyntax &Syntax) {
279-
if (Syntax.isToken()) {
292+
static NodeKind nodeKind(const syntax::RawSyntax &Syntax,
293+
UserInfoMap &UserInfo) {
294+
if (shouldOmitNode(Syntax, UserInfo)) {
295+
return Omitted;
296+
} else if (Syntax.isToken()) {
280297
return Token;
281298
} else {
282299
return Layout;
283300
}
284301
}
285302

286-
static void write(ByteTreeWriter &Writer, const syntax::RawSyntax &Syntax) {
287-
auto Kind = nodeKind(Syntax);
303+
static unsigned numFields(const syntax::RawSyntax &Syntax,
304+
UserInfoMap &UserInfo) {
305+
switch (nodeKind(Syntax, UserInfo)) {
306+
case Token:
307+
return 6;
308+
case Layout:
309+
return 5;
310+
case Omitted:
311+
return 2;
312+
}
313+
}
314+
315+
static void write(ByteTreeWriter &Writer, const syntax::RawSyntax &Syntax,
316+
UserInfoMap &UserInfo) {
317+
auto Kind = nodeKind(Syntax, UserInfo);
288318

289319
Writer.write(static_cast<uint8_t>(Kind), /*Index=*/0);
290-
Writer.write(Syntax.getPresence(), /*Index=*/1);
291-
Writer.write(static_cast<uint32_t>(Syntax.getId()), /*Index=*/2);
320+
Writer.write(static_cast<uint32_t>(Syntax.getId()), /*Index=*/1);
292321

293322
switch (Kind) {
294323
case Token:
324+
Writer.write(Syntax.getPresence(), /*Index=*/2);
295325
Writer.write(std::make_pair(Syntax.getTokenKind(), Syntax.getTokenText()),
296326
/*Index=*/3);
297327
Writer.write(Syntax.getLeadingTrivia(), /*Index=*/4);
298328
Writer.write(Syntax.getTrailingTrivia(), /*Index=*/5);
299329
break;
300330
case Layout:
331+
Writer.write(Syntax.getPresence(), /*Index=*/2);
301332
Writer.write(Syntax.getKind(), /*Index=*/3);
302333
Writer.write(Syntax.getLayout(), /*Index=*/4);
303334
break;
335+
case Omitted:
336+
// Nothing more to write
337+
break;
304338
}
305339
}
306340
};

‎include/swift/Syntax/Trivia.h.gyb

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,9 +394,13 @@ struct WrapperTypeTraits<syntax::TriviaKind> {
394394

395395
template <>
396396
struct ObjectTraits<syntax::TriviaPiece> {
397-
static unsigned numFields(const syntax::TriviaPiece &Trivia) { return 2; }
397+
static unsigned numFields(const syntax::TriviaPiece &Trivia,
398+
UserInfoMap &UserInfo) {
399+
return 2;
400+
}
398401

399-
static void write(ByteTreeWriter &Writer, const syntax::TriviaPiece &Trivia) {
402+
static void write(ByteTreeWriter &Writer, const syntax::TriviaPiece &Trivia,
403+
UserInfoMap &UserInfo) {
400404
Writer.write(Trivia.getKind(), /*Index=*/0);
401405
// Write the trivia's text or count depending on its kind
402406
switch (Trivia.getKind()) {

‎lib/Syntax/SyntaxSerialization.cpp.gyb

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
namespace swift {
2222
namespace byteTree {
2323
void ObjectTraits<ArrayRef<RC<syntax::RawSyntax>>>::write(
24-
ByteTreeWriter &Writer, const ArrayRef<RC<syntax::RawSyntax>> &Layout) {
24+
ByteTreeWriter &Writer, const ArrayRef<RC<syntax::RawSyntax>> &Layout,
25+
UserInfoMap &UserInfo) {
2526
for (unsigned I = 0, E = Layout.size(); I < E; ++I) {
2627
if (Layout[I]) {
2728
Writer.write(*Layout[I], /*Index=*/I);

‎test/SwiftSyntax/AbsolutePosition.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ func getInput(_ file: String) -> URL {
1818

1919
func getSyntaxTree(_ url: URL) throws -> SourceFileSyntax {
2020
let content = try SwiftLang.parse(path: url.path).data(using: .utf8)!
21-
return try SyntaxTreeDeserializer().deserialize(content)
21+
return try SyntaxTreeDeserializer().deserialize(content,
22+
serializationFormat: .json)
2223
}
2324

2425

‎test/SwiftSyntax/DeserializeFile.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ DecodeTests.test("Basic") {
2323
let content = try SwiftLang.parse(path: getInput("visitor.swift").path)
2424
let contentData = content.data(using: .utf8)!
2525
let source = try String(contentsOf: getInput("visitor.swift"))
26-
let parsed = try SyntaxTreeDeserializer().deserialize(contentData)
26+
let parsed = try SyntaxTreeDeserializer().deserialize(contentData,
27+
serializationFormat: .json)
2728
expectEqual("\(parsed)", source)
2829
})
2930
}

‎test/SwiftSyntax/ParseFile.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ ParseFile.test("ParseSingleFile") {
3232
expectDoesNotThrow({
3333
let fileContents = try String(contentsOf: currentFile)
3434
let syntaxTreeData = try SwiftLang.parse(source: fileContents).data(using: .utf8)!
35-
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData)
35+
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData,
36+
serializationFormat: .json)
3637
expectEqual("\(parsed)", fileContents)
3738
})
3839
}
@@ -41,7 +42,8 @@ ParseFile.test("ParseBuffer") {
4142
expectDoesNotThrow({
4243
let content = "func foo() {}"
4344
let syntaxTreeData = try SwiftLang.parse(source: content).data(using: .utf8)!
44-
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData)
45+
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData,
46+
serializationFormat: .json)
4547
expectEqual("\(parsed)", content)
4648
})
4749
}

‎test/SwiftSyntax/VisitorTest.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ func getInput(_ file: String) -> URL {
1818

1919
func getSyntaxTree(_ url: URL) throws -> SourceFileSyntax {
2020
let content = try SwiftLang.parse(path: url.path).data(using: .utf8)!
21-
return try SyntaxTreeDeserializer().deserialize(content)
21+
return try SyntaxTreeDeserializer().deserialize(content,
22+
serializationFormat: .json)
2223
}
2324

2425
var VisitorTests = TestSuite("SyntaxVisitor")

‎test/incrParse/simple.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
// FIXME: Remove the requires flag when swiftSyntax builds on Linux
1919
// REQUIRES: OS=macosx
2020
// RUN: %incr-transfer-roundtrip %s --test-case REPLACE
21+
// RUN: %incr-transfer-roundtrip %s --test-case REPLACE --serialization-format byteTree
2122
// RUN: %incr-transfer-roundtrip %s --test-case INSERT
2223
// RUN: %incr-transfer-roundtrip %s --test-case REMOVE
2324
// RUN: %incr-transfer-roundtrip %s --test-case CLASS_SURROUNDING
2425
// RUN: %incr-transfer-roundtrip %s --test-case MULTI_EDIT
2526
// RUN: %incr-transfer-roundtrip %s --test-case REPLACE_WITH_MULTI_BYTE_CHAR
27+
// RUN: %incr-transfer-roundtrip %s --test-case REPLACE_WITH_MULTI_BYTE_CHAR --serialization-format byteTree
2628

2729
func start() {}
2830

‎test/lit.cfg

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,8 +348,7 @@ lit_config.note("Using code completion cache: " + completion_cache_path)
348348

349349
config.substitutions.append( ('%validate-incrparse', '%utils/incrparse/validate_parse.py --temp-dir %t --swift-syntax-test %swift-syntax-test') )
350350
config.substitutions.append( ('%incr-transfer-tree', '%utils/incrparse/incr_transfer_tree.py --temp-dir %t --swift-syntax-test %swift-syntax-test') )
351-
config.substitutions.append( ('%incr-transfer-roundtrip', '%%utils/incrparse/incr_transfer_round_trip.py --temp-dir %%t --swift-syntax-test %%swift-syntax-test --swift-swiftsyntax-test %r' % (config.swift_swiftsyntax_test)) )
352-
config.substitutions.append( ('%swift-swiftsyntax-test', '%r' % config.swift_swiftsyntax_test))
351+
config.substitutions.append( ('%incr-transfer-roundtrip', '%utils/incrparse/incr_transfer_round_trip.py --temp-dir %t --swift-syntax-test %swift-syntax-test --swift-swiftsyntax-test %swift-swiftsyntax-test') )
353352
config.substitutions.append( ('%swift_obj_root', config.swift_obj_root) )
354353
config.substitutions.append( ('%swift_src_root', config.swift_src_root) )
355354
config.substitutions.append( ('%{python}', sys.executable) )
@@ -367,6 +366,7 @@ config.substitutions.append( ('%lldb-moduleimport-test', "%r %s" % (config.lldb_
367366
config.substitutions.append( ('%swift-ide-test_plain', config.swift_ide_test) )
368367
config.substitutions.append( ('%swift-ide-test', "%r %s %s -swift-version %s" % (config.swift_ide_test, mcp_opt, ccp_opt, swift_version)) )
369368
config.substitutions.append( ('%swift-syntax-test', config.swift_syntax_test) )
369+
config.substitutions.append( ('%swift-swiftsyntax-test', config.swift_swiftsyntax_test) )
370370
config.substitutions.append( ('%swift-format', config.swift_format) )
371371
config.substitutions.append( ('%llvm-link', config.llvm_link) )
372372
config.substitutions.append( ('%swift-llvm-opt', config.swift_llvm_opt) )

‎tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2037,6 +2037,7 @@ class SKEditorConsumer : public EditorConsumer {
20372037
ResponseReceiver RespReceiver;
20382038
ResponseBuilder RespBuilder;
20392039

2040+
public:
20402041
ResponseBuilder::Dictionary Dict;
20412042
DocStructureArrayBuilder DocStructure;
20422043
TokenAnnotationsArrayBuilder SyntaxMap;
@@ -2443,8 +2444,10 @@ void serializeSyntaxTreeAsByteTree(
24432444
// Serialize the syntax tree as a ByteTree
24442445
llvm::AppendingBinaryByteStream Stream(llvm::support::endianness::little);
24452446
llvm::BinaryStreamWriter Writer(Stream);
2447+
std::map<void *, void *> UserInfo;
2448+
UserInfo[swift::byteTree::UserInfoKeyReusedNodeIds] = &ReusedNodeIds;
24462449
swift::byteTree::ByteTreeWriter::write(/*ProtocolVersion=*/1, Writer,
2447-
*SyntaxTree.getRaw());
2450+
*SyntaxTree.getRaw(), UserInfo);
24482451

24492452
std::unique_ptr<llvm::WritableMemoryBuffer> Buf =
24502453
llvm::WritableMemoryBuffer::getNewUninitMemBuffer(Stream.data().size());

0 commit comments

Comments
 (0)