Skip to content

Commit c8fc286

Browse files
ahoppenahoppen
authored
Merge pull request #18152 from ahoppen/01-incr-tree-transfer
[libSyntax] Incremental syntax tree transfer
2 parents 3958855 + 52acc04 commit c8fc286

37 files changed

+1189
-290
lines changed

‎cmake/modules/AddSwift.cmake

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2022,7 +2022,7 @@ function(_add_swift_executable_single name)
20222022
cmake_parse_arguments(SWIFTEXE_SINGLE
20232023
"EXCLUDE_FROM_ALL;DONT_STRIP_NON_MAIN_SYMBOLS;DISABLE_ASLR"
20242024
"SDK;ARCHITECTURE"
2025-
"DEPENDS;LLVM_COMPONENT_DEPENDS;LINK_LIBRARIES;LINK_FAT_LIBRARIES"
2025+
"DEPENDS;LLVM_COMPONENT_DEPENDS;LINK_LIBRARIES;LINK_FAT_LIBRARIES;COMPILE_FLAGS"
20262026
${ARGN})
20272027

20282028
set(SWIFTEXE_SINGLE_SOURCES ${SWIFTEXE_SINGLE_UNPARSED_ARGUMENTS})
@@ -2096,6 +2096,7 @@ function(_add_swift_executable_single name)
20962096
MODULE_NAME ${name}
20972097
SDK ${SWIFTEXE_SINGLE_SDK}
20982098
ARCHITECTURE ${SWIFTEXE_SINGLE_ARCHITECTURE}
2099+
COMPILE_FLAGS ${SWIFTEXE_SINGLE_COMPILE_FLAGS}
20992100
IS_MAIN)
21002101
add_swift_source_group("${SWIFTEXE_SINGLE_EXTERNAL_SOURCES}")
21012102

@@ -2260,7 +2261,7 @@ function(add_swift_executable name)
22602261
cmake_parse_arguments(SWIFTEXE
22612262
"EXCLUDE_FROM_ALL;DONT_STRIP_NON_MAIN_SYMBOLS;DISABLE_ASLR"
22622263
""
2263-
"DEPENDS;LLVM_COMPONENT_DEPENDS;LINK_LIBRARIES"
2264+
"DEPENDS;LLVM_COMPONENT_DEPENDS;LINK_LIBRARIES;COMPILE_FLAGS"
22642265
${ARGN})
22652266

22662267
translate_flag(${SWIFTEXE_EXCLUDE_FROM_ALL}
@@ -2283,6 +2284,7 @@ function(add_swift_executable name)
22832284
LINK_LIBRARIES ${SWIFTEXE_LINK_LIBRARIES}
22842285
SDK ${SWIFT_HOST_VARIANT_SDK}
22852286
ARCHITECTURE ${SWIFT_HOST_VARIANT_ARCH}
2287+
COMPILE_FLAGS ${SWIFTEXE_COMPILE_FLAGS}
22862288
${SWIFTEXE_EXCLUDE_FROM_ALL_FLAG}
22872289
${SWIFTEXE_DONT_STRIP_NON_MAIN_SYMBOLS_FLAG}
22882290
${SWIFTEXE_DISABLE_ASLR_FLAG})
@@ -2301,11 +2303,16 @@ function(add_swift_host_tool executable)
23012303
ADDSWIFTHOSTTOOL # prefix
23022304
"" # options
23032305
"" # single-value args
2304-
"SWIFT_COMPONENT" # multi-value args
2306+
"SWIFT_COMPONENT;COMPILE_FLAGS;DEPENDS" # multi-value args
23052307
${ARGN})
23062308

23072309
# Create the executable rule.
2308-
add_swift_executable(${executable} ${ADDSWIFTHOSTTOOL_UNPARSED_ARGUMENTS})
2310+
add_swift_executable(
2311+
${executable}
2312+
${ADDSWIFTHOSTTOOL_UNPARSED_ARGUMENTS}
2313+
DEPENDS ${ADDSWIFTHOSTTOOL_DEPENDS}
2314+
COMPILE_FLAGS ${ADDSWIFTHOSTTOOL_COMPILE_FLAGS}
2315+
)
23092316

23102317
# And then create the install rule if we are asked to.
23112318
if (ADDSWIFTHOSTTOOL_SWIFT_COMPONENT)

‎include/swift/Syntax/Serialization/SyntaxSerialization.h

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@
1818
#ifndef SWIFT_SYNTAX_SERIALIZATION_SYNTAXSERIALIZATION_H
1919
#define SWIFT_SYNTAX_SERIALIZATION_SYNTAXSERIALIZATION_H
2020

21-
#include "swift/Syntax/RawSyntax.h"
2221
#include "swift/Basic/JSONSerialization.h"
2322
#include "swift/Basic/StringExtras.h"
23+
#include "swift/Syntax/RawSyntax.h"
2424
#include "llvm/ADT/StringSwitch.h"
2525
#include <forward_list>
26+
#include <unordered_set>
2627

2728
namespace swift {
2829
namespace json {
@@ -31,6 +32,10 @@ namespace json {
3132
/// will not be included in the serialized JSON.
3233
static void *DontSerializeNodeIdsUserInfoKey = &DontSerializeNodeIdsUserInfoKey;
3334

35+
/// The user info key pointing to a std::unordered_set of IDs of nodes that
36+
/// shall be omitted when the tree gets serialized
37+
static void *OmitNodesUserInfoKey = &OmitNodesUserInfoKey;
38+
3439
/// Serialization traits for SourcePresence.
3540
template <>
3641
struct ScalarEnumerationTraits<syntax::SourcePresence> {
@@ -125,6 +130,22 @@ struct ObjectTraits<TokenDescription> {
125130
template<>
126131
struct ObjectTraits<syntax::RawSyntax> {
127132
static void mapping(Output &out, syntax::RawSyntax &value) {
133+
bool dontSerializeIds =
134+
(bool)out.getUserInfo()[DontSerializeNodeIdsUserInfoKey];
135+
if (!dontSerializeIds) {
136+
auto nodeId = value.getId();
137+
out.mapRequired("id", nodeId);
138+
}
139+
140+
auto omitNodes =
141+
(std::unordered_set<unsigned> *)out.getUserInfo()[OmitNodesUserInfoKey];
142+
143+
if (omitNodes && omitNodes->count(value.getId()) > 0) {
144+
bool omitted = true;
145+
out.mapRequired("omitted", omitted);
146+
return;
147+
}
148+
128149
if (value.isToken()) {
129150
auto tokenKind = value.getTokenKind();
130151
auto text = value.getTokenText();
@@ -145,12 +166,6 @@ struct ObjectTraits<syntax::RawSyntax> {
145166
}
146167
auto presence = value.getPresence();
147168
out.mapRequired("presence", presence);
148-
149-
bool omitNodeId = (bool)out.getUserInfo()[DontSerializeNodeIdsUserInfoKey];
150-
if (!omitNodeId) {
151-
auto nodeId = value.getId();
152-
out.mapRequired("id", nodeId);
153-
}
154169
}
155170
};
156171

‎test/SwiftSyntax/AbsolutePosition.swift

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ func getInput(_ file: String) -> URL {
1616
return result
1717
}
1818

19+
func getSyntaxTree(_ url: URL) throws -> SourceFileSyntax {
20+
let content = try SwiftLang.parse(url).data(using: .utf8)!
21+
return try SyntaxTreeDeserializer().deserialize(content)
22+
}
23+
1924

2025
class FuncRenamer: SyntaxRewriter {
2126
override func visit(_ node: FunctionDeclSyntax) ->DeclSyntax {
@@ -28,9 +33,8 @@ var PositionTests = TestSuite("AbsolutePositionTests")
2833

2934
PositionTests.test("Visitor") {
3035
expectDoesNotThrow({
31-
let content = try SwiftLang.parse(getInput("visitor.swift"))
3236
let source = try String(contentsOf: getInput("visitor.swift"))
33-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
37+
let parsed = try getSyntaxTree(getInput("visitor.swift"))
3438
expectEqual(0, parsed.position.utf8Offset)
3539
expectEqual(source.count,
3640
parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset)
@@ -41,9 +45,8 @@ PositionTests.test("Visitor") {
4145

4246
PositionTests.test("Closure") {
4347
expectDoesNotThrow({
44-
let content = try SwiftLang.parse(getInput("closure.swift"))
4548
let source = try String(contentsOf: getInput("closure.swift"))
46-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
49+
let parsed = try getSyntaxTree(getInput("closure.swift"))
4750
expectEqual(source.count,
4851
parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset)
4952
expectEqual(0, parsed.position.utf8Offset)
@@ -53,8 +56,7 @@ PositionTests.test("Closure") {
5356

5457
PositionTests.test("Rename") {
5558
expectDoesNotThrow({
56-
let content = try SwiftLang.parse(getInput("visitor.swift"))
57-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
59+
let parsed = try getSyntaxTree(getInput("visitor.swift"))
5860
let renamed = FuncRenamer().visit(parsed) as! SourceFileSyntax
5961
let renamedSource = renamed.description
6062
expectEqual(renamedSource.count,
@@ -65,8 +67,7 @@ PositionTests.test("Rename") {
6567

6668
PositionTests.test("CurrentFile") {
6769
expectDoesNotThrow({
68-
let content = try SwiftLang.parse(URL(fileURLWithPath: #file))
69-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
70+
let parsed = try getSyntaxTree(URL(fileURLWithPath: #file))
7071
class Visitor: SyntaxVisitor {
7172
override func visitPre(_ node: Syntax) {
7273
_ = node.position

‎test/SwiftSyntax/DeserializeFile.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ var DecodeTests = TestSuite("DecodeSyntax")
2121
DecodeTests.test("Basic") {
2222
expectDoesNotThrow({
2323
let content = try SwiftLang.parse(getInput("visitor.swift"))
24+
let contentData = content.data(using: .utf8)!
2425
let source = try String(contentsOf: getInput("visitor.swift"))
25-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
26+
let parsed = try SyntaxTreeDeserializer().deserialize(contentData)
2627
expectEqual("\(parsed)", source)
2728
})
2829
}

‎test/SwiftSyntax/DiagnosticTest.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ Diagnostics.test("SourceLocations") {
9797
}
9898

9999
expectDoesNotThrow({
100-
let file = try SourceFileSyntax.parse(url)
100+
let file = try SyntaxTreeParser.parse(url)
101101
Visitor(url: url, engine: engine).visit(file)
102102
})
103103

‎test/SwiftSyntax/ParseFile.swift

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,18 @@ class Test: NSObject {
3030
ParseFile.test("ParseSingleFile") {
3131
let currentFile = URL(fileURLWithPath: #file)
3232
expectDoesNotThrow({
33-
let currentFileContents = try String(contentsOf: currentFile)
34-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
35-
SwiftLang.parse(currentFile))
36-
expectEqual("\(parsed)", currentFileContents)
33+
let fileContents = try String(contentsOf: currentFile)
34+
let syntaxTreeData = try SwiftLang.parse(fileContents).data(using: .utf8)!
35+
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData)
36+
expectEqual("\(parsed)", fileContents)
3737
})
3838
}
3939

4040
ParseFile.test("ParseBuffer") {
4141
expectDoesNotThrow({
4242
let content = "func foo() {}"
43-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
44-
SwiftLang.parse(content))
43+
let syntaxTreeData = try SwiftLang.parse(content).data(using: .utf8)!
44+
let parsed = try SyntaxTreeDeserializer().deserialize(syntaxTreeData)
4545
expectEqual("\(parsed)", content)
4646
})
4747
}

‎test/SwiftSyntax/VisitorTest.swift

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ func getInput(_ file: String) -> URL {
1616
return result
1717
}
1818

19+
func getSyntaxTree(_ url: URL) throws -> SourceFileSyntax {
20+
let content = try SwiftLang.parse(url).data(using: .utf8)!
21+
return try SyntaxTreeDeserializer().deserialize(content)
22+
}
23+
1924
var VisitorTests = TestSuite("SyntaxVisitor")
2025

2126
VisitorTests.test("Basic") {
@@ -27,8 +32,7 @@ VisitorTests.test("Basic") {
2732
}
2833
}
2934
expectDoesNotThrow({
30-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
31-
SwiftLang.parse(getInput("visitor.swift")))
35+
let parsed = try getSyntaxTree(getInput("visitor.swift"))
3236
let counter = FuncCounter()
3337
let hashBefore = parsed.hashValue
3438
counter.visit(parsed)
@@ -45,8 +49,7 @@ VisitorTests.test("RewritingNodeWithEmptyChild") {
4549
}
4650
}
4751
expectDoesNotThrow({
48-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
49-
SwiftLang.parse(getInput("closure.swift")))
52+
let parsed = try getSyntaxTree(getInput("closure.swift"))
5053
let rewriter = ClosureRewriter()
5154
let rewritten = rewriter.visit(parsed)
5255
expectEqual(parsed.description, rewritten.description)
@@ -67,8 +70,7 @@ VisitorTests.test("SyntaxRewriter.visitAny") {
6770
}
6871
}
6972
expectDoesNotThrow({
70-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(try
71-
SwiftLang.parse(getInput("near-empty.swift")))
73+
let parsed = try getSyntaxTree(getInput("near-empty.swift"))
7274
let rewriter = VisitAnyRewriter(transform: { _ in
7375
return SyntaxFactory.makeIdentifier("")
7476
})
@@ -88,9 +90,7 @@ VisitorTests.test("SyntaxRewriter.visitCollection") {
8890
}
8991

9092
expectDoesNotThrow({
91-
let parsed = try SourceFileSyntax.decodeSourceFileSyntax(
92-
try SwiftLang.parse(getInput("nested-blocks.swift"))
93-
)
93+
let parsed = try getSyntaxTree(getInput("nested-blocks.swift"))
9494
let visitor = VisitCollections()
9595
visitor.visit(parsed)
9696
expectEqual(4, visitor.numberOfCodeBlockItems)

0 commit comments

Comments
 (0)