Skip to content

Commit 486a86c

Browse files
kimdvkimdv
authored
Merge pull request #1197 from kimdv/kimdv/move-syntax-rewriter-from-gyb-to-code-gen
2 parents 37d0027 + 2d357b1 commit 486a86c

File tree

7 files changed

+4753
-3660
lines changed

7 files changed

+4753
-3660
lines changed

‎CodeGeneration/Sources/Utils/SyntaxBuildableNode.swift

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@ public extension Node {
2222

2323
/// The node's syntax kind as `SyntaxBuildableType`.
2424
var baseType: SyntaxBuildableType {
25-
SyntaxBuildableType(syntaxKind: baseKind)
25+
if baseKind == "SyntaxCollection" {
26+
return SyntaxBuildableType(syntaxKind: "Syntax")
27+
} else {
28+
return SyntaxBuildableType(syntaxKind: baseKind)
29+
}
30+
2631
}
2732

2833
/// If documentation exists for this node, return it as a single-line string.

‎CodeGeneration/Sources/generate-swiftsyntax/GenerateSwiftSyntax.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
6060
TemplateSpec(sourceFile: syntaxCollectionsFile, module: swiftSyntaxDir, filename: "SyntaxCollections.swift"),
6161
TemplateSpec(sourceFile: syntaxEnumFile, module: swiftSyntaxDir, filename: "SyntaxEnum.swift"),
6262
TemplateSpec(sourceFile: syntaxKindFile, module: swiftSyntaxDir, filename: "SyntaxKind.swift"),
63+
TemplateSpec(sourceFile: syntaxRewriterFile, module: swiftSyntaxDir, filename: "SyntaxRewriter.swift"),
6364
TemplateSpec(sourceFile: syntaxTraitsFile, module: swiftSyntaxDir, filename: "SyntaxTraits.swift"),
6465
TemplateSpec(sourceFile: syntaxTransformFile, module: swiftSyntaxDir, filename: "SyntaxTransform.swift"),
6566
TemplateSpec(sourceFile: syntaxVisitorFile, module: swiftSyntaxDir, filename: "SyntaxVisitor.swift"),

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxRewriterFile.swift

Lines changed: 309 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,309 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import SwiftSyntax
14+
import SwiftSyntaxBuilder
15+
import SyntaxSupport
16+
import Utils
17+
18+
let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: .docLineComment(generateCopyrightHeader(for: "generate-swiftsyntax"))) {
19+
ClassDeclSyntax("""
20+
//
21+
// This file defines the SyntaxRewriter, a class that performs a standard walk
22+
// and tree-rebuilding pattern.
23+
//
24+
// Subclassers of this class can override the walking behavior for any syntax
25+
// node and transform nodes however they like.
26+
//
27+
//===----------------------------------------------------------------------===//
28+
29+
open class SyntaxRewriter
30+
""") {
31+
InitializerDeclSyntax("public init() {}")
32+
33+
for node in SYNTAX_NODES where node.isVisitable {
34+
if node.baseType.baseName == "Syntax" && node.name != "MissingSyntax" {
35+
FunctionDeclSyntax("""
36+
/// Visit a `\(raw: node.name)`.
37+
/// - Parameter node: the node that is being visited
38+
/// - Returns: the rewritten node
39+
open func visit(_ node: \(raw: node.name)) -> \(raw: node.name) {
40+
return Syntax(visitChildren(node)).cast(\(raw: node.name).self)
41+
}
42+
""")
43+
} else {
44+
FunctionDeclSyntax("""
45+
/// Visit a `\(raw: node.name)`.
46+
/// - Parameter node: the node that is being visited
47+
/// - Returns: the rewritten node
48+
open func visit(_ node: \(raw: node.name)) -> \(raw: node.baseType.syntaxBaseName) {
49+
return \(raw: node.baseType.syntaxBaseName)(visitChildren(node))
50+
}
51+
""")
52+
}
53+
}
54+
55+
FunctionDeclSyntax("""
56+
/// Visit a `TokenSyntax`.
57+
/// - Parameter node: the node that is being visited
58+
/// - Returns: the rewritten node
59+
open func visit(_ token: TokenSyntax) -> TokenSyntax {
60+
return token
61+
}
62+
""")
63+
64+
FunctionDeclSyntax("""
65+
/// The function called before visiting the node and its descendents.
66+
/// - node: the node we are about to visit.
67+
open func visitPre(_ node: Syntax) {}
68+
""")
69+
70+
FunctionDeclSyntax("""
71+
/// Override point to choose custom visitation dispatch instead of the
72+
/// specialized `visit(_:)` methods. Use this instead of those methods if
73+
/// you intend to dynamically dispatch rewriting behavior.
74+
/// - note: If this method returns a non-nil result, the specialized
75+
/// `visit(_:)` methods will not be called for this node and the
76+
/// visited node will be replaced by the returned node in the
77+
/// rewritten tree.
78+
open func visitAny(_ node: Syntax) -> Syntax? {
79+
return nil
80+
}
81+
""")
82+
83+
FunctionDeclSyntax("""
84+
/// The function called after visiting the node and its descendents.
85+
/// - node: the node we just finished visiting.
86+
open func visitPost(_ node: Syntax) {}
87+
""")
88+
89+
FunctionDeclSyntax("""
90+
/// Visit any Syntax node.
91+
/// - Parameter node: the node that is being visited
92+
/// - Returns: the rewritten node
93+
public func visit(_ node: Syntax) -> Syntax {
94+
return visit(node.data)
95+
}
96+
""")
97+
98+
FunctionDeclSyntax("""
99+
public func visit<T: SyntaxChildChoices>(_ node: T) -> T {
100+
return visit(Syntax(node)).cast(T.self)
101+
}
102+
""")
103+
104+
for baseKind in SYNTAX_BASE_KINDS.filter { !["Syntax", "SyntaxCollection"].contains($0) } {
105+
FunctionDeclSyntax("""
106+
/// Visit any \(raw: baseKind)Syntax node.
107+
/// - Parameter node: the node that is being visited
108+
/// - Returns: the rewritten node
109+
public func visit(_ node: \(raw: baseKind)Syntax) -> \(raw: baseKind)Syntax {
110+
return visit(node.data).cast(\(raw: baseKind)Syntax.self)
111+
}
112+
""")
113+
}
114+
115+
for node in NON_BASE_SYNTAX_NODES {
116+
FunctionDeclSyntax("""
117+
/// Implementation detail of visit(_:). Do not call directly.
118+
private func visitImpl\(raw: node.name)(_ data: SyntaxData) -> Syntax {
119+
let node = \(raw: node.name)(data)
120+
// Accessing _syntaxNode directly is faster than calling Syntax(node)
121+
visitPre(node._syntaxNode)
122+
defer { visitPost(node._syntaxNode) }
123+
if let newNode = visitAny(node._syntaxNode) { return newNode }
124+
return Syntax(visit(node))
125+
}
126+
""")
127+
}
128+
129+
FunctionDeclSyntax("""
130+
/// Implementation detail of visit(_:). Do not call directly.
131+
private func visitImplTokenSyntax(_ data: SyntaxData) -> Syntax {
132+
let node = TokenSyntax(data)
133+
// Accessing _syntaxNode directly is faster than calling Syntax(node)
134+
visitPre(node._syntaxNode)
135+
defer { visitPost(node._syntaxNode) }
136+
if let newNode = visitAny(node._syntaxNode) { return newNode }
137+
return Syntax(visit(node))
138+
}
139+
""")
140+
141+
IfConfigDeclSyntax(
142+
leadingTrivia: [
143+
.blockComment("// SwiftSyntax requires a lot of stack space in debug builds for syntax tree"),
144+
.newlines(1),
145+
.blockComment("// rewriting. In scenarios with reduced stack space (in particular dispatch"),
146+
.newlines(1),
147+
.blockComment("// queues), this easily results in a stack overflow. To work around this issue,"),
148+
.newlines(1),
149+
.blockComment("// use a less performant but also less stack-hungry version of SwiftSyntax's"),
150+
.newlines(1),
151+
.blockComment("// SyntaxRewriter in debug builds."),
152+
.newlines(1)
153+
],
154+
clauses: IfConfigClauseListSyntax {
155+
IfConfigClauseSyntax(
156+
poundKeyword: .poundIfKeyword(),
157+
condition: ExprSyntax("DEBUG"),
158+
elements: .statements(CodeBlockItemListSyntax {
159+
FunctionDeclSyntax("""
160+
/// Implementation detail of visit(_:). Do not call directly.
161+
///
162+
/// Returns the function that shall be called to visit a specific syntax node.
163+
///
164+
/// To determine the correct specific visitation function for a syntax node,
165+
/// we need to switch through a huge switch statement that covers all syntax
166+
/// types. In debug builds, the cases of this switch statement do not share
167+
/// stack space (rdar://55929175). Because of this, the switch statement
168+
/// requires allocates about 15KB of stack space. In scenarios with reduced
169+
/// stack size (in particular dispatch queues), this often results in a stack
170+
/// overflow during syntax tree rewriting.
171+
///
172+
/// To circumvent this problem, make calling the specific visitation function
173+
/// a two-step process: First determine the function to call in this function
174+
/// and return a reference to it, then call it. This way, the stack frame
175+
/// that determines the correct visitiation function will be popped of the
176+
/// stack before the function is being called, making the switch's stack
177+
/// space transient instead of having it linger in the call stack.
178+
private func visitationFunc(for data: SyntaxData) -> ((SyntaxData) -> Syntax)
179+
""") {
180+
SwitchStmtSyntax(expression: ExprSyntax("data.raw.kind")) {
181+
SwitchCaseSyntax("case .token:") {
182+
ReturnStmtSyntax("return visitImplTokenSyntax")
183+
}
184+
185+
for node in NON_BASE_SYNTAX_NODES {
186+
SwitchCaseSyntax("case .\(raw: node.swiftSyntaxKind):") {
187+
ReturnStmtSyntax("return visitImpl\(raw: node.name)")
188+
}
189+
}
190+
}
191+
}
192+
193+
FunctionDeclSyntax("""
194+
private func visit(_ data: SyntaxData) -> Syntax {
195+
return visitationFunc(for: data)(data)
196+
}
197+
""")
198+
})
199+
)
200+
IfConfigClauseSyntax(
201+
poundKeyword: .poundElseKeyword(leadingTrivia: .newline),
202+
elements: .statements(CodeBlockItemListSyntax {
203+
FunctionDeclSyntax("private func visit(_ data: SyntaxData) -> Syntax") {
204+
SwitchStmtSyntax(expression: ExprSyntax("data.raw.kind")) {
205+
SwitchCaseSyntax("case .token:") {
206+
ReturnStmtSyntax("return visitImplTokenSyntax(data)")
207+
}
208+
209+
for node in NON_BASE_SYNTAX_NODES {
210+
SwitchCaseSyntax("case .\(raw: node.swiftSyntaxKind):") {
211+
ReturnStmtSyntax("return visitImpl\(raw: node.name)(data)")
212+
}
213+
}
214+
}
215+
}
216+
})
217+
)
218+
},
219+
poundEndif: .poundEndifKeyword(leadingTrivia: .newline)
220+
)
221+
222+
FunctionDeclSyntax("""
223+
private func visitChildren<SyntaxType: SyntaxProtocol>(
224+
_ node: SyntaxType
225+
) -> SyntaxType {
226+
// Walk over all children of this node and rewrite them. Don't store any
227+
// rewritten nodes until the first non-`nil` value is encountered. When this
228+
// happens, retrieve all previous syntax nodes from the parent node to
229+
// initialize the new layout. Once we know that we have to rewrite the
230+
// layout, we need to collect all futher children, regardless of whether
231+
// they are rewritten or not.
232+
233+
// newLayout is nil until the first child node is rewritten and rewritten
234+
// nodes are being collected.
235+
var newLayout: ContiguousArray<RawSyntax?>?
236+
237+
// Rewritten children just to keep their 'SyntaxArena' alive until they are
238+
// wrapped with 'Syntax'
239+
var rewrittens: ContiguousArray<Syntax> = []
240+
241+
let syntaxNode = node._syntaxNode
242+
243+
// Incrementing i manually is faster than using .enumerated()
244+
var childIndex = 0
245+
for (raw, info) in RawSyntaxChildren(syntaxNode) {
246+
defer { childIndex += 1 }
247+
guard let child = raw else {
248+
// Node does not exist. If we are collecting rewritten nodes, we need to
249+
// collect this one as well, otherwise we can ignore it.
250+
if newLayout != nil {
251+
newLayout!.append(nil)
252+
}
253+
continue
254+
}
255+
256+
// Build the Syntax node to rewrite
257+
let absoluteRaw = AbsoluteRawSyntax(raw: child, info: info)
258+
let data = SyntaxData(absoluteRaw, parent: syntaxNode)
259+
260+
let rewritten = visit(data)
261+
if rewritten.data.nodeId != info.nodeId {
262+
// The node was rewritten, let's handle it
263+
if newLayout == nil {
264+
// We have not yet collected any previous rewritten nodes. Initialize
265+
// the new layout with the previous nodes of the parent. This is
266+
// possible, since we know they were not rewritten.
267+
268+
// The below implementation is based on Collection.map but directly
269+
// reserves enough capacity for the entire layout.
270+
newLayout = ContiguousArray<RawSyntax?>()
271+
newLayout!.reserveCapacity(node.raw.layoutView!.children.count)
272+
for j in 0..<childIndex {
273+
newLayout!.append(node.raw.layoutView!.children[j])
274+
}
275+
}
276+
277+
// Now that we know we have a new layout in which we collect rewritten
278+
// nodes, add it.
279+
rewrittens.append(rewritten)
280+
newLayout!.append(rewritten.raw)
281+
} else {
282+
// The node was not changed by the rewriter. Only store it if a previous
283+
// node has been rewritten and we are collecting a rewritten layout.
284+
if newLayout != nil {
285+
newLayout!.append(raw)
286+
}
287+
}
288+
}
289+
290+
if let newLayout = newLayout {
291+
// A child node was rewritten. Build the updated node.
292+
293+
// Sanity check, ensure the new children are the same length.
294+
assert(newLayout.count == node.raw.layoutView!.children.count)
295+
296+
let arena = SyntaxArena()
297+
let newRaw = node.raw.layoutView!.replacingLayout(with: Array(newLayout), arena: arena)
298+
// 'withExtendedLifetime' to keep 'SyntaxArena's of them alive until here.
299+
return withExtendedLifetime((arena, rewrittens)) {
300+
Syntax(raw: newRaw).cast(SyntaxType.self)
301+
}
302+
} else {
303+
// No child node was rewritten. So no need to change this node as well.
304+
return node
305+
}
306+
}
307+
""")
308+
}
309+
}

‎Package.swift

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ let package = Package(
6767
"Raw/RawSyntaxValidation.swift.gyb",
6868
"SyntaxFactory.swift.gyb",
6969
"SyntaxNodes.swift.gyb.template",
70-
"SyntaxRewriter.swift.gyb",
7170
"Trivia.swift.gyb",
7271
],
7372
swiftSettings: swiftSyntaxSwiftSettings

‎Sources/SwiftSyntax/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ add_swift_host_library(SwiftSyntax
4040
generated/SyntaxEnum.swift
4141
gyb_generated/SyntaxFactory.swift
4242
generated/SyntaxKind.swift
43-
gyb_generated/SyntaxRewriter.swift
43+
generated/SyntaxRewriter.swift
4444
generated/SyntaxTraits.swift
4545
generated/SyntaxTransform.swift
4646
generated/SyntaxVisitor.swift

0 commit comments

Comments
 (0)