Skip to content

Commit 2388b99

Browse files
ahoppenahoppen
committed
Simplify trivia reclassification logic
1 parent f611130 commit 2388b99

File tree

3 files changed

+41
-253
lines changed

3 files changed

+41
-253
lines changed

‎Sources/SwiftParser/StringLiterals.swift

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,15 @@ fileprivate class StringLiteralExpressionIndentationChecker {
9191
// MARK: - Post-process multi-line string literals
9292

9393
fileprivate extension SyntaxText {
94-
/// If the text ends with any newline character, return the trivia for that
95-
/// newline, otherwise `nil`.
96-
var newlineSuffix: RawTriviaPiece? {
94+
/// If the text ends with any newline character, return that character,
95+
/// otherwise `nil`.
96+
var newlineSuffix: SyntaxText? {
9797
if hasSuffix("\r\n") {
98-
return .carriageReturnLineFeeds(1)
98+
return "\r\n"
9999
} else if hasSuffix("\n") {
100-
return .newlines(1)
100+
return "\n"
101101
} else if hasSuffix("\r") {
102-
return .carriageReturns(1)
102+
return "\r"
103103
} else {
104104
return nil
105105
}
@@ -132,6 +132,25 @@ extension Parser {
132132
}
133133
}
134134

135+
private func reclassifyTrivia(
136+
in token: RawTokenSyntax,
137+
leading reclassifyLeading: SyntaxText = "",
138+
trailing reclassifyTrailing: SyntaxText = "",
139+
lexerError: LexerError? = nil
140+
) -> RawTokenSyntax {
141+
assert(SyntaxText(rebasing: token.tokenText.prefix(reclassifyLeading.count)) == reclassifyLeading)
142+
assert(SyntaxText(rebasing: token.tokenText.suffix(reclassifyTrailing.count)) == reclassifyTrailing)
143+
return RawTokenSyntax(
144+
kind: token.tokenKind,
145+
text: SyntaxText(rebasing: token.tokenText.dropFirst(reclassifyLeading.count).dropLast(reclassifyTrailing.count)),
146+
leadingTriviaPieces: token.leadingTriviaPieces + TriviaParser.parseTrivia(reclassifyLeading, position: .trailing),
147+
trailingTriviaPieces: TriviaParser.parseTrivia(reclassifyTrailing, position: .trailing) + token.trailingTriviaPieces,
148+
presence: token.presence,
149+
lexerError: token.tokenView.lexerError ?? lexerError,
150+
arena: self.arena
151+
)
152+
}
153+
135154
/// Re-classify the newline of the last line as trivia since the newline is
136155
/// not part of the represented string. If the last line has its newline
137156
/// escaped by a trailing `\`, mark that string segment as unexpected and
@@ -178,7 +197,7 @@ extension Parser {
178197
} else if let newlineSuffix = lastMiddleSegment.content.tokenText.newlineSuffix {
179198
// The newline at the end of the last line in the string literal is not part of the represented string.
180199
// Mark it as trivia.
181-
let content = lastMiddleSegment.content.reclassifyAsTrailingTrivia([newlineSuffix], arena: self.arena)
200+
let content = self.reclassifyTrivia(in: lastMiddleSegment.content, trailing: newlineSuffix)
182201

183202
middleSegments[middleSegments.count - 1] = .stringSegment(
184203
RawStringSegmentSyntax(
@@ -208,8 +227,7 @@ extension Parser {
208227
rawStringDelimitersToken: RawTokenSyntax?,
209228
middleSegments: inout [RawStringLiteralSegmentsSyntax.Element],
210229
isFirstSegmentOnNewLine: Bool,
211-
indentation: SyntaxText,
212-
indentationTrivia: [RawTriviaPiece]
230+
indentation: SyntaxText
213231
) {
214232
let expressionIndentationChecker = StringLiteralExpressionIndentationChecker(expectedIndentation: indentation, arena: self.arena)
215233

@@ -227,20 +245,16 @@ extension Parser {
227245
if segment.content.tokenText.hasPrefix(indentation) {
228246
segment = RawStringSegmentSyntax(
229247
segment.unexpectedBeforeContent,
230-
content: segment.content.reclassifyAsLeadingTrivia(indentationTrivia, arena: self.arena),
248+
content: self.reclassifyTrivia(in: segment.content, leading: indentation),
231249
segment.unexpectedAfterContent,
232250
arena: self.arena
233251
)
234252
} else if (segment.content.tokenText == "" || segment.content.tokenText.triviaPieceIfNewline != nil) && segment.content.trailingTriviaPieces.allSatisfy({ $0.isNewline }) {
235253
// Empty lines don't need to be indented and there's no indentation we need to strip away.
236254
} else {
237-
let actualIndentation = segment.content.tokenText.prefix(while: { $0 == UInt8(ascii: " ") || $0 == UInt8(ascii: "\t") })
238-
let actualIndentationTrivia = TriviaParser.parseTrivia(SyntaxText(rebasing: actualIndentation), position: .leading)
239-
let content = segment.content.reclassifyAsLeadingTrivia(
240-
actualIndentationTrivia,
241-
lexerError: LexerError(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0),
242-
arena: self.arena
243-
)
255+
let actualIndentation = SyntaxText(rebasing: segment.content.tokenText.prefix(while: { $0 == UInt8(ascii: " ") || $0 == UInt8(ascii: "\t") }))
256+
let lexerError = LexerError(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0)
257+
let content = self.reclassifyTrivia(in: segment.content, leading: actualIndentation, lexerError: lexerError)
244258
segment = RawStringSegmentSyntax(
245259
segment.unexpectedBeforeContent,
246260
content: content,
@@ -352,7 +366,15 @@ extension Parser {
352366
{
353367
indentationTrivia = parsedTrivia
354368
indentation = lastSegment.content.tokenText
355-
closeQuote = closeQuote.extendingLeadingTrivia(by: parsedTrivia, arena: self.arena)
369+
closeQuote = RawTokenSyntax(
370+
kind: closeQuote.tokenKind,
371+
text: closeQuote.tokenText,
372+
leadingTriviaPieces: parsedTrivia + closeQuote.leadingTriviaPieces,
373+
trailingTriviaPieces: closeQuote.trailingTriviaPieces,
374+
presence: closeQuote.presence,
375+
lexerError: closeQuote.tokenView.lexerError,
376+
arena: self.arena
377+
)
356378
} else {
357379
if let lastSegment = lastSegment {
358380
indentationTrivia = TriviaParser.parseTrivia(lastSegment.content.tokenText, position: .leading).prefix(while: { $0.isIndentationWhitespace })
@@ -386,8 +408,7 @@ extension Parser {
386408
rawStringDelimitersToken: rawStringDelimitersToken,
387409
middleSegments: &middleSegments,
388410
isFirstSegmentOnNewLine: openQuoteHasTrailingNewline,
389-
indentation: indentation,
390-
indentationTrivia: indentationTrivia
411+
indentation: indentation
391412
)
392413

393414
// -------------------------------------------------------------------------

‎Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

Lines changed: 0 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -249,175 +249,4 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
249249
arena: arena
250250
)
251251
}
252-
253-
/// Assuming that text representing `extendedTrivia` preceeds this token,
254-
/// return a token that has its leading trivia prepended by `extendedTrivia`.
255-
/// This can be used to transfer trivia from a preceeding token to this token.
256-
/// The caller is responsible to delete preceeding trivia from the tree to
257-
/// maintain source-fidelity.
258-
public func extendingLeadingTrivia(by extendedTrivia: [RawTriviaPiece], arena: SyntaxArena) -> RawTokenSyntax {
259-
let extendedTriviaByteLength = extendedTrivia.reduce(0, { $0 + $1.byteLength })
260-
switch raw.rawData.payload {
261-
case .parsedToken(let dat):
262-
assert(
263-
String(syntaxText: SyntaxText(baseAddress: dat.wholeText.baseAddress?.advanced(by: -extendedTriviaByteLength), count: extendedTriviaByteLength))
264-
== Trivia(pieces: extendedTrivia.map(TriviaPiece.init)).description
265-
)
266-
let wholeText = SyntaxText(baseAddress: dat.wholeText.baseAddress?.advanced(by: -extendedTriviaByteLength), count: dat.wholeText.count + extendedTriviaByteLength)
267-
let textRange = (dat.textRange.lowerBound + extendedTriviaByteLength)..<(dat.textRange.upperBound + extendedTriviaByteLength)
268-
return RawSyntax.parsedToken(
269-
kind: dat.tokenKind,
270-
wholeText: arena.intern(wholeText),
271-
textRange: textRange,
272-
presence: dat.presence,
273-
lexerError: dat.lexerError,
274-
arena: arena
275-
).as(RawTokenSyntax.self)!
276-
case .materializedToken(let dat):
277-
let triviaBuffer = arena.allocateRawTriviaPieceBuffer(count: dat.triviaPieces.count + extendedTrivia.count)
278-
let (_, extenedTriviaEndIndex) = triviaBuffer.initialize(from: extendedTrivia)
279-
let (_, triviaEndIndex) = triviaBuffer[extenedTriviaEndIndex...].initialize(from: dat.triviaPieces)
280-
assert(triviaEndIndex == triviaBuffer.endIndex)
281-
return RawSyntax.materializedToken(
282-
kind: dat.tokenKind,
283-
text: dat.tokenText,
284-
triviaPieces: RawTriviaPieceBuffer(triviaBuffer),
285-
numLeadingTrivia: dat.numLeadingTrivia + UInt32(extendedTrivia.count),
286-
byteLength: dat.byteLength + UInt32(extendedTriviaByteLength),
287-
presence: dat.presence,
288-
lexerError: dat.lexerError,
289-
arena: arena
290-
).as(RawTokenSyntax.self)!
291-
case .layout(_):
292-
preconditionFailure("Should be a token")
293-
}
294-
}
295-
296-
/// Assuming that text representing `extendedTrivia` comes after this token,
297-
/// return a token that has its trailing trivia appended by `extendedTrivia`.
298-
/// This can be used to transfer trivia from the next token to this token.
299-
/// The caller is responsible to delete succeeding trivia from the tree to
300-
/// maintain source-fidelity.
301-
public func extendingTrailingTrivia(by extendedTrivia: [RawTriviaPiece], arena: SyntaxArena) -> RawTokenSyntax {
302-
let extendedTriviaByteLength = extendedTrivia.reduce(0, { $0 + $1.byteLength })
303-
switch raw.rawData.payload {
304-
case .parsedToken(let dat):
305-
assert(
306-
String(syntaxText: SyntaxText(baseAddress: dat.wholeText.baseAddress?.advanced(by: dat.wholeText.count), count: extendedTriviaByteLength))
307-
== Trivia(pieces: extendedTrivia.map(TriviaPiece.init)).description
308-
)
309-
let wholeText = SyntaxText(baseAddress: dat.wholeText.baseAddress, count: dat.wholeText.count + extendedTriviaByteLength)
310-
return RawSyntax.parsedToken(
311-
kind: dat.tokenKind,
312-
wholeText: arena.intern(wholeText),
313-
textRange: dat.textRange,
314-
presence: dat.presence,
315-
lexerError: dat.lexerError,
316-
arena: arena
317-
).as(RawTokenSyntax.self)!
318-
case .materializedToken(let dat):
319-
let triviaBuffer = arena.allocateRawTriviaPieceBuffer(count: dat.triviaPieces.count + extendedTrivia.count)
320-
let (_, existingTriviaEndIndex) = triviaBuffer.initialize(from: dat.triviaPieces)
321-
let (_, triviaEndIndex) = triviaBuffer[existingTriviaEndIndex...].initialize(from: extendedTrivia)
322-
assert(triviaEndIndex == triviaBuffer.endIndex)
323-
return RawSyntax.materializedToken(
324-
kind: dat.tokenKind,
325-
text: dat.tokenText,
326-
triviaPieces: RawTriviaPieceBuffer(triviaBuffer),
327-
numLeadingTrivia: dat.numLeadingTrivia,
328-
byteLength: dat.byteLength + UInt32(extendedTriviaByteLength),
329-
presence: dat.presence,
330-
lexerError: dat.lexerError,
331-
arena: arena
332-
).as(RawTokenSyntax.self)!
333-
case .layout(_):
334-
preconditionFailure("Should be a token")
335-
}
336-
}
337-
338-
/// Assuming that the tokens tet starts with text representing `reclassifiedTrivia`,
339-
/// re-classify those characters as no longer being part of the token's text
340-
/// but as part of the token's leading trivia.
341-
/// If `error` is not `nil` and the token currently doesn't have a lexer error,
342-
/// that error will be set as the lexer error.
343-
public func reclassifyAsLeadingTrivia(_ reclassifiedTrivia: [RawTriviaPiece], lexerError: LexerError? = nil, arena: SyntaxArena) -> RawTokenSyntax {
344-
let reclassifiedTriviaByteLength = reclassifiedTrivia.reduce(0, { $0 + $1.byteLength })
345-
assert(
346-
String(syntaxText: SyntaxText(rebasing: self.tokenText[0..<reclassifiedTriviaByteLength]))
347-
== Trivia(pieces: reclassifiedTrivia.map(TriviaPiece.init)).description
348-
)
349-
switch raw.rawData.payload {
350-
case .parsedToken(let dat):
351-
let textRange = (dat.textRange.lowerBound + reclassifiedTriviaByteLength)..<dat.textRange.upperBound
352-
return RawSyntax.parsedToken(
353-
kind: dat.tokenKind,
354-
wholeText: dat.wholeText,
355-
textRange: textRange,
356-
presence: dat.presence,
357-
lexerError: dat.lexerError ?? lexerError,
358-
arena: arena
359-
).as(RawTokenSyntax.self)!
360-
case .materializedToken(let dat):
361-
let triviaBuffer = arena.allocateRawTriviaPieceBuffer(count: dat.triviaPieces.count + reclassifiedTrivia.count)
362-
let (_, existingLeadingTriviaEndIndex) = triviaBuffer.initialize(from: dat.leadingTrivia)
363-
let (_, reclassifiedTriviaEndIndex) = triviaBuffer[existingLeadingTriviaEndIndex...].initialize(from: reclassifiedTrivia)
364-
let (_, triviaEndIndex) = triviaBuffer[reclassifiedTriviaEndIndex...].initialize(from: dat.trailingTrivia)
365-
assert(triviaEndIndex == triviaBuffer.endIndex)
366-
return RawSyntax.materializedToken(
367-
kind: dat.tokenKind,
368-
text: SyntaxText(rebasing: dat.tokenText[reclassifiedTriviaByteLength...]),
369-
triviaPieces: RawTriviaPieceBuffer(triviaBuffer),
370-
numLeadingTrivia: dat.numLeadingTrivia + UInt32(reclassifiedTrivia.count),
371-
byteLength: dat.byteLength,
372-
presence: dat.presence,
373-
lexerError: dat.lexerError ?? lexerError,
374-
arena: arena
375-
).as(RawTokenSyntax.self)!
376-
case .layout(_):
377-
preconditionFailure("Should be a token")
378-
}
379-
}
380-
381-
/// Assuming that the tokens tet ends with text representing `reclassifiedTrivia`,
382-
/// re-classify those characters as no longer being part of the token's text
383-
/// but as part of the token's trailing trivia.
384-
/// If `error` is not `nil` and the token currently doesn't have a lexer error,
385-
/// that error will be set as the lexer error.
386-
public func reclassifyAsTrailingTrivia(_ reclassifiedTrivia: [RawTriviaPiece], lexerError: LexerError? = nil, arena: SyntaxArena) -> RawTokenSyntax {
387-
let reclassifiedTriviaByteLength = reclassifiedTrivia.reduce(0, { $0 + $1.byteLength })
388-
assert(
389-
String(syntaxText: SyntaxText(rebasing: self.tokenText[(self.tokenText.count - reclassifiedTriviaByteLength)...]))
390-
== Trivia(pieces: reclassifiedTrivia.map(TriviaPiece.init)).description
391-
)
392-
switch raw.rawData.payload {
393-
case .parsedToken(let dat):
394-
let textRange = dat.textRange.lowerBound..<(dat.textRange.upperBound - reclassifiedTriviaByteLength)
395-
return RawSyntax.parsedToken(
396-
kind: dat.tokenKind,
397-
wholeText: dat.wholeText,
398-
textRange: textRange,
399-
presence: dat.presence,
400-
lexerError: dat.lexerError ?? lexerError,
401-
arena: arena
402-
).as(RawTokenSyntax.self)!
403-
case .materializedToken(let dat):
404-
let triviaBuffer = arena.allocateRawTriviaPieceBuffer(count: dat.triviaPieces.count + reclassifiedTrivia.count)
405-
let (_, existingLeadingTriviaEndIndex) = triviaBuffer.initialize(from: dat.leadingTrivia)
406-
let (_, reclassifiedTriviaEndIndex) = triviaBuffer[existingLeadingTriviaEndIndex...].initialize(from: reclassifiedTrivia)
407-
let (_, triviaEndIndex) = triviaBuffer[reclassifiedTriviaEndIndex...].initialize(from: dat.trailingTrivia)
408-
assert(triviaEndIndex == triviaBuffer.endIndex)
409-
return RawSyntax.materializedToken(
410-
kind: dat.tokenKind,
411-
text: SyntaxText(rebasing: dat.tokenText[0..<(dat.tokenText.endIndex - reclassifiedTriviaByteLength)]),
412-
triviaPieces: RawTriviaPieceBuffer(triviaBuffer),
413-
numLeadingTrivia: dat.numLeadingTrivia,
414-
byteLength: dat.byteLength,
415-
presence: dat.presence,
416-
lexerError: dat.lexerError ?? lexerError,
417-
arena: arena
418-
).as(RawTokenSyntax.self)!
419-
case .layout(_):
420-
preconditionFailure("Should be a token")
421-
}
422-
}
423252
}

‎Tests/SwiftSyntaxTest/RawSyntaxTests.swift

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -152,66 +152,4 @@ final class RawSyntaxTests: XCTestCase {
152152
XCTAssertEqual(barIdent.description, "\nopen ")
153153
}
154154
}
155-
156-
func testExtendingTrivia() {
157-
// We are only testing materialized token here because parsed token should
158-
// be covered pretty well in the parser. Materialized tokens are less common
159-
// and need dedicated testing.
160-
withExtendedLifetime(SyntaxArena()) { arena in
161-
let baseToken = RawTokenSyntax(
162-
kind: .identifier,
163-
text: "hello",
164-
leadingTriviaPieces: [.spaces(2), .unexpectedText("a")],
165-
trailingTriviaPieces: [.tabs(3), .unexpectedText("b"), .tabs(1)],
166-
presence: .present,
167-
arena: arena
168-
)
169-
170-
let extendedLeading = baseToken.extendingLeadingTrivia(by: [.newlines(4)], arena: arena)
171-
XCTAssertEqual(extendedLeading.tokenKind, .identifier)
172-
XCTAssertEqual(extendedLeading.tokenText, "hello")
173-
XCTAssertEqual(extendedLeading.leadingTriviaPieces, [.newlines(4), .spaces(2), .unexpectedText("a")])
174-
XCTAssertEqual(extendedLeading.trailingTriviaPieces, [.tabs(3), .unexpectedText("b"), .tabs(1)])
175-
XCTAssertEqual(extendedLeading.presence, .present)
176-
XCTAssertEqual(extendedLeading.byteLength, baseToken.byteLength + 4)
177-
178-
let extendedTrailing = baseToken.extendingTrailingTrivia(by: [.newlines(4)], arena: arena)
179-
XCTAssertEqual(extendedTrailing.tokenKind, .identifier)
180-
XCTAssertEqual(extendedTrailing.tokenText, "hello")
181-
XCTAssertEqual(extendedTrailing.leadingTriviaPieces, [.spaces(2), .unexpectedText("a")])
182-
XCTAssertEqual(extendedTrailing.trailingTriviaPieces, [.tabs(3), .unexpectedText("b"), .tabs(1), .newlines(4)])
183-
XCTAssertEqual(extendedTrailing.presence, .present)
184-
XCTAssertEqual(extendedTrailing.byteLength, baseToken.byteLength + 4)
185-
}
186-
}
187-
188-
func testReclassifyingAsTrivia() {
189-
// We are only testing materialized token here because parsed token should
190-
// be covered pretty well in the parser. Materialized tokens are less common
191-
// and need dedicated testing.
192-
withExtendedLifetime(SyntaxArena()) { arena in
193-
let baseToken = RawTokenSyntax(
194-
kind: .identifier,
195-
text: " hello\r\n",
196-
leadingTriviaPieces: [.spaces(2), .unexpectedText("a")],
197-
trailingTriviaPieces: [.tabs(3), .unexpectedText("b"), .tabs(1)],
198-
presence: .present,
199-
arena: arena
200-
)
201-
202-
let reclassifiedLeading = baseToken.reclassifyAsLeadingTrivia([.spaces(2)], arena: arena)
203-
XCTAssertEqual(reclassifiedLeading.tokenKind, .identifier)
204-
XCTAssertEqual(reclassifiedLeading.tokenText, "hello\r\n")
205-
XCTAssertEqual(reclassifiedLeading.leadingTriviaPieces, [.spaces(2), .unexpectedText("a"), .spaces(2)])
206-
XCTAssertEqual(reclassifiedLeading.trailingTriviaPieces, [.tabs(3), .unexpectedText("b"), .tabs(1)])
207-
XCTAssertEqual(reclassifiedLeading.presence, .present)
208-
209-
let reclassifiedTrailing = baseToken.reclassifyAsTrailingTrivia([.carriageReturnLineFeeds(1)], arena: arena)
210-
XCTAssertEqual(reclassifiedTrailing.tokenKind, .identifier)
211-
XCTAssertEqual(reclassifiedTrailing.tokenText, " hello")
212-
XCTAssertEqual(reclassifiedTrailing.leadingTriviaPieces, [.spaces(2), .unexpectedText("a")])
213-
XCTAssertEqual(reclassifiedTrailing.trailingTriviaPieces, [.carriageReturnLineFeeds(1), .tabs(3), .unexpectedText("b"), .tabs(1)])
214-
XCTAssertEqual(reclassifiedTrailing.presence, .present)
215-
}
216-
}
217155
}

0 commit comments

Comments
 (0)