Skip to content

Commit c98c16e

Browse files
authored
Merge pull request #857 from ahoppen/ahoppen/syntax-tree-manager
Store syntax trees in semantic token in syntax tree/semantic token managers instead of in the `Document`
2 parents 1a79393 + c1771de commit c98c16e

15 files changed

+387
-283
lines changed

Sources/SourceKitLSP/CMakeLists.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
add_library(SourceKitLSP STATIC
33
CapabilityRegistry.swift
44
DocumentManager.swift
5-
DocumentTokens.swift
65
IndexStoreDB+MainFilesProvider.swift
76
RangeAdjuster.swift
87
Sequence+AsyncMap.swift
@@ -25,11 +24,14 @@ target_sources(SourceKitLSP PRIVATE
2524
Swift/OpenInterface.swift
2625
Swift/SemanticRefactorCommand.swift
2726
Swift/SemanticRefactoring.swift
27+
Swift/SemanticTokens.swift
28+
Swift/SemanticTokensManager.swift
2829
Swift/SourceKitD+ResponseError.swift
2930
Swift/SwiftCommand.swift
3031
Swift/SwiftLanguageServer.swift
3132
Swift/SyntaxHighlightingToken.swift
3233
Swift/SyntaxHighlightingTokenParser.swift
34+
Swift/SyntaxTreeManager.swift
3335
Swift/VariableTypeInfo.swift)
3436
set_target_properties(SourceKitLSP PROPERTIES
3537
INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/SourceKitLSP/DocumentManager.swift

Lines changed: 58 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -15,27 +15,46 @@ import LanguageServerProtocol
1515
import LSPLogging
1616
import SKSupport
1717

18-
public struct DocumentSnapshot {
19-
public var document: Document
20-
public var version: Int
21-
public var lineTable: LineTable
22-
/// Syntax highlighting tokens for the document. Note that
23-
/// `uri` + `latestVersion` only uniquely identifies a snapshot's content,
24-
/// the tokens are updated independently and only used internally.
25-
public var tokens: DocumentTokens
18+
/// An immutable snapshot of a document at a given time.
19+
///
20+
/// ``DocumentSnapshot`` is always derived from a ``Document``. That is, the
21+
/// data structure that is stored internally by the ``DocumentManager`` is a
22+
/// ``Document``. The purpose of a ``DocumentSnapshot`` is to be able to work
23+
/// with one version of a document without having to think about it changing.
24+
public struct DocumentSnapshot: Identifiable {
25+
/// An ID that uniquely identifies the version of the document stored in this
26+
/// snapshot.
27+
public struct ID: Hashable, Comparable {
28+
public let uri: DocumentURI
29+
public let version: Int
30+
31+
/// Returns `true` if the snapshots reference the same document but rhs has a
32+
/// later version than `lhs`.
33+
///
34+
/// Snapshot IDs of different documents are not comparable to each other and
35+
/// will always return `false`.
36+
public static func < (lhs: DocumentSnapshot.ID, rhs: DocumentSnapshot.ID) -> Bool {
37+
return lhs.uri == rhs.uri && lhs.version < rhs.version
38+
}
39+
}
40+
41+
public let id: ID
42+
public let language: Language
43+
public let lineTable: LineTable
2644

45+
public var uri: DocumentURI { id.uri }
46+
public var version: Int { id.version }
2747
public var text: String { lineTable.content }
2848

2949
public init(
30-
document: Document,
50+
uri: DocumentURI,
51+
language: Language,
3152
version: Int,
32-
lineTable: LineTable,
33-
tokens: DocumentTokens
53+
lineTable: LineTable
3454
) {
35-
self.document = document
36-
self.version = version
55+
self.id = ID(uri: uri, version: version)
56+
self.language = language
3757
self.lineTable = lineTable
38-
self.tokens = tokens
3958
}
4059

4160
func index(of pos: Position) -> String.Index? {
@@ -48,23 +67,21 @@ public final class Document {
4867
public let language: Language
4968
var latestVersion: Int
5069
var latestLineTable: LineTable
51-
var latestTokens: DocumentTokens
5270

5371
init(uri: DocumentURI, language: Language, version: Int, text: String) {
5472
self.uri = uri
5573
self.language = language
5674
self.latestVersion = version
5775
self.latestLineTable = LineTable(text)
58-
self.latestTokens = DocumentTokens()
5976
}
6077

6178
/// **Not thread safe!** Use `DocumentManager.latestSnapshot` instead.
6279
fileprivate var latestSnapshot: DocumentSnapshot {
6380
DocumentSnapshot(
64-
document: self,
81+
uri: self.uri,
82+
language: self.language,
6583
version: latestVersion,
66-
lineTable: latestLineTable,
67-
tokens: latestTokens
84+
lineTable: latestLineTable
6885
)
6986
}
7087
}
@@ -118,28 +135,31 @@ public final class DocumentManager {
118135

119136
/// Applies the given edits to the document.
120137
///
121-
/// - parameter willEditDocument: Optional closure to call before each edit.
122-
/// - parameter updateDocumentTokens: Optional closure to call after each edit.
123-
/// - parameter before: The document contents *before* the edit is applied.
124-
/// - parameter after: The document contents *after* the edit is applied.
125-
/// - returns: The contents of the file after all the edits are applied.
126-
/// - throws: Error.missingDocument if the document is not open.
138+
/// - Parameters:
139+
/// - uri: The URI of the document to update
140+
/// - newVersion: The new version of the document. Must be greater than the
141+
/// latest version of the document.
142+
/// - edits: The edits to apply to the document
143+
/// - willEditDocument: Optional closure to call before each edit. Will be
144+
/// called multiple times if there are multiple edits.
145+
/// - Returns: The snapshot of the document before the edit and the snapshot
146+
/// of the document after the edit.
127147
@discardableResult
128148
public func edit(
129149
_ uri: DocumentURI,
130150
newVersion: Int,
131151
edits: [TextDocumentContentChangeEvent],
132-
willEditDocument: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
133-
updateDocumentTokens: ((_ after: DocumentSnapshot) -> DocumentTokens)? = nil
134-
) throws -> DocumentSnapshot {
152+
willEditDocument: ((_ before: LineTable, TextDocumentContentChangeEvent) -> Void)? = nil
153+
) throws -> (preEditSnapshot: DocumentSnapshot, postEditSnapshot: DocumentSnapshot) {
135154
return try queue.sync {
136155
guard let document = documents[uri] else {
137156
throw Error.missingDocument(uri)
138157
}
158+
let preEditSnapshot = document.latestSnapshot
139159

140160
for edit in edits {
141-
if let f = willEditDocument {
142-
f(document.latestSnapshot, edit)
161+
if let willEditDocument {
162+
willEditDocument(document.latestLineTable, edit)
143163
}
144164

145165
if let range = edit.range {
@@ -149,49 +169,17 @@ public final class DocumentManager {
149169
toLine: range.upperBound.line,
150170
utf16Offset: range.upperBound.utf16index,
151171
with: edit.text)
152-
153-
// Remove all tokens in the updated range and shift later ones.
154-
let rangeAdjuster = RangeAdjuster(edit: edit)!
155-
156-
document.latestTokens.semantic = document.latestTokens.semantic.compactMap {
157-
var token = $0
158-
if let adjustedRange = rangeAdjuster.adjust(token.range) {
159-
token.range = adjustedRange
160-
return token
161-
} else {
162-
return nil
163-
}
164-
}
165172
} else {
166173
// Full text replacement.
167174
document.latestLineTable = LineTable(edit.text)
168-
document.latestTokens = DocumentTokens()
169-
}
170-
171-
if let f = updateDocumentTokens {
172-
document.latestTokens = f(document.latestSnapshot)
173175
}
174176
}
175177

176-
document.latestVersion = newVersion
177-
return document.latestSnapshot
178-
}
179-
}
180-
181-
/// Updates the tokens in a document.
182-
///
183-
/// - parameter uri: The URI of the document to be updated
184-
/// - parameter tokens: The new tokens for the document
185-
@discardableResult
186-
public func updateTokens(_ uri: DocumentURI, tokens: DocumentTokens) throws -> DocumentSnapshot {
187-
return try queue.sync {
188-
guard let document = documents[uri] else {
189-
throw Error.missingDocument(uri)
178+
if newVersion <= document.latestVersion {
179+
log("Document version did not increase on edit from \(document.latestVersion) to \(newVersion)", level: .error)
190180
}
191-
192-
document.latestTokens = tokens
193-
194-
return document.latestSnapshot
181+
document.latestVersion = newVersion
182+
return (preEditSnapshot, document.latestSnapshot)
195183
}
196184
}
197185

@@ -230,16 +218,14 @@ extension DocumentManager {
230218
@discardableResult
231219
func edit(
232220
_ note: DidChangeTextDocumentNotification,
233-
willEditDocument: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
234-
updateDocumentTokens: ((_ after: DocumentSnapshot) -> DocumentTokens)? = nil
235-
) -> DocumentSnapshot? {
221+
willEditDocument: ((_ before: LineTable, TextDocumentContentChangeEvent) -> Void)? = nil
222+
) -> (preEditSnapshot: DocumentSnapshot, postEditSnapshot: DocumentSnapshot)? {
236223
return orLog("failed to edit document", level: .error) {
237-
try edit(
224+
return try edit(
238225
note.textDocument.uri,
239226
newVersion: note.textDocument.version,
240227
edits: note.contentChanges,
241-
willEditDocument: willEditDocument,
242-
updateDocumentTokens: updateDocumentTokens
228+
willEditDocument: willEditDocument
243229
)
244230
}
245231
}

Sources/SourceKitLSP/SourceKitServer.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ public actor SourceKitServer {
374374
await self.closeDocument(closeNotification, workspace: workspace)
375375

376376
let textDocument = TextDocumentItem(uri: documentUri,
377-
language: snapshot.document.language,
377+
language: snapshot.language,
378378
version: snapshot.version,
379379
text: snapshot.text)
380380
await self.openDocument(DidOpenTextDocumentNotification(textDocument: textDocument), workspace: workspace)
@@ -1146,7 +1146,7 @@ extension SourceKitServer {
11461146
if let newWorkspace = newWorkspace {
11471147
await self.openDocument(DidOpenTextDocumentNotification(textDocument: TextDocumentItem(
11481148
uri: docUri,
1149-
language: snapshot.document.language,
1149+
language: snapshot.language,
11501150
version: snapshot.version,
11511151
text: snapshot.text
11521152
)), workspace: newWorkspace)

Sources/SourceKitLSP/Swift/CodeCompletion.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ extension SwiftLanguageServer {
6363
log("triggerFromIncompleteCompletions with no existing completion session", level: .warning)
6464
throw ResponseError.serverCancelled
6565
}
66-
guard currentSession.uri == snapshot.document.uri, currentSession.utf8StartOffset == offset else {
67-
log("triggerFromIncompleteCompletions with incompatible completion session; expected \(currentSession.uri)@\(currentSession.utf8StartOffset), but got \(snapshot.document.uri)@\(offset)", level: .warning)
66+
guard currentSession.uri == snapshot.uri, currentSession.utf8StartOffset == offset else {
67+
log("triggerFromIncompleteCompletions with incompatible completion session; expected \(currentSession.uri)@\(currentSession.utf8StartOffset), but got \(snapshot.uri)@\(offset)", level: .warning)
6868
throw ResponseError.serverCancelled
6969
}
7070
session = currentSession
@@ -76,7 +76,7 @@ extension SwiftLanguageServer {
7676
snapshot: snapshot,
7777
utf8Offset: offset,
7878
position: completionPos,
79-
compileCommand: await buildSettings(for: snapshot.document.uri))
79+
compileCommand: await buildSettings(for: snapshot.uri))
8080

8181
await currentCompletionSession?.close()
8282
currentCompletionSession = session
@@ -95,15 +95,15 @@ extension SwiftLanguageServer {
9595
let skreq = SKDRequestDictionary(sourcekitd: sourcekitd)
9696
skreq[keys.request] = requests.codecomplete
9797
skreq[keys.offset] = offset
98-
skreq[keys.sourcefile] = snapshot.document.uri.pseudoPath
98+
skreq[keys.sourcefile] = snapshot.uri.pseudoPath
9999
skreq[keys.sourcetext] = snapshot.text
100100

101101
let skreqOptions = SKDRequestDictionary(sourcekitd: sourcekitd)
102102
skreqOptions[keys.codecomplete_sort_byname] = 1
103103
skreq[keys.codecomplete_options] = skreqOptions
104104

105105
// FIXME: SourceKit should probably cache this for us.
106-
if let compileCommand = await buildSettings(for: snapshot.document.uri) {
106+
if let compileCommand = await buildSettings(for: snapshot.uri) {
107107
skreq[keys.compilerargs] = compileCommand.compilerArgs
108108
}
109109

Sources/SourceKitLSP/Swift/CodeCompletionSession.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ actor CodeCompletionSession {
4141
case open
4242
}
4343

44-
nonisolated var uri: DocumentURI { snapshot.document.uri }
44+
nonisolated var uri: DocumentURI { snapshot.uri }
4545

4646
init(
4747
server: SwiftLanguageServer,
@@ -174,7 +174,7 @@ actor CodeCompletionSession {
174174
let keys = server.sourcekitd.keys
175175
req[keys.request] = server.sourcekitd.requests.codecomplete_close
176176
req[keys.offset] = self.utf8StartOffset
177-
req[keys.name] = self.snapshot.document.uri.pseudoPath
177+
req[keys.name] = self.snapshot.uri.pseudoPath
178178
log("\(Self.self) Closing: \(self)")
179179
_ = try? server.sourcekitd.sendSync(req)
180180
}

Sources/SourceKitLSP/Swift/CursorInfo.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ extension SwiftLanguageServer {
101101
if offsetRange.upperBound != offsetRange.lowerBound {
102102
skreq[keys.length] = offsetRange.count
103103
}
104-
skreq[keys.sourcefile] = snapshot.document.uri.pseudoPath
104+
skreq[keys.sourcefile] = snapshot.uri.pseudoPath
105105

106106
// FIXME: SourceKit should probably cache this for us.
107107
if let compileCommand = await self.buildSettings(for: uri) {

Sources/SourceKitLSP/Swift/Diagnostic.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ extension CodeAction {
6565
title: title,
6666
kind: .quickFix,
6767
diagnostics: nil,
68-
edit: WorkspaceEdit(changes: [snapshot.document.uri:edits]))
68+
edit: WorkspaceEdit(changes: [snapshot.uri:edits]))
6969
}
7070

7171
/// Describe a fixit's edit briefly.
@@ -257,7 +257,7 @@ extension DiagnosticRelatedInformation {
257257
}
258258

259259
self.init(
260-
location: Location(uri: snapshot.document.uri, range: Range(position!)),
260+
location: Location(uri: snapshot.uri, range: Range(position!)),
261261
message: message,
262262
codeActions: actions)
263263
}

Sources/SourceKitLSP/Swift/ExpressionTypeInfo.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ extension SwiftLanguageServer {
6666

6767
let skreq = SKDRequestDictionary(sourcekitd: sourcekitd)
6868
skreq[keys.request] = requests.expression_type
69-
skreq[keys.sourcefile] = snapshot.document.uri.pseudoPath
69+
skreq[keys.sourcefile] = snapshot.uri.pseudoPath
7070

7171
// FIXME: SourceKit should probably cache this for us.
7272
if let compileCommand = await self.buildSettings(for: uri) {

Sources/SourceKitLSP/Swift/SemanticRefactoring.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ struct SemanticRefactoring {
7474
}
7575

7676
self.title = title
77-
self.edit = WorkspaceEdit(changes: [snapshot.document.uri: textEdits])
77+
self.edit = WorkspaceEdit(changes: [snapshot.uri: textEdits])
7878
}
7979
}
8080

@@ -155,7 +155,7 @@ extension SwiftLanguageServer {
155155
skreq[keys.actionuid] = self.sourcekitd.api.uid_get_from_cstr(refactorCommand.actionString)!
156156

157157
// FIXME: SourceKit should probably cache this for us.
158-
if let compileCommand = await self.buildSettings(for: snapshot.document.uri) {
158+
if let compileCommand = await self.buildSettings(for: snapshot.uri) {
159159
skreq[keys.compilerargs] = compileCommand.compilerArgs
160160
}
161161

0 commit comments

Comments
 (0)