Extract function for merging tokens

Author: fwcd

Sources/SourceKitLSP/DocumentManager.swift

@@ -24,11 +24,7 @@ public struct DocumentSnapshot {
 
   public var text: String { lineTable.content }
 
-  public var allTokens: [SemanticToken] {
-    // Only pick syntactic tokens for which no semantic token exists
-    let semaRanges = Set(semanticTokens.map(\.range))
-    return syntacticTokens.filter { !semaRanges.contains($0.range) } + semanticTokens
-  }
+  public var allTokens: [SemanticToken] { mergeSemanticTokens(syntacticTokens, semanticTokens) }
   public var sortedTokens: [SemanticToken] { allTokens.sorted { $0.start < $1.start } }
 
   public init(

Sources/SourceKitLSP/Swift/SemanticTokens.swift

@@ -204,6 +204,14 @@ public func decodeFromIntArray(rawSemanticTokens rawTokens: [UInt32]) -> [Semant
   return tokens
 }
 
+/// Merges the given token arrays into a new token array,
+/// preferring the second array's tokens if duplicate ranges are
+/// found.
+public func mergeSemanticTokens(_ lhs: [SemanticToken], _ rhs: [SemanticToken]) -> [SemanticToken] {
+  let rhsRanges = Set(rhs.map(\.range))
+  return lhs.filter { !rhsRanges.contains($0.range) } + rhs
+}
+
 /// Parses semantic tokens from sourcekitd response dictionaries.
 struct SemanticTokenParser {
   private let sourcekitd: SourceKitD