[SwiftSyntax] Simplify AbsolutePosition offset calculation and support columns

tools/SwiftSyntax/Diagnostic.swift

@@ -30,7 +30,6 @@ public struct SourceLocation: Codable {
   public let file: String
 
   public init(file: String, position: AbsolutePosition) {
-    assert(position is UTF8Position, "must be utf8 position")
     self.init(line: position.line, column: position.column,
               offset: position.byteOffset, file: file)
   }
@@ -88,7 +87,7 @@ public enum FixIt: Codable {
       let string = try container.decode(String.self, forKey: .string)
       let loc = try container.decode(SourceLocation.self, forKey: .location)
       self = .insert(loc, string)
-    case "replace": 
+    case "replace":
       let string = try container.decode(String.self, forKey: .string)
       let range = try container.decode(SourceRange.self, forKey: .range)
       self = .replace(range, string)
@@ -202,7 +201,7 @@ public struct Diagnostic: Codable {
   /// An array of possible FixIts to apply to this diagnostic.
   public let fixIts: [FixIt]
 
-  /// A diagnostic builder that 
+  /// A diagnostic builder that
   public struct Builder {
     /// An in-flight array of notes.
     internal var notes = [Note]()
@@ -225,7 +224,7 @@ public struct Diagnostic: Codable {
     ///   - fixIts: Any FixIts that should be attached to this note.
     public mutating func note(_ message: Message,
                               location: SourceLocation? = nil,
-                              highlights: [SourceRange] = [], 
+                              highlights: [SourceRange] = [],
                               fixIts: [FixIt] = []) {
       self.notes.append(Note(message: message, location: location,
                              highlights: highlights, fixIts: fixIts))
@@ -252,7 +251,7 @@ public struct Diagnostic: Codable {
 
     /// Adds a FixIt to replace the contents of the source file corresponding
     /// to the provided SourceRange with the provided text.
-    public mutating 
+    public mutating
     func fixItReplace(_ sourceRange: SourceRange, with text: String) {
       fixIts.append(.replace(sourceRange, text))
     }

tools/SwiftSyntax/SyntaxData.swift

@@ -65,10 +65,10 @@ final class SyntaxData: Equatable {
   }
 
   var position: AbsolutePosition {
-    return positionCache.value { return calculatePosition(UTF8Position()) }
+    return positionCache.value { return calculatePosition(AbsolutePosition()) }
   }
 
-  var positionAfterSkippingLeadingTrivia: AbsolutePosition {
+  var positionBeforeLeadingTrivia: AbsolutePosition {
     let result = position.copy()
     _ = raw.accumulateLeadingTrivia(result)
     return result
@@ -261,78 +261,57 @@ final class SyntaxData: Equatable {
 
 /// An absolute position in a source file as text - the absolute byteOffset from
 /// the start, line, and column.
-public class AbsolutePosition {
+public final class AbsolutePosition {
   public fileprivate(set) var byteOffset: Int
   public fileprivate(set) var line: Int
   public fileprivate(set) var column: Int
+  public let encoding: Encoding
 
-  required public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0) {
+  public enum Encoding {
+    case utf8
+    case utf16
+  }
+
+  public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0,
+              encoding: Encoding = .utf8) {
     self.line = line
     self.column = column
     self.byteOffset = byteOffset
+    self.encoding = encoding
   }
 
-  internal func add(text: String) {
-     preconditionFailure("this function must be overridden")
-  }
-
-  internal func copy() -> Self {
-    return type(of: self).init(line: line, column: column, byteOffset: byteOffset)
-  }
-}
-
-extension AbsolutePosition {
-
-  /// Add some number of columns to the position.
   internal func add(columns: Int) {
-    column += columns
-    byteOffset += columns
+    self.column += columns
   }
 
-  /// Add some number of newlines to the position, resetting the column.
-  /// Size is byte size of newline char.
-  /// '\n' and '\r' are 1, '\r\n' is 2.
   internal func add(lines: Int, size: Int) {
-    line += lines
-    column = 1
-    byteOffset += lines * size
+    self.line += lines * size
+    self.column = 1
   }
 
   /// Use some text as a reference for adding to the absolute position,
   /// taking note of newlines, etc.
-  fileprivate func add<C: BidirectionalCollection>(text chars: C)
-      where C.Element: UnsignedInteger  {
-    let cr: C.Element = 13
-    let nl: C.Element = 10
-    var idx = chars.startIndex
-    while idx != chars.endIndex {
-      let c = chars[idx]
-      idx = chars.index(after: idx)
-      switch c {
-      case cr:
-        if chars[idx] == nl {
-          add(lines: 1, size: 2)
-          idx = chars.index(after: idx)
-        } else {
-          add(lines: 1, size: 1)
-        }
-      case nl:
-        add(lines: 1, size: 1)
+  internal func add(text: String) {
+    for char in text {
+      switch char {
+      case "\n", "\r\n":
+        line += 1
+        column = 1
       default:
-        add(columns: 1)
+        column += 1
       }
-    }
-  }
-}
 
-class UTF8Position: AbsolutePosition {
-  internal override func add(text: String) {
-    add(text: text.utf8)
+      switch encoding {
+      case .utf8:
+        byteOffset += String(char).utf8.count
+      case .utf16:
+        byteOffset += String(char).utf16.count * 2
+      }
+    }
   }
-}
 
-class UTF16Position: AbsolutePosition {
-  internal override func add(text: String) {
-    add(text: text.utf16)
+  internal func copy() -> Self {
+    return type(of: self).init(line: line, column: column,
+      byteOffset: byteOffset)
   }
 }

tools/SwiftSyntax/Trivia.swift.gyb

@@ -99,59 +99,6 @@ extension TriviaPiece: TextOutputStreamable {
     case let .${trivia.lower_name}(text):
       target.write(text)
 %   end
-% end
-    }
-  }
-
-  /// Computes the information from this trivia to inform the source locations
-  /// of the associated tokens.
-  /// Specifically, walks through the trivia and keeps track of every newline
-  /// to give a number of how many newlines and UTF8 characters appear in the
-  /// trivia, along with the UTF8 offset of the last column.
-  func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) {
-    func calculateTextSizes(_ text: String) ->
-        (lines: Int, lastColumn: Int, utf8Length: Int) {
-      var lines = 0
-      var col = 0
-      var total = 0
-      var prevChar: UInt8? = nil
-      // TODO: CR + LF should be regarded as one newline
-      for char in text.utf8 {
-        total += 1
-        switch char {
-        case 0x0a:
-          if prevChar == 0x0d {
-            /* ASCII CR LF */
-            assert(col == 0)
-          } else {
-            /* ASCII newline */
-            col = 0
-            lines += 1
-          }
-        /* ASCII carriage-return */
-        case 0x0d:
-          col = 0
-          lines += 1
-
-        default:
-          col += 1
-        }
-        prevChar = char
-      }
-      return (lines: lines, lastColumn: col, utf8Length: total)
-    }
-    switch self {
-% for trivia in TRIVIAS:
-%   if trivia.is_new_line:
-    case let .${trivia.lower_name}s(n):
-      return (lines: n, lastColumn: 0, utf8Length: n * ${trivia.characters_len()})
-%   elif trivia.is_collection():
-    case let .${trivia.lower_name}s(n):
-      return (lines: 0, lastColumn: n, utf8Length: n * ${trivia.characters_len()})
-%   else:
-    case let .${trivia.lower_name}(text):
-      return calculateTextSizes(text)
-%   end
 % end
     }
   }
@@ -209,20 +156,6 @@ public struct Trivia: Codable {
     }
 %   end
 % end
-
-  /// Computes the total sizes and offsets of all pieces in this Trivia.
-  func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) {
-    var lines = 0
-    var lastColumn = 0
-    var length = 0
-    for piece in pieces {
-      let (ln, col, len) = piece.characterSizes()
-      lines += ln
-      lastColumn = col
-      length += len
-    }
-    return (lines: lines, lastColumn: lastColumn, utf8Length: length)
-  }
 }
 
 /// Conformance for Trivia to the Collection protocol.
@@ -245,7 +178,7 @@ extension Trivia: Collection {
 
   /// Get the byteSize of this trivia
   public var byteSize: Int {
-    let pos = UTF8Position()
+    let pos = AbsolutePosition()
     for piece in pieces {
       piece.accumulateAbsolutePosition(pos)
     }