[SwiftSyntax] Simplify AbsolutePosition offset calculation and support columns

test/SwiftSyntax/AbsolutePosition.swift

@@ -31,11 +31,11 @@ PositionTests.test("Visitor") {
     let content = try SwiftLang.parse(getInput("visitor.swift"))
     let source = try String(contentsOf: getInput("visitor.swift"))
     let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
-    expectEqual(parsed.position.byteOffset, 0)
-    expectEqual(parsed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset,
-                source.count)
-    expectEqual(parsed.position.byteOffset, 0)
-    expectEqual(parsed.byteSize, source.count)
+    expectEqual(0, parsed.position.utf8Offset)
+    expectEqual(source.count,
+      parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset)
+    expectEqual(0, parsed.position.utf8Offset)
+    expectEqual(source.count, parsed.byteSize)
   })
 }
 
@@ -44,10 +44,10 @@ PositionTests.test("Closure") {
     let content = try SwiftLang.parse(getInput("closure.swift"))
     let source = try String(contentsOf: getInput("closure.swift"))
     let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
-    expectEqual(parsed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset,
-                source.count)
-    expectEqual(parsed.position.byteOffset, 0)
-    expectEqual(parsed.byteSize, source.count)
+    expectEqual(source.count, 
+      parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset)
+    expectEqual(0, parsed.position.utf8Offset)
+    expectEqual(source.count, parsed.byteSize)
   })
 }
 
@@ -57,9 +57,9 @@ PositionTests.test("Rename") {
     let parsed = try SourceFileSyntax.decodeSourceFileSyntax(content)
     let renamed = FuncRenamer().visit(parsed) as! SourceFileSyntax
     let renamedSource = renamed.description
-    expectEqual(renamed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset,
-                renamedSource.count)
-    expectEqual(renamed.byteSize, renamedSource.count)
+    expectEqual(renamedSource.count, 
+      renamed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset)
+    expectEqual(renamedSource.count, renamed.byteSize)
   })
 }
 
@@ -74,8 +74,8 @@ PositionTests.test("CurrentFile") {
         _ = node.positionAfterSkippingLeadingTrivia
       }
       override func visit(_ node: TokenSyntax) {
-        expectEqual(node.position.byteOffset + node.leadingTrivia.byteSize,
-                    node.positionAfterSkippingLeadingTrivia.byteOffset)
+        expectEqual(node.positionAfterSkippingLeadingTrivia.utf8Offset,
+          node.position.utf8Offset + node.leadingTrivia.byteSize)
       }
     }
     Visitor().visit(parsed)
@@ -125,13 +125,14 @@ PositionTests.test("Trivias") {
   expectDoesNotThrow({
     let idx = 5
     let root = createSourceFile(idx + 1)
-    expectEqual(root.leadingTrivia!.count, 3)
-    expectEqual(root.trailingTrivia!.count, 0)
+    expectEqual(3, root.leadingTrivia!.count)
+    expectEqual(0, root.trailingTrivia!.count)
     let state = root.statements[idx]
-    expectEqual(state.leadingTrivia!.count, 3)
-    expectEqual(state.trailingTrivia!.count, 1)
-    expectEqual(state.leadingTrivia!.byteSize + state.trailingTrivia!.byteSize
-      + state.byteSizeAfterTrimmingTrivia, state.byteSize)
+    expectEqual(3, state.leadingTrivia!.count)
+    expectEqual(1, state.trailingTrivia!.count)
+    expectEqual(state.byteSize,
+      state.leadingTrivia!.byteSize + state.trailingTrivia!.byteSize
+        + state.byteSizeAfterTrimmingTrivia)
     expectFalse(root.statements.isImplicit)
   })
 }

tools/SwiftSyntax/AbsolutePosition.swift

@@ -0,0 +1,59 @@
+//===--------------- AbsolutePosition.swift - Source Positions ------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// An absolute position in a source file as text - the absolute utf8Offset from
+/// the start, line, and column.
+public final class AbsolutePosition {
+  public fileprivate(set) var utf8Offset: Int
+  public fileprivate(set) var line: Int
+  public fileprivate(set) var column: Int
+
+  public init(line: Int = 1, column: Int = 1, utf8Offset: Int = 0) {
+    self.line = line
+    self.column = column
+    self.utf8Offset = utf8Offset
+  }
+
+  internal func add(columns: Int) {
+    self.column += columns
+    self.utf8Offset += columns
+  }
+
+  internal func add(lines: Int, size: Int) {
+    self.line += lines * size
+    self.column = 1
+    self.utf8Offset += lines * size
+  }
+
+  /// Use some text as a reference for adding to the absolute position,
+  /// taking note of newlines, etc.
+  internal func add(text: String) {
+    for char in text {
+      switch char {
+      case "\n", "\r\n":
+        line += 1
+        column = 1
+      default:
+        column += 1
+      }
+
+      // FIXME: This is currently very wasteful, but should be fast once the
+      //        small-string optimization lands.
+      utf8Offset += String(char).utf8.count
+    }
+  }
+
+  internal func copy() -> AbsolutePosition {
+    return AbsolutePosition(line: line, column: column,
+      utf8Offset: utf8Offset)
+  }
+}

tools/SwiftSyntax/CMakeLists.txt

@@ -2,6 +2,7 @@ add_swift_library(swiftSwiftSyntax SHARED
   # This file should be listed the first.  Module name is inferred from the
   # filename.
   SwiftSyntax.swift
+  AbsolutePosition.swift
   AtomicCache.swift
   Diagnostic.swift
   DiagnosticConsumer.swift

tools/SwiftSyntax/Diagnostic.swift

@@ -30,9 +30,8 @@ public struct SourceLocation: Codable {
   public let file: String
 
   public init(file: String, position: AbsolutePosition) {
-    assert(position is UTF8Position, "must be utf8 position")
     self.init(line: position.line, column: position.column,
-              offset: position.byteOffset, file: file)
+              offset: position.utf8Offset, file: file)
   }
 
   public init(line: Int, column: Int, offset: Int, file: String) {
@@ -88,7 +87,7 @@ public enum FixIt: Codable {
       let string = try container.decode(String.self, forKey: .string)
       let loc = try container.decode(SourceLocation.self, forKey: .location)
       self = .insert(loc, string)
-    case "replace": 
+    case "replace":
       let string = try container.decode(String.self, forKey: .string)
       let range = try container.decode(SourceRange.self, forKey: .range)
       self = .replace(range, string)
@@ -202,7 +201,11 @@ public struct Diagnostic: Codable {
   /// An array of possible FixIts to apply to this diagnostic.
   public let fixIts: [FixIt]
 
-  /// A diagnostic builder that 
+  /// A diagnostic builder that exposes mutating operations for notes,
+  /// highlights, and FixIts. When a Diagnostic is created, a builder
+  /// will be provided in a closure where the user can conditionally
+  /// add notes, highlights, and FixIts, that will then be wrapped
+  /// into the immutable Diagnostic object.
   public struct Builder {
     /// An in-flight array of notes.
     internal var notes = [Note]()
@@ -225,7 +228,7 @@ public struct Diagnostic: Codable {
     ///   - fixIts: Any FixIts that should be attached to this note.
     public mutating func note(_ message: Message,
                               location: SourceLocation? = nil,
-                              highlights: [SourceRange] = [], 
+                              highlights: [SourceRange] = [],
                               fixIts: [FixIt] = []) {
       self.notes.append(Note(message: message, location: location,
                              highlights: highlights, fixIts: fixIts))
@@ -252,7 +255,7 @@ public struct Diagnostic: Codable {
 
     /// Adds a FixIt to replace the contents of the source file corresponding
     /// to the provided SourceRange with the provided text.
-    public mutating 
+    public mutating
     func fixItReplace(_ sourceRange: SourceRange, with text: String) {
       fixIts.append(.replace(sourceRange, text))
     }

tools/SwiftSyntax/SyntaxData.swift

@@ -65,7 +65,7 @@ final class SyntaxData: Equatable {
   }
 
   var position: AbsolutePosition {
-    return positionCache.value { return calculatePosition(UTF8Position()) }
+    return positionCache.value { return calculatePosition(AbsolutePosition()) }
   }
 
   var positionAfterSkippingLeadingTrivia: AbsolutePosition {
@@ -94,7 +94,7 @@ final class SyntaxData: Equatable {
   }
 
   var byteSize: Int {
-    return getNextSiblingPos().byteOffset - self.position.byteOffset
+    return getNextSiblingPos().utf8Offset - self.position.utf8Offset
   }
 
   /// Creates a SyntaxData with the provided raw syntax, pointing to the
@@ -258,81 +258,3 @@ final class SyntaxData: Equatable {
     return lhs === rhs
   }
 }
-
-/// An absolute position in a source file as text - the absolute byteOffset from
-/// the start, line, and column.
-public class AbsolutePosition {
-  public fileprivate(set) var byteOffset: Int
-  public fileprivate(set) var line: Int
-  public fileprivate(set) var column: Int
-
-  required public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0) {
-    self.line = line
-    self.column = column
-    self.byteOffset = byteOffset
-  }
-
-  internal func add(text: String) {
-     preconditionFailure("this function must be overridden")
-  }
-
-  internal func copy() -> Self {
-    return type(of: self).init(line: line, column: column, byteOffset: byteOffset)
-  }
-}
-
-extension AbsolutePosition {
-
-  /// Add some number of columns to the position.
-  internal func add(columns: Int) {
-    column += columns
-    byteOffset += columns
-  }
-
-  /// Add some number of newlines to the position, resetting the column.
-  /// Size is byte size of newline char.
-  /// '\n' and '\r' are 1, '\r\n' is 2.
-  internal func add(lines: Int, size: Int) {
-    line += lines
-    column = 1
-    byteOffset += lines * size
-  }
-
-  /// Use some text as a reference for adding to the absolute position,
-  /// taking note of newlines, etc.
-  fileprivate func add<C: BidirectionalCollection>(text chars: C)
-      where C.Element: UnsignedInteger  {
-    let cr: C.Element = 13
-    let nl: C.Element = 10
-    var idx = chars.startIndex
-    while idx != chars.endIndex {
-      let c = chars[idx]
-      idx = chars.index(after: idx)
-      switch c {
-      case cr:
-        if chars[idx] == nl {
-          add(lines: 1, size: 2)
-          idx = chars.index(after: idx)
-        } else {
-          add(lines: 1, size: 1)
-        }
-      case nl:
-        add(lines: 1, size: 1)
-      default:
-        add(columns: 1)
-      }
-    }
-  }
-}
-
-class UTF8Position: AbsolutePosition {
-  internal override func add(text: String) {
-    add(text: text.utf8)
-  }
-}
-
-class UTF16Position: AbsolutePosition {
-  internal override func add(text: String) {
-    add(text: text.utf16)
-  }
-}

tools/SwiftSyntax/Trivia.swift.gyb

@@ -99,59 +99,6 @@ extension TriviaPiece: TextOutputStreamable {
     case let .${trivia.lower_name}(text):
       target.write(text)
 %   end
-% end
-    }
-  }
-
-  /// Computes the information from this trivia to inform the source locations
-  /// of the associated tokens.
-  /// Specifically, walks through the trivia and keeps track of every newline
-  /// to give a number of how many newlines and UTF8 characters appear in the
-  /// trivia, along with the UTF8 offset of the last column.
-  func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) {
-    func calculateTextSizes(_ text: String) ->
-        (lines: Int, lastColumn: Int, utf8Length: Int) {
-      var lines = 0
-      var col = 0
-      var total = 0
-      var prevChar: UInt8? = nil
-      // TODO: CR + LF should be regarded as one newline
-      for char in text.utf8 {
-        total += 1
-        switch char {
-        case 0x0a:
-          if prevChar == 0x0d {
-            /* ASCII CR LF */
-            assert(col == 0)
-          } else {
-            /* ASCII newline */
-            col = 0
-            lines += 1
-          }
-        /* ASCII carriage-return */
-        case 0x0d:
-          col = 0
-          lines += 1
-
-        default:
-          col += 1
-        }
-        prevChar = char
-      }
-      return (lines: lines, lastColumn: col, utf8Length: total)
-    }
-    switch self {
-% for trivia in TRIVIAS:
-%   if trivia.is_new_line:
-    case let .${trivia.lower_name}s(n):
-      return (lines: n, lastColumn: 0, utf8Length: n * ${trivia.characters_len()})
-%   elif trivia.is_collection():
-    case let .${trivia.lower_name}s(n):
-      return (lines: 0, lastColumn: n, utf8Length: n * ${trivia.characters_len()})
-%   else:
-    case let .${trivia.lower_name}(text):
-      return calculateTextSizes(text)
-%   end
 % end
     }
   }
@@ -209,20 +156,6 @@ public struct Trivia: Codable {
     }
 %   end
 % end
-
-  /// Computes the total sizes and offsets of all pieces in this Trivia.
-  func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) {
-    var lines = 0
-    var lastColumn = 0
-    var length = 0
-    for piece in pieces {
-      let (ln, col, len) = piece.characterSizes()
-      lines += ln
-      lastColumn = col
-      length += len
-    }
-    return (lines: lines, lastColumn: lastColumn, utf8Length: length)
-  }
 }
 
 /// Conformance for Trivia to the Collection protocol.
@@ -245,11 +178,11 @@ extension Trivia: Collection {
 
   /// Get the byteSize of this trivia
   public var byteSize: Int {
-    let pos = UTF8Position()
+    let pos = AbsolutePosition()
     for piece in pieces {
       piece.accumulateAbsolutePosition(pos)
     }
-    return pos.byteOffset
+    return pos.utf8Offset
   }
 }
 
@@ -281,6 +214,6 @@ extension TriviaPiece {
       pos.add(text: text)
 %   end
 % end
-  }
+    }
   }
 }