Implement named backreferences

Use the CaptureList as the source of truth on
which index a name corresponds to, and query it
when emitting a named backreference.

Author: hamishknight

Sources/_RegexParser/Regex/Parse/Diagnostics.swift

@@ -86,6 +86,7 @@ enum ParseError: Error, Hashable {
   case unsupported(String)
   case deprecatedUnicode(String)
   case invalidReference(Int)
+  case invalidNamedReference(String)
   case duplicateNamedCapture(String)
   case invalidCharacterClassRangeOperand
   case invalidQuantifierRange(Int, Int)
@@ -211,6 +212,8 @@ extension ParseError: CustomStringConvertible {
       return "\(kind) is a deprecated Unicode property, and is not supported"
     case let .invalidReference(i):
       return "no capture numbered \(i)"
+    case let .invalidNamedReference(name):
+      return "no capture named '\(name)'"
     case let .duplicateNamedCapture(str):
       return "group named '\(str)' already exists"
     case let .invalidQuantifierRange(lhs, rhs):

Sources/_RegexParser/Regex/Parse/Sema.swift

@@ -72,20 +72,20 @@ extension RegexValidator {
   }
 
   func validateReference(_ ref: AST.Reference) throws {
+    if let recLevel = ref.recursionLevel {
+      throw error(.unsupported("recursion level"), at: recLevel.location)
+    }
     switch ref.kind {
     case .absolute(let i):
       guard i <= captures.captures.count else {
         throw error(.invalidReference(i), at: ref.innerLoc)
       }
+    case .named(let name):
+      guard captures.captures.contains(where: { $0.name == name }) else {
+        throw error(.invalidNamedReference(name), at: ref.innerLoc)
+      }
     case .relative:
       throw error(.unsupported("relative capture reference"), at: ref.innerLoc)
-    case .named:
-      // TODO: This could be implemented by querying the capture list for an
-      // index.
-      throw error(.unsupported("named capture reference"), at: ref.innerLoc)
-    }
-    if let recLevel = ref.recursionLevel {
-      throw error(.unsupported("recursion level"), at: recLevel.location)
     }
   }
 

Sources/_StringProcessing/ByteCodeGen.swift

@@ -5,6 +5,11 @@ extension Compiler {
     var options: MatchingOptions
     var builder = Program.Builder()
 
+    init(options: MatchingOptions, captureList: CaptureList) {
+      self.options = options
+      self.builder.captureList = captureList
+    }
+
     mutating func finish(
     ) throws -> Program {
       builder.buildAccept()
@@ -62,7 +67,9 @@ extension Compiler.ByteCodeGen {
     case .absolute(let i):
       // Backreferences number starting at 1
       builder.buildBackreference(.init(i-1))
-    case .relative, .named:
+    case .named(let name):
+      try builder.buildNamedReference(name)
+    case .relative:
       throw Unsupported("Backreference kind: \(ref)")
     }
   }

Sources/_StringProcessing/Compiler.swift

@@ -27,8 +27,9 @@ class Compiler {
 
   __consuming func emit() throws -> Program {
     // TODO: Handle global options
-    var codegen = ByteCodeGen(options: options)
-    codegen.builder.captureList = tree.root._captureList
+    var codegen = ByteCodeGen(
+      options: options, captureList: tree.root._captureList
+    )
     try codegen.emitNode(tree.root)
     let program = try codegen.finish()
     return program

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -38,13 +38,24 @@ extension MEProgram where Input.Element: Hashable {
     // Special addresses or instructions
     var failAddressToken: AddressToken? = nil
 
-    var captureList = CaptureList()
+    private(set) var namedCaptureOffsets: [String: Int] = [:]
+
+    var captureList = CaptureList() {
+      didSet {
+        // Named references are guaranteed to be unique for literal ASTs by
+        // Sema. The DSL tree does not use named references.
+        for (i, capture) in captureList.captures.enumerated() {
+          guard let name = capture.name else { continue }
+          namedCaptureOffsets[name] = i
+        }
+      }
+    }
     var initialOptions = MatchingOptions()
 
     // Symbolic reference resolution
     var unresolvedReferences: [ReferenceID: [InstructionAddress]] = [:]
     var referencedCaptureOffsets: [ReferenceID: Int] = [:]
-    var namedCaptureOffsets: [String: Int] = [:]
+
     var captureCount: Int {
       // We currently deduce the capture count from the capture register number.
       nextCaptureRegister.rawValue
@@ -284,6 +295,13 @@ extension MEProgram.Builder {
     unresolvedReferences[id, default: []].append(lastInstructionAddress)
   }
 
+  mutating func buildNamedReference(_ name: String) throws {
+    guard let index = namedCaptureOffsets[name] else {
+      throw RegexCompilationError.uncapturedReference
+    }
+    buildBackreference(.init(index))
+  }
+
   // TODO: Mutating because of fail address fixup, drop when
   // that's removed
   mutating func assemble() throws -> MEProgram {
@@ -456,9 +474,9 @@ extension MEProgram.Builder {
       assert(preexistingValue == nil)
     }
     if let name = name {
-      // TODO: Reject duplicate capture names unless `(?J)`?
-      namedCaptureOffsets.updateValue(captureCount, forKey: name)
+      assert(namedCaptureOffsets[name] == nextCaptureRegister.rawValue)
     }
+    assert(nextCaptureRegister.rawValue < captureList.captures.count)
     return nextCaptureRegister
   }
 

Tests/RegexTests/MatchTests.swift

@@ -1142,23 +1142,32 @@ extension RegexTests {
   }
 
   func testMatchReferences() {
-    // TODO: Implement backreference/subpattern matching.
     firstMatchTest(
       #"(.)\1"#,
       input: "112", match: "11")
     firstMatchTest(
       #"(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)\10"#,
       input: "aaaaaaaaabbc", match: "aaaaaaaaabb")
 
+    firstMatchTest(
+      #"(.)(.)(.)(.)(.)(.)(.)(.)(.)(?<a1>.)(?P=a1)"#,
+      input: "aaaaaaaaabbc", match: "aaaaaaaaabb")
+
     firstMatchTest(
       #"(.)\g001"#,
       input: "112", match: "11")
 
-    firstMatchTest(#"(.)(.)\g-02"#, input: "abac", match: "aba", xfail: true)
-    firstMatchTest(#"(?<a>.)(.)\k<a>"#, input: "abac", match: "aba", xfail: true)
-    firstMatchTest(#"\g'+2'(.)(.)"#, input: "abac", match: "aba", xfail: true)
+    firstMatchTest(#"(?<a>.)(.)\k<a>"#, input: "abac", match: "aba")
+
+    firstMatchTest(#"(?<a>.)(?<b>.)(?<c>.)\k<c>\k<a>\k<b>"#,
+                   input: "xyzzxy", match: "xyzzxy")
 
     firstMatchTest(#"\1(.)"#, input: "112", match: nil)
+    firstMatchTest(#"\k<a>(?<a>.)"#, input: "112", match: nil)
+
+    // TODO: Implement subpattern matching.
+    firstMatchTest(#"(.)(.)\g-02"#, input: "abac", match: "aba", xfail: true)
+    firstMatchTest(#"\g'+2'(.)(.)"#, input: "abac", match: "aba", xfail: true)
   }
 
   func testMatchExamples() {

Tests/RegexTests/ParseTests.swift

@@ -1242,16 +1242,37 @@ extension RegexTests {
     parseTest(#"\k'-3'"#, backreference(.relative(-3)), throwsError: .unsupported)
     parseTest(#"\k'1'"#, backreference(.absolute(1)), throwsError: .invalid)
 
-    parseTest(#"\k{a0}"#, backreference(.named("a0")), throwsError: .unsupported)
-    parseTest(#"\k<bc>"#, backreference(.named("bc")), throwsError: .unsupported)
-    parseTest(#"\g{abc}"#, backreference(.named("abc")), throwsError: .unsupported)
-    parseTest(#"(?P=abc)"#, backreference(.named("abc")), throwsError: .unsupported)
+    parseTest(
+      #"(?<a>)\k<a>"#, concat(
+        namedCapture("a", empty()), backreference(.named("a"))
+      ), captures: [.named("a")]
+    )
+    parseTest(
+      #"(?<a>)\k{a}"#, concat(
+        namedCapture("a", empty()), backreference(.named("a"))
+      ), captures: [.named("a")]
+    )
+    parseTest(
+      #"(?<a>)\g{a}"#, concat(
+        namedCapture("a", empty()), backreference(.named("a"))
+      ), captures: [.named("a")]
+    )
+    parseTest(
+      #"(?<a>)(?P=a)"#, concat(
+        namedCapture("a", empty()), backreference(.named("a"))
+      ), captures: [.named("a")]
+    )
+
+    parseTest(#"\k{a0}"#, backreference(.named("a0")), throwsError: .invalid)
+    parseTest(#"\k<bc>"#, backreference(.named("bc")), throwsError: .invalid)
+    parseTest(#"\g{abc}"#, backreference(.named("abc")), throwsError: .invalid)
+    parseTest(#"(?P=abc)"#, backreference(.named("abc")), throwsError: .invalid)
 
     // Oniguruma recursion levels.
     parseTest(#"\k<bc-0>"#, backreference(.named("bc"), recursionLevel: 0), throwsError: .unsupported)
     parseTest(#"\k<a+0>"#, backreference(.named("a"), recursionLevel: 0), throwsError: .unsupported)
-    parseTest(#"\k<1+1>"#, backreference(.absolute(1), recursionLevel: 1), throwsError: .invalid)
-    parseTest(#"\k<3-8>"#, backreference(.absolute(3), recursionLevel: -8), throwsError: .invalid)
+    parseTest(#"\k<1+1>"#, backreference(.absolute(1), recursionLevel: 1), throwsError: .unsupported)
+    parseTest(#"\k<3-8>"#, backreference(.absolute(3), recursionLevel: -8), throwsError: .unsupported)
     parseTest(#"\k'-3-8'"#, backreference(.relative(-3), recursionLevel: -8), throwsError: .unsupported)
     parseTest(#"\k'bc-8'"#, backreference(.named("bc"), recursionLevel: -8), throwsError: .unsupported)
     parseTest(#"\k'+3-8'"#, backreference(.relative(3), recursionLevel: -8), throwsError: .unsupported)
@@ -2167,7 +2188,7 @@ extension RegexTests {
       throwsError: .unsupported
     )
     parseWithDelimitersTest(
-      #"re'a\k'b0A''"#, concat("a", backreference(.named("b0A"))), throwsError: .unsupported)
+      #"re'a\k'b0A''"#, concat("a", backreference(.named("b0A"))), throwsError: .invalid)
     parseWithDelimitersTest(
       #"re'\k'+2-1''"#, backreference(.relative(2), recursionLevel: -1),
       throwsError: .unsupported
@@ -2811,6 +2832,12 @@ extension RegexTests {
     diagnosticTest(#"(?:)()\2"#, .invalidReference(2))
     diagnosticTest(#"(?:)(?:)\2"#, .invalidReference(2))
 
+    diagnosticTest(#"\k<a>"#, .invalidNamedReference("a"))
+    diagnosticTest(#"(?:)\k<a>"#, .invalidNamedReference("a"))
+    diagnosticTest(#"()\k<a>"#, .invalidNamedReference("a"))
+    diagnosticTest(#"()\k<a>()"#, .invalidNamedReference("a"))
+    diagnosticTest(#"(?<b>)\k<a>()"#, .invalidNamedReference("a"))
+
     // MARK: Conditionals
 
     diagnosticTest(#"(?(1)a|b|c)"#, .tooManyBranchesInConditional(3))