Add documentation, some small renaming.

Author: kareman

Sources/Patterns/Decoder.swift

@@ -135,7 +135,7 @@ extension Parser.Match where Input == String {
 			var codingPath: [CodingKey] = []
 			@usableFromInline
 			var allKeys: [Key] {
-				matchDecoder.match.names.compactMap(Key.init(stringValue:))
+				matchDecoder.match.captureNames.compactMap(Key.init(stringValue:))
 			}
 
 			@usableFromInline

Sources/Patterns/Grammar.swift

@@ -5,10 +5,33 @@
 //  Created by Kåre Morstøl on 27/05/2020.
 //
 
+/// Allows for recursive patterns, also indirectly.
+///
+/// Define subpatterns using `<-`, like this arithmetic pattern:
+/// ```
+/// let g = Grammar { g in
+///   g.all <- g.expr • !any
+///   g.expr <- g.sum
+///   g.sum <- g.product • (("+" / "-") • g.product)*
+///   g.product <- g.power • (("*" / "/") • g.power)*
+///   g.power <- g.value • ("^" • g.power)¿
+///   g.value <- digit+ / "(" • g.expr • ")"
+/// }
+/// ```
+/// This recognises e.g. "1+2-3*(4+3)"
+///
+/// - warning: Does not support left recursion:
+///   ```
+///   g.a <- g.a • g.b
+///   ```
+///   will lead to infinite recursion.
 @dynamicMemberLookup
 public class Grammar: Pattern {
+	/// Calls another subpattern in a grammar.
 	public struct CallPattern: Pattern {
+		/// The grammar that contains the subpattern being called.
 		public let grammar: Grammar
+		/// The name of the subpattern being called.
 		public let name: String
 		public var description: String { "<\(name)>" }
 
@@ -26,8 +49,10 @@ public class Grammar: Pattern {
 
 	public var description: String { "Grammar" } // TODO:
 
+	/// All the subpatterns and their names.
 	public internal(set) var patterns: [(name: String, pattern: AnyPattern)] = []
 
+	/// The main subpattern, which will be called when this Grammar is being used.
 	public var firstPattern: String? { patterns.first?.name }
 
 	@inlinable
@@ -40,35 +65,41 @@ public class Grammar: Pattern {
 	}
 
 	@inlinable
+	/// Allows the use of e.g. `g.a` to refer to subpatterns.
 	public subscript(dynamicMember name: String) -> CallPattern {
 		CallPattern(grammar: self, name: name)
 	}
 
 	@inlinable
-	public func createInstructions(_ finalInstructions: inout Instructions) throws {
-		var instructions = finalInstructions
+	public func createInstructions(_ instructions: inout Instructions) throws {
+		// We begin with a call to the first subpattern, followed by a jump to the end.
+		// This enables this grammar to be used inside other patterns (including other grammars).
+
 		let startIndex = instructions.endIndex
 		instructions.append(
-			.openCall(name: try firstPattern ?? Parser<Input>.InitError.message("Grammar is empty.")))
+			.openCall(name: try firstPattern ?? Parser<Input>.PatternError.message("Grammar is empty.")))
 		instructions.append(.jump(offset: .max)) // replaced later
 		var callTable = [String: Range<Instructions.Index>]()
+
+		// Create instructions for all subpatterns. Store their positions in `callTable`.
 		for (name, pattern) in patterns {
 			let startIndex = instructions.endIndex
 			try pattern.createInstructions(&instructions)
 			instructions.append(.return)
 			guard (startIndex ..< instructions.endIndex).count > 1 else {
-				throw Parser<Input>.InitError.message("Pattern '\(name) <- \(pattern)' was empty.")
+				throw Parser<Input>.PatternError.message("Pattern '\(name) <- \(pattern)' was empty.")
 			}
 			callTable[name] = startIndex ..< instructions.endIndex
 		}
 
+		// Replace all `.openCall` with `.call(offset)` and the correct offsets.
 		for i in instructions.indices[startIndex...] {
 			if case let .openCall(name) = instructions[i] {
 				guard let subpatternRange = callTable[name] else {
-					throw Parser<Input>.InitError.message("Pattern '\(name)' was never defined with ´<-´ operator.")
+					throw Parser<Input>.PatternError.message("Pattern '\(name)' was never defined with ´<-´ operator.")
 				}
-				// If the last non-dummy (i.e. .choiceEnd) instruction in a subpattern is a call to itself we perform
-				// a tail call optimisation by jumping directly instead.
+				// If the last non-dummy (i.e. not .choiceEnd) instruction in a subpattern is a call to itself we
+				// perform a tail call optimisation by jumping directly instead.
 				// The very last instruction is a .return, so skip that.
 				if subpatternRange.upperBound - 2 == i
 					|| (subpatternRange.upperBound - 3 == i && instructions[i + 1].doesNotDoAnything) {
@@ -78,18 +109,19 @@ public class Grammar: Pattern {
 				}
 			}
 		}
-		instructions[startIndex + 1] = .jump(offset: instructions.endIndex - startIndex - 1)
 
-		finalInstructions = instructions
+		instructions[startIndex + 1] = .jump(offset: instructions.endIndex - startIndex - 1)
 	}
 }
 
 infix operator <-: AssignmentPrecedence
 
+/// Used by grammars to define subpatterns with `g.a <- ...`.
 public func <- <P: Pattern>(call: Grammar.CallPattern, pattern: P) {
 	call.grammar.patterns.append((call.name, AnyPattern(pattern)))
 }
 
+/// In case of `g.name <- Capture(...)`, names the nameless Capture "name".
 public func <- <P: Pattern>(call: Grammar.CallPattern, capture: Capture<P>) {
 	let newPattern = capture.name == nil
 		? Capture(name: call.name, capture.wrapped)

Sources/Patterns/Operations on Patterns/Skip.swift

@@ -5,6 +5,9 @@
 //  Created by Kåre Morstøl on 25/05/2020.
 //
 
+/// Skips 0 or more elements until a match for the next patterns are found.
+///
+/// If this is at the end of a pattern, it skips to the end of input.
 public struct Skip: Pattern {
 	public var description: String { "Skip()" }
 
@@ -109,7 +112,7 @@ extension MutableCollection where Self: RandomAccessCollection, Self: RangeRepla
 		}
 	}
 
-	/// Inserts new instructions at `location`. Adjusts the offsets of other instructions accordingly.
+	/// Inserts new instructions at `location`. Adjusts the offsets of the other instructions accordingly.
 	@usableFromInline
 	mutating func insertInstructions<Input>(_ newInstructions: Element..., at location: Index)
 		where Element == Instruction<Input> {

Sources/Patterns/Optimise Instructions.swift

@@ -27,9 +27,12 @@ private extension Instruction {
 import SE0270_RangeSet
 
 extension MutableCollection where Self: RandomAccessCollection, Index == Int {
+	/// Moves any `.checkIndex, .captureStart, .captureEnd` past any `.elementEquals, .checkElement`.
+	///
+	/// Improves performance noticeably.
 	@usableFromInline
 	mutating func moveMovablesForward<Input>() where Element == Instruction<Input> {
-		var movables = ContiguousArray<Index>()[...]
+		var movables = ContiguousArray<Index>()
 		for i in indices {
 			if self[i].isMovable {
 				movables.append(i)
@@ -52,7 +55,7 @@ extension MutableCollection where Self: RandomAccessCollection, Index == Int {
 				}
 				movables.removeAll()
 
-				// All `.checkIndex` should be first.
+				// All `.checkIndex` should be first. If they fail there is no point in capturing anything.
 				moveSubranges(checkIndexIndices, to: moved.lowerBound)
 			}
 		}

Sources/Patterns/Parser.swift

@@ -5,42 +5,47 @@
 //  Created by Kåre Morstøl on 23/10/2018.
 //
 
+/// Takes a pattern, optimises it and tries to match it over an input.
 public struct Parser<Input: BidirectionalCollection> where Input.Element: Hashable {
-	public enum InitError: Error, CustomStringConvertible {
-		case invalid([Pattern])
+	/// Indicates a problem with a malformed pattern.
+	public enum PatternError: Error, CustomStringConvertible {
+		/// The error message from the parser.
 		case message(String)
 
 		public var description: String {
 			switch self {
-			case let .invalid(patterns):
-				return "Invalid series of patterns: \(patterns)"
 			case let .message(string):
 				return string
 			}
 		}
 	}
 
 	@usableFromInline
-	let matcher: VMBacktrackEngine<Input>
+	let matcher: VMEngine<Input>
 
+	/// A parser which matches `pattern` _at_ a given position.
 	@inlinable
 	public init<P: Pattern>(_ pattern: P) throws where P.Input == Input {
-		self.matcher = try VMBacktrackEngine(pattern)
+		self.matcher = try VMEngine(pattern)
 	}
 
+	/// A parser which searches for `pattern` _from_ a given position.
+	///
+	/// Is the same as `Parser(Skip() • pattern)`.
 	@inlinable
 	public init<P: Pattern>(search pattern: P) throws where P.Input == Input {
 		try self.init(Skip() • pattern)
 	}
 
-	@inlinable
-	public func ranges(in input: Input, from startindex: Input.Index? = nil)
-		-> AnySequence<Range<Input.Index>> {
-		AnySequence(matches(in: input, from: startindex).lazy.map { $0.range })
-	}
-
+	/// Contains information about a patterns successfully completed match.
 	public struct Match: Equatable {
+		/// The position in the input when the pattern completed.
+		///
+		/// - note: If the last part of the pattern is a `!` or `&&`,
+		/// `endIndex` is the position when that last part _started_.
 		public let endIndex: Input.Index
+
+		/// The names and ranges of all captures.
 		public let captures: [(name: String?, range: Range<Input.Index>)]
 
 		@inlinable
@@ -57,6 +62,8 @@ public struct Parser<Input: BidirectionalCollection> where Input.Element: Hashab
 				})
 		}
 
+		/// The range from the beginning of the first capture to the end of the last one.
+		/// If there are no captures, the empty range at the `endIndex` of this Match.
 		@inlinable
 		public var range: Range<Input.Index> {
 			// TODO: Is `captures.last!.range.upperBound` always the highest captured index?
@@ -74,25 +81,40 @@ public struct Parser<Input: BidirectionalCollection> where Input.Element: Hashab
 			"""
 		}
 
+		/// Returns the first capture named `name`.
 		@inlinable
 		public subscript(one name: String) -> Range<Input.Index>? {
 			captures.first(where: { $0.name == name })?.range
 		}
 
+		/// Returns all captures named `name`.
 		@inlinable
 		public subscript(multiple name: String) -> [Range<Input.Index>] {
 			captures.filter { $0.name == name }.map { $0.range }
 		}
 
+		/// The names of all the captures.
 		@inlinable
-		public var names: Set<String> { Set(captures.compactMap { $0.name }) }
+		public var captureNames: Set<String> { Set(captures.compactMap { $0.name }) }
 	}
 
+	/// Tries to match the pattern in `input` at `index`.
+	/// - Parameters:
+	///   - index: The position to match at, if not provided the beginning of input will be used.
 	@inlinable
-	public func match(in input: Input, at startIndex: Input.Index? = nil) -> Match? {
-		matcher.match(in: input, from: startIndex ?? input.startIndex)
+	public func match(in input: Input, at index: Input.Index? = nil) -> Match? {
+		matcher.match(in: input, at: index ?? input.startIndex)
 	}
 
+	/// A lazily generated sequence of consecutive matches of the pattern in `input`.
+	///
+	/// Each match attempt starts at the `.range.upperBound` of the previous match,
+	/// so the matches can be overlapping.
+	///
+	/// You can dictate where the next match should start by where you place the last capture.
+	///
+	/// - Parameters:
+	///   - startindex: The position to match from, if not provided the beginning of input will be used.
 	@inlinable
 	public func matches(in input: Input, from startindex: Input.Index? = nil)
 		-> UnfoldSequence<Match, Input.Index> {
@@ -107,15 +129,15 @@ public struct Parser<Input: BidirectionalCollection> where Input.Element: Hashab
 				match = newMatch
 			}
 			lastMatch = match
-			let range = match.range
-			if range.upperBound == index {
-				guard range.upperBound != input.endIndex else {
+			let matchEnd = match.range.upperBound
+			if matchEnd == index {
+				guard matchEnd != input.endIndex else {
 					stop = true
 					return match
 				}
 				input.formIndex(after: &index)
 			} else {
-				index = range.upperBound
+				index = matchEnd
 			}
 			return match
 		})

Sources/Patterns/Pattern And Instruction.swift

@@ -70,12 +70,15 @@ public enum Instruction<Input: BidirectionalCollection> where Input.Element: Has
 
 	/// Will be replaced by .call in preprocessing. Is never executed.
 	case openCall(name: String)
-	/// Goes to the sub-expression at `offset` relative to this instruction.
+	/// Goes to the subpattern at `offset` relative to this instruction.
+	/// When the subpattern finishes we move on to the instruction after this.
 	case call(offset: Distance)
-	/// Returns from this subexpression to where it was called from.
+	/// Returns from this subpattern to the instruction after where this was called from.
 	case `return`
 
 	/// Signals a failure.
+	///
+	/// The snapshot from the previous `.choice` is restored, if there aren't any left we stop matching altogether.
 	case fail
 	/// A match has been successfully completed!
 	///

Sources/Patterns/Regex.swift

@@ -7,10 +7,14 @@
 
 import Foundation
 
-public protocol RegexConvertible {
+/// A pattern that can be converted to regex.
+public protocol RegexConvertible: Pattern {
+	/// The equivalent regex for this pattern.
 	var regex: String { get }
 }
 
+// For `OneOf` to be convertible the regex has to be provided manually when it is created.
+
 extension Literal: RegexConvertible {
 	public var regex: String { NSRegularExpression.escapedPattern(for: String(elements)) }
 }

Sources/Patterns/VMBacktrack.swift

@@ -6,7 +6,7 @@
 //
 
 @usableFromInline
-struct VMBacktrackEngine<Input: BidirectionalCollection> where Input.Element: Hashable {
+struct VMEngine<Input: BidirectionalCollection> where Input.Element: Hashable {
 	@usableFromInline
 	typealias Instructions = ContiguousArray<Instruction<Input>>
 	let instructions: Instructions
@@ -24,14 +24,14 @@ struct VMBacktrackEngine<Input: BidirectionalCollection> where Input.Element: Ha
 	}
 
 	@usableFromInline
-	func match(in input: Input, from startIndex: Input.Index) -> Parser<Input>.Match? {
+	func match(in input: Input, at startIndex: Input.Index) -> Parser<Input>.Match? {
 		launch(input: input, startIndex: startIndex)
 	}
 }
 
 extension Parser.Match {
 	@usableFromInline
-	init(_ thread: VMBacktrackEngine<Input>.Thread, instructions: VMBacktrackEngine<Input>.Instructions) {
+	init(_ thread: VMEngine<Input>.Thread, instructions: VMEngine<Input>.Instructions) {
 		var captures = [(name: String?, range: Range<Input.Index>)]()
 		captures.reserveCapacity(thread.captures.count / 2)
 		var captureBeginnings = [(name: String?, start: Input.Index)]()
@@ -53,7 +53,7 @@ extension Parser.Match {
 	}
 }
 
-extension VMBacktrackEngine {
+extension VMEngine {
 	@usableFromInline
 	struct Thread {
 		var instructionIndex: Instructions.Index

Tests/PatternsTests/TestHelpers.swift

@@ -17,6 +17,13 @@ extension Array where Element: Hashable {
 	}
 }
 
+extension Parser {
+	func ranges(in input: Input, from startindex: Input.Index? = nil)
+		-> AnySequence<Range<Input.Index>> {
+		AnySequence(matches(in: input, from: startindex).lazy.map { $0.range })
+	}
+}
+
 extension XCTestCase {
 	func assertParseAll(_ parser: Parser<String>, input: String, result: [String],
 	                    file: StaticString = #file, line: UInt = #line) {