Merge pull request #1289 from ahoppen/ahoppen/assert-token-choices-preparation

Improve modeling of `Child` in CodeGeneration

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/Child.swift

@@ -10,28 +10,79 @@
 //
 //===----------------------------------------------------------------------===//
 
+/// The kind of token a node can contain. Either a token of a specific kind or a
+/// keyword with the given text.
+public enum TokenChoice {
+  case keyword(text: String)
+  case token(tokenKind: String)
+
+  public var isKeyword: Bool {
+    switch self {
+    case .keyword: return true
+    case .token: return false
+    }
+  }
+}
+
+public enum ChildKind {
+  /// The child always contains a node of the given `kind`.
+  case node(kind: String)
+  /// The child always contains a node that matches one of the `choices`.
+  case nodeChoices(choices: [Child])
+  /// The child is a collection of `kind`.
+  case collection(kind: String, collectionElementName: String)
+  /// The child is a token that matches one of the given `choices`.
+  case token(choices: [TokenChoice])
+  
+  public var isNodeChoices: Bool {
+    if case .nodeChoices = self {
+      return true
+    } else {
+      return false
+    }
+  }
+}
+
 /// A child of a node, that may be declared optional or a token with a
 /// restricted subset of acceptable kinds or texts.
 public class Child {
   public let name: String
-  public let syntaxKind: String
+  public let kind: ChildKind
   public let description: String?
-  public let collectionElementName: String?
   public let forceClassification: Bool
   public let isIndented: Bool
   public let requiresLeadingNewline: Bool
   public let isOptional: Bool
-  public let textChoices: [String]
-  public let nodeChoices: [Child]
   public let classification: SyntaxClassification?
-  /// A restricted set of token kinds that will be accepted for this child.
-  public let tokenChoices: [TokenSpec]
-  public let tokenCanContainArbitraryText: Bool
 
   public var swiftName: String {
     return lowercaseFirstWord(name: name)
   }
 
+  public var syntaxKind: String {
+    switch kind {
+    case .node(kind: let kind):
+      return kind
+    case .nodeChoices:
+      return "syntax"
+    case .collection(kind: let kind, collectionElementName: _):
+      return kind
+    case .token(choices: let choices):
+      if choices.count == 1 {
+        switch choices.first! {
+        case .keyword: return "KeywordToken"
+        case .token(tokenKind: let tokenKind): return tokenKind
+        }
+      } else {
+        if choices.allSatisfy({ $0.isKeyword }) {
+          return "KeywordToken"
+        } else {
+          return "Token"
+        }
+      }
+    }
+  }
+
   public var swiftSyntaxKind: String {
     return lowercaseFirstWord(name: syntaxKind)
   }
@@ -59,11 +110,6 @@ public class Child {
     return SYNTAX_TOKEN_MAP[tokenKind]
   }
 
-  /// Returns the first choice from the `tokenChoices` if there are any, otherwise returns `nil`.
-  public var mainToken: TokenSpec? {
-    return tokenChoices.first
-  }
-
   /// Whether this child has syntax kind `UnexpectedNodes`.
   public var isUnexpectedNodes: Bool {
     syntaxKind == "UnexpectedNodes"
@@ -75,47 +121,20 @@ public class Child {
   /// If force_classification is also set to true, all child nodes (not only
   /// identifiers) inherit the syntax classification.
   init(name: String,
-       kind: String,
+       kind: ChildKind,
        description: String? = nil,
        isOptional: Bool = false,
-       tokenChoices: [String] = [],
-       textChoices: [String] = [],
-       nodeChoices: [Child] = [],
-       collectionElementName: String? = nil,
        classification: String? = nil,
        forceClassification: Bool = false,
        isIndented: Bool = false,
        requiresLeadingNewline: Bool = false) {
     self.name = name
-    self.syntaxKind = kind
+    self.kind = kind
     self.description = description
-    self.collectionElementName = collectionElementName
     self.classification = classificationByName(classification)
     self.forceClassification = forceClassification
     self.isIndented = isIndented
     self.requiresLeadingNewline = requiresLeadingNewline
     self.isOptional = isOptional
-
-    self.tokenChoices = tokenChoices.compactMap { SYNTAX_TOKEN_MAP["\($0)Token"] }
-
-    // If mappedTokenChoices contains `nil`, the token can contain arbitrary text
-    self.tokenCanContainArbitraryText = self.tokenChoices.contains { $0.text == nil }
-    
-    // A list of valid text for tokens, if specified.
-    // This will force validation logic to check the text passed into the
-    // token against the choices.
-    self.textChoices = textChoices
-
-    // A list of valid choices for a child
-    self.nodeChoices = nodeChoices
-
-    // Check the choices are either empty or multiple
-    assert(nodeChoices.count != 1)
-
-    // Check node choices are well-formed
-    for choice in self.nodeChoices {
-      assert(!choice.isOptional, "node choice \(choice.name) cannot be optional")
-      assert(choice.nodeChoices.isEmpty, "node choice \(choice.name) cannot have further choices")
-    }
   }
 }

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -98,13 +98,15 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("available"),
   KeywordSpec("await"),
   KeywordSpec("before"),
+  KeywordSpec("block"),
   KeywordSpec("break", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("case", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("catch", isLexerClassified: true, requiresLeadingSpace: true),
   KeywordSpec("class", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("continue", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("convenience"),
   KeywordSpec("convention"),
+  KeywordSpec("cType"),
   KeywordSpec("default", isLexerClassified: true),
   KeywordSpec("defer", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("deinit", isLexerClassified: true, requiresTrailingSpace: true),
@@ -124,6 +126,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("extension", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("fallthrough", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("false", isLexerClassified: true),
+  KeywordSpec("file"),
   KeywordSpec("fileprivate", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("final"),
   KeywordSpec("for", isLexerClassified: true, requiresTrailingSpace: true),
@@ -145,7 +148,9 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("isolated"),
   KeywordSpec("kind"),
   KeywordSpec("lazy"),
+  KeywordSpec("left"),
   KeywordSpec("let", isLexerClassified: true, requiresTrailingSpace: true),
+  KeywordSpec("line"),
   KeywordSpec("lowerThan"),
   KeywordSpec("macro"),
   KeywordSpec("message"),

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -113,17 +113,15 @@ public class Node {
         return [
           Child(
             name: unexpectedName,
-            kind: "UnexpectedNodes",
-            isOptional: true,
-            collectionElementName: unexpectedName
+            kind: .collection(kind: "UnexpectedNodes", collectionElementName: unexpectedName),
+            isOptional: true
           ),
           child
         ]
-      } + (children.last != nil ? [Child(
+      } + (!children.isEmpty ? [Child(
         name: "UnexpectedAfter\(children.last!.name)",
-        kind: "UnexpectedNodes",
-        isOptional: true,
-        collectionElementName: "UnexpectedAfter\(children.last!.name)"
+        kind: .collection(kind: "UnexpectedNodes", collectionElementName: "UnexpectedAfter\(children.last!.name)"),
+        isOptional: true
       )] : [])
     }
 

CodeGeneration/Sources/SyntaxSupport/Traits.swift.gyb

@@ -37,7 +37,12 @@ public let TRAITS: [Trait] = [
 %   for child in trait.children:
 %     is_optional = ", isOptional: true" if child.is_optional else ""
 %     description = ", description: %s" % child.description if child.description else ""
-          Child(name: "${child.name}", kind: "${child.syntax_kind}"${is_optional}${description}),
+%     if child.syntax_kind.endswith('Token'):
+%       kind = f'.token(choices: [.token(tokenKind: "{child.syntax_kind}")])'
+%     else:
+%       kind = f'.node(kind: "{child.syntax_kind}")'
+%     end
+          Child(name: "${child.name}", kind: ${kind}${is_optional}${description}),
 %   end
   ${"],\n description: %s" % trait.description if trait.description else "      ]"}
   ),