Refactor parsing of string template literals. (rescript-lang#94)

The previous approach put the scanner in "Template" mode when the parser
was going to parse template literals. This resulted in some very awkward
code; upon scanning a token there was logic checking whether we were
in "Template" mode. Every non-template literal token would pay for this
extra branch…

The new parsing strategy works differently: when the parser needs to
parse template literals, it just asks the scanner immediately for a
template literal token. This is both more performant an easier to reason
about. Template literals are a different language, it makes sense to
split this into a separate scanning function.

Author: IwanKaramazow

syntax/src/napkin_core.ml

@@ -2093,9 +2093,10 @@ and parseTemplateExpr ?(prefix="js") p =
     let op = Location.mknoloc (Longident.Lident "^") in
     Ast_helper.Exp.ident op
   in
-  let rec loop acc p =
+  let rec parseParts acc =
     let startPos = p.Parser.startPos in
-    match p.Parser.token with
+    Parser.nextTemplateLiteralToken p;
+    match p.token with
     | TemplateTail txt ->
       Parser.next p;
       let loc = mkLoc startPos p.prevEndPos in
@@ -2111,8 +2112,6 @@ and parseTemplateExpr ?(prefix="js") p =
       let loc = mkLoc startPos p.prevEndPos in
       let expr = parseExprBlock p in
       let fullLoc = mkLoc startPos p.prevEndPos in
-      Scanner.setTemplateMode p.scanner;
-      Parser.expect Rbrace p;
       let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
       let str = Ast_helper.Exp.constant ~loc (Pconst_string(txt, Some prefix)) in
       let next =
@@ -2123,27 +2122,23 @@ and parseTemplateExpr ?(prefix="js") p =
         Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator
           [Nolabel, a; Nolabel, expr]
       in
-      loop next p
-    | token ->
-      Parser.err p (Diagnostics.unexpected token p.breadcrumbs);
-      acc
+      parseParts next
+   | token ->
+     Parser.err p (Diagnostics.unexpected token p.breadcrumbs);
+     Ast_helper.Exp.constant (Pconst_string("", None))
   in
-  Scanner.setTemplateMode p.scanner;
-  Parser.expect Backtick p;
-  let startPos = p.Parser.startPos in
-  match p.Parser.token with
+  let startPos = p.startPos in
+  Parser.nextTemplateLiteralToken p;
+  match p.token with
   | TemplateTail txt ->
-    let loc = mkLoc startPos p.endPos in
     Parser.next p;
     let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
-    Ast_helper.Exp.constant ~loc (Pconst_string(txt, Some prefix))
+    Ast_helper.Exp.constant ~loc:(mkLoc startPos p.prevEndPos) (Pconst_string(txt, Some prefix))
   | TemplatePart txt ->
-    let constantLoc = mkLoc startPos p.endPos in
     Parser.next p;
+    let constantLoc = mkLoc startPos p.prevEndPos in
     let expr = parseExprBlock p in
     let fullLoc = mkLoc startPos p.prevEndPos in
-    Scanner.setTemplateMode p.scanner;
-    Parser.expect Rbrace p;
     let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
     let str = Ast_helper.Exp.constant ~loc:constantLoc (Pconst_string(txt, Some prefix)) in
     let next =
@@ -2152,7 +2147,7 @@ and parseTemplateExpr ?(prefix="js") p =
       else
         expr
     in
-    loop next p
+    parseParts next
  | token ->
    Parser.err p (Diagnostics.unexpected token p.breadcrumbs);
    Ast_helper.Exp.constant (Pconst_string("", None))
@@ -2643,7 +2638,7 @@ and parseBracedOrRecordExpr  p =
         Parser.expect Rbrace p;
         let loc = mkLoc startPos p.prevEndPos in
         let braces = makeBracesAttr loc in
-        {expr with pexp_attributes = braces::expr.pexp_attributes}
+        {expr with Parsetree.pexp_attributes = braces::expr.Parsetree.pexp_attributes}
       | Rbrace ->
         Parser.next p;
         let loc = mkLoc startPos p.prevEndPos in

syntax/src/napkin_parser.ml

@@ -62,6 +62,13 @@ let rec next ?prevEndPos p =
    p.startPos <- startPos;
    p.endPos <- endPos
 
+let nextTemplateLiteralToken p =
+  let (startPos, endPos, token) = Scanner.scanTemplateLiteralToken p.scanner in
+  p.token <- token;
+  p.prevEndPos <- p.endPos;
+  p.startPos <- startPos;
+  p.endPos <- endPos
+
 let checkProgress ~prevEndPos ~result p =
   if p.endPos == prevEndPos
   then None

syntax/src/napkin_parser.mli

@@ -28,6 +28,7 @@ val make: ?mode:mode -> ?line:int -> string -> string -> t
 val expect: ?grammar:Grammar.t -> Token.t -> t -> unit
 val optional: t -> Token.t -> bool
 val next: ?prevEndPos:Lexing.position -> t -> unit
+val nextTemplateLiteralToken: t -> unit
 val lookahead: t -> (t -> 'a) -> 'a
 val err:
   ?startPos:Lexing.position ->

syntax/src/napkin_scanner.ml

@@ -43,10 +43,6 @@ let inJsxMode scanner = match scanner.mode with
   | Jsx::_ -> true
   | _ -> false
 
-let inTemplateMode scanner = match scanner.mode with
-  | Template::_ -> true
-  | _ -> false
-
 let position scanner = Lexing.{
   pos_fname = scanner.filename;
   (* line number *)
@@ -405,43 +401,42 @@ let scanMultiLineComment scanner =
       (Bytes.sub_string scanner.src startOff (scanner.offset - 2 - startOff))
   )
 
-let scanTemplate scanner =
+let scanTemplateLiteralToken scanner =
   let startOff = scanner.offset in
+
+  (* if starting } here, consume it *)
+  if scanner.ch == CharacterCodes.rbrace then (
+    next scanner
+  );
   let startPos = position scanner in
 
   let rec scan () =
     if scanner.ch == CharacterCodes.eof then (
       let endPos = position scanner in
       scanner.err ~startPos ~endPos Diagnostics.unclosedTemplate;
-      popMode scanner Template;
       Token.TemplateTail(
-        Bytes.sub_string scanner.src startOff (scanner.offset - 2 - startOff)
+        Bytes.sub_string scanner.src startOff (scanner.offset - 1 - startOff)
       )
-    )
-    else if scanner.ch == CharacterCodes.backslash then (
+    ) else if scanner.ch == CharacterCodes.backtick then (
       next scanner;
-      if   scanner.ch == CharacterCodes.backtick
+      Token.TemplateTail(
+        Bytes.sub_string scanner.src startOff (scanner.offset - 1 - startOff)
+      )
+    ) else if scanner.ch == CharacterCodes.dollar &&
+              (peek scanner) == CharacterCodes.lbrace then (
+      next scanner; (* consume $ *)
+      next scanner; (* consume { *)
+      let contents =
+        Bytes.sub_string scanner.src startOff (scanner.offset - 2 - startOff)
+      in
+      Token.TemplatePart contents
+    ) else if scanner.ch == CharacterCodes.backslash then (
+      next scanner;
+      if scanner.ch == CharacterCodes.backtick
         || scanner.ch == CharacterCodes.backslash
         || scanner.ch == CharacterCodes.dollar
       then next scanner;
       scan()
-    ) else if scanner.ch == CharacterCodes.backtick then (
-      next scanner;
-      let contents =
-        Bytes.sub_string scanner.src startOff (scanner.offset - 1 - startOff)
-      in
-      popMode scanner Template;
-      Token.TemplateTail contents
-    ) else if scanner.ch == CharacterCodes.dollar &&
-              peek scanner == CharacterCodes.lbrace
-      then (
-        next scanner; (* consume $ *)
-        next scanner; (* consume { *)
-        let contents =
-          Bytes.sub_string scanner.src startOff (scanner.offset - 2 - startOff)
-        in
-        popMode scanner Template;
-        Token.TemplatePart contents
     ) else (
       if CharacterCodes.isLineBreak scanner.ch then (
         scanner.lineOffset <- scanner.offset + 1;
@@ -451,15 +446,15 @@ let scanTemplate scanner =
       scan()
     )
   in
-  scan()
+  let token = scan() in
+  let endPos = position scanner in
+  (startPos, endPos, token)
 
 let rec scan scanner =
-  if not (inTemplateMode scanner) then skipWhitespace scanner;
+  skipWhitespace scanner;
   let startPos = position scanner in
   let ch = scanner.ch in
-  let token = if inTemplateMode scanner then
-    scanTemplate scanner
-  else if ch == CharacterCodes.underscore then (
+  let token = if ch == CharacterCodes.underscore then (
     let nextCh = peek scanner in
     if nextCh == CharacterCodes.underscore || CharacterCodes.isDigit nextCh || CharacterCodes.isLetter nextCh then
       scanIdentifier scanner

syntax/src/napkin_scanner.mli

@@ -29,3 +29,5 @@ val setDiamondMode: t -> unit
 val popMode: t -> mode -> unit
 
 val reconsiderLessThan: t -> Napkin_token.t
+
+val scanTemplateLiteralToken: t -> (Lexing.position * Lexing.position * Napkin_token.t)

syntax/src/napkin_token.ml

@@ -1,4 +1,3 @@
-
 module Comment = Napkin_comment
 module CharacterCodes = Napkin_character_codes
 
@@ -221,4 +220,4 @@ let isKeywordTxt str =
   try let _ = keywordTable str in true with
   | Not_found -> false
 
-let catch = Lident "catch"
+let catch = Lident "catch"

syntax/tests/parsing/recovery/string/__snapshots__/parse.spec.js.snap

@@ -2,7 +2,7 @@
 
 exports[`eof.js 1`] = `"let x = \\"eof here\\""`;
 
-exports[`es6template.js 1`] = `"let x = ({js|this contains |js} ^ foo) ^ {js|, missing closin|js}"`;
+exports[`es6template.js 1`] = `"let x = ({js|this contains |js} ^ foo) ^ {js|, missing closing|js}"`;
 
 exports[`unclosed.js 1`] = `
 "let x = \\"unclosed\\"