Improve skipping on syntax errors

Simplify and improve the algorithm used for skipping tokens when a syntax
error is encountered. The new algorithm is based on the region stack maintained
in Scanner instead of maintaining separate counters for open parentheses. This
improves the precision of skipping in general, since there's is no chance that
the two datastructures get out of sync. It seems overall we produce fewer follow
on error after a syntax error that way.

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -37,14 +37,6 @@ object Parsers {
 
   case class OpInfo(operand: Tree, operator: Ident, offset: Offset)
 
-  class ParensCounters {
-    private var parCounts = new Array[Int](lastParen - firstParen)
-
-    def count(tok: Token): Int = parCounts(tok - firstParen)
-    def change(tok: Token, delta: Int): Unit = parCounts(tok - firstParen) += delta
-    def nonePositive: Boolean = parCounts forall (_ <= 0)
-  }
-
   enum Location(val inParens: Boolean, val inPattern: Boolean, val inArgs: Boolean):
     case InParens      extends Location(true, false, false)
     case InArgs        extends Location(true, false, true)
@@ -173,8 +165,6 @@ object Parsers {
 
     val in: Scanner = new Scanner(source)
 
-    val openParens: ParensCounters = new ParensCounters
-
     /** This is the general parse entry point.
      *  Overridden by ScriptParser
      */
@@ -261,55 +251,14 @@ object Parsers {
     }
 
     /** Skip on error to next safe point.
-     *  Safe points are:
-     *   - Closing braces, provided they match an opening brace before the error point.
-     *   - Closing parens and brackets, provided they match an opening parent or bracket
-     *     before the error point and there are no intervening other kinds of parens.
-     *   - Semicolons and newlines, provided there are no intervening braces.
-     *   - Definite statement starts on new lines, provided they are not more indented
-     *     than the last known statement start before the error point.
-     */
-    protected def skip(): Unit = {
-      val skippedParens = new ParensCounters
-      while (true) {
-        (in.token: @switch) match {
-          case EOF =>
-            return
-          case SEMI | NEWLINE | NEWLINES =>
-            if (skippedParens.count(LBRACE) == 0) return
-          case RBRACE =>
-            if (openParens.count(LBRACE) > 0 && skippedParens.count(LBRACE) == 0)
-              return
-            skippedParens.change(LBRACE, -1)
-          case RPAREN =>
-            if (openParens.count(LPAREN) > 0 && skippedParens.nonePositive)
-              return
-            skippedParens.change(LPAREN, -1)
-          case RBRACKET =>
-            if (openParens.count(LBRACKET) > 0 && skippedParens.nonePositive)
-              return
-            skippedParens.change(LBRACKET, -1)
-          case OUTDENT =>
-            if (openParens.count(INDENT) > 0 && skippedParens.count(INDENT) == 0)
-              return
-            skippedParens.change(INDENT, -1)
-          case LBRACE =>
-            skippedParens.change(LBRACE, +1)
-          case LPAREN =>
-            skippedParens.change(LPAREN, +1)
-          case LBRACKET=>
-            skippedParens.change(LBRACKET, +1)
-          case INDENT =>
-            skippedParens.change(INDENT, +1)
-          case _ =>
-            if (mustStartStat &&
-                in.isAfterLineEnd &&
-                isLeqIndented(in.offset, lastStatOffset max 0))
-              return
-        }
+     */
+    protected def skip(): Unit =
+      val lastRegion = in.currentRegion
+      def atStop =
+        in.token == EOF
+        || skipStopTokens.contains(in.token) && (in.currentRegion eq lastRegion)
+      while !atStop do
         in.nextToken()
-      }
-    }
 
     def warning(msg: Message, sourcePos: SourcePosition): Unit =
       report.warning(msg, sourcePos)
@@ -557,15 +506,9 @@ object Parsers {
 
 /* -------- COMBINATORS -------------------------------------------------------- */
 
-    def enclosed[T](tok: Token, body: => T): T = {
+    def enclosed[T](tok: Token, body: => T): T =
       accept(tok)
-      openParens.change(tok, 1)
-      try body
-      finally {
-        accept(tok + 1)
-        openParens.change(tok, -1)
-      }
-    }
+      try body finally accept(tok + 1)
 
     def inParens[T](body: => T): T = enclosed(LPAREN, body)
     def inBraces[T](body: => T): T = enclosed(LBRACE, body)
@@ -1429,7 +1372,6 @@ object Parsers {
             functionRest(Nil)
           }
           else {
-            openParens.change(LPAREN, 1)
             imods = modifiers(funTypeArgMods)
             val paramStart = in.offset
             val ts = funArgType() match {
@@ -1441,7 +1383,6 @@ object Parsers {
               case t =>
                 funArgTypesRest(t, funArgType)
             }
-            openParens.change(LPAREN, -1)
             accept(RPAREN)
             if isValParamList || in.token == ARROW || in.token == CTXARROW then
               functionRest(ts)
@@ -2149,14 +2090,12 @@ object Parsers {
         if in.token == RPAREN then
           Nil
         else
-          openParens.change(LPAREN, 1)
           var mods1 = mods
           if in.token == ERASED then mods1 = addModifier(mods1)
           try
             commaSeparated(() => binding(mods1))
           finally
             accept(RPAREN)
-            openParens.change(LPAREN, -1)
       else {
         val start = in.offset
         val name = bindingName()
@@ -2504,7 +2443,6 @@ object Parsers {
         val enums =
           if (leading == LBRACE || leading == LPAREN && followingIsEnclosedGenerators()) {
             in.nextToken()
-            openParens.change(leading, 1)
             val res =
               if (leading == LBRACE || in.token == CASE)
                 enumerators()
@@ -2514,7 +2452,6 @@ object Parsers {
                   if (in.token == RPAREN || pats.length > 1) {
                     wrappedEnums = false
                     accept(RPAREN)
-                    openParens.change(LPAREN, -1)
                     atSpan(start) { makeTupleOrParens(pats) } // note: alternatives `|' need to be weeded out by typer.
                   }
                   else pats.head
@@ -2523,7 +2460,6 @@ object Parsers {
             if (wrappedEnums) {
               val closingOnNewLine = in.isAfterLineEnd
               accept(leading + 1)
-              openParens.change(leading, -1)
               def hasMultiLineEnum =
                 res.exists { t =>
                   val pos = t.sourcePos

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -1383,6 +1383,22 @@ object Scanners {
     private def useOuterWidth(): Unit =
       if enclosing.knownWidth == null then enclosing.useOuterWidth()
       knownWidth = enclosing.knownWidth
+
+    private def delimiter = this match
+      case _: InString => "}(in string)"
+      case InParens(LPAREN, _) => ")"
+      case InParens(LBRACKET, _) => "]"
+      case _: InBraces => "}"
+      case _: InCase => "=>"
+      case _: Indented => "UNDENT"
+
+    /** Show open regions as list of lines with decreasing indentations */
+    def visualize: String =
+      indentWidth.toPrefix
+      + delimiter
+      + outer.match
+          case null => ""
+          case next: Region => "\n" + next.visualize
   end Region
 
   case class InString(multiLine: Boolean, outer: Region) extends Region

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -287,5 +287,7 @@ object Tokens extends TokensCommon {
 
   final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, THROW, GIVEN, VAL, THIS)
 
+  final val skipStopTokens = BitSet(SEMI, NEWLINE, NEWLINES, RBRACE, RPAREN, RBRACKET, OUTDENT)
+
   final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent, nme.infix)
 }

tests/neg/errpos.scala

@@ -1,11 +1,11 @@
 object Test {
   val x =     // error: expression expected
-  val y = 2   // error: ';' expected
+  val y = 2
 
   val z =     // error: expression expected
 
   // ...
-  val a = 3   // error: ';' expected
+  val a = 3
 
   val b = type // error: expression expected (on "type")
 

tests/neg/extensions-can-only-have-defs.scala

@@ -1,5 +1,5 @@
 object Test {
   extension [T] (t: T) {
-    val v: T = ??? // error // error
+    val v: T = ??? // error
   }
 }

tests/neg/i10546.scala

@@ -2,4 +2,4 @@ object test:
   def times(num : Int)(block : => Unit) : Unit = ()
     times(10): println("ah") // error: end of statement expected but '(' found // error
 
-  def foo: Set(Int) = Set(1) // error: end of statement expected but '(' found // error // error
+  def foo: Set(Int) = Set(1)

tests/neg/i4373.scala

@@ -16,7 +16,7 @@ class A4 extends _ with Base // error
 object Test {
   type T1 = _ // error
   type T2 = _[Int] // error // error
-  type T3 = _ { type S } // error
+  type T3 = _ { type S } // error // error
   type T4 = [X] =>> _ // error
 
   // Open questions:

tests/neg/i4934.scala

@@ -8,4 +8,4 @@ object Main {
 
 object Foo {
   val a = ""); // error: `}` expected but `)` found
-} // error: eof expected
+}

tests/neg/i5004.scala

@@ -1,6 +1,6 @@
 object i0 {
 1 match {
-def this(): Int  // error // error
-  def this()     // error
-}                // error
+def this(): Int  // error
+  def this()
+}
 }

tests/neg/i5032b.scala

@@ -1,2 +1,2 @@
 class a@
-class b@ // error // error
+class b@ // error

tests/neg/i7751.scala

@@ -1,2 +1,2 @@
 val a = Some(a=a,)=> // error // error
-val a = Some(x=y,)=> // error
+val a = Some(x=y,)=>

tests/neg/i8731.scala

@@ -3,27 +3,17 @@ object test:
   3 match
   case 3 => ???
   end match
-  case _ => () // error: missing parameter type
+  case _ => () // error: expected start of definition
   end match
 
   if 3 == 3 then
     ()
   end if
   else     // error: illegal start of definition
     ()
-  end if
+  end if   // error: misaligned end marker
 
   class Test {
     val test = 3
-  end Test   // error: misaligned end marker
-  }
-
-  while
-    3 == 3
-  end while  // error: `do` expected
-  do ()
-
-  for
-    a <- Seq()
-  end for    // error: `yield` or `do` expected
-  do ()
+  end Test // error: misaligned end marker
+  }  // error: eof expected, but unindent found

tests/neg/i8731a.scala

@@ -0,0 +1,10 @@
+object test:
+  while
+    3 == 3
+  end while  // error: `do` expected
+  do ()
+
+  for
+    a <- Seq()
+  end for    // error: `yield` or `do` expected
+  do ()            // error: expression expected but eof found

tests/neg/i9328.scala

@@ -14,8 +14,8 @@ case class Foo()
 
 case class Bar(
 } { ;  // error
-object Bar { // error
+object Bar {
   class Foo(a: Int) extends AnyVal
   Foo()
 }
-type Bar // error
+type Bar

tests/neg/indent.scala

@@ -4,6 +4,6 @@ object Test {
   val y3 =
     if (1) max 10 gt 0  // error: end of statement expected but integer literal found // error // error // error
       1
-    else  // error: end of statement expected but 'else' found
-      2   // error
-}
+    else
+      2
+} // error