Partest for Dotty with pos tests and neg tests with error count

Author: vsalvis

.gitignore

@@ -1,3 +1,4 @@
+*.DS_Store
 *.class
 *.log
 *~
@@ -24,3 +25,7 @@ classes/
 .idea
 .idea_modules
 /.worksheet/
+
+# Partest
+tests/partest-generated/
+tests/runPartest.flag

project/Build.scala

@@ -32,10 +32,11 @@ object DottyBuild extends Build {
     libraryDependencies ++= Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value,
                                 "org.scala-lang.modules" %% "scala-xml" % "1.0.1",
                                 "me.d-d" % "scala-compiler" % "2.11.5-20150416-144435-09c4a520e1",
+                                "org.scala-lang.modules" %% "scala-partest" % "1.0.5" % "test",
                                 "jline" % "jline" % "2.12"),
 
     // get junit onboard
-    libraryDependencies += "com.novocode" % "junit-interface" % "0.11-RC1" % "test",
+    libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
 
     // scalac options
     scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:_"),
@@ -46,7 +47,13 @@ object DottyBuild extends Build {
     incOptions := incOptions.value.withNameHashing(true),
 
     // enable verbose exception messages for JUnit
-    testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+    testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+    // when this file is present, running test generates the files for partest
+    // otherwise it just executes the tests directly
+    createPartestFile := { new java.io.File("./tests", "runPartest.flag").createNewFile },
+    runPartestRunner <<= runTask(Test, "dotty.partest.DPConsoleRunner", "") dependsOn (test in Test),
+    deletePartestFile := { new java.io.File("./tests", "runPartest.flag").delete },
+
     // Adjust classpath for running dotty
     mainClass in (Compile, run) := Some("dotty.tools.dotc.Main"),
     fork in run := true,
@@ -79,7 +86,7 @@ object DottyBuild extends Build {
 
       tuning ::: agentOptions ::: travis_build ::: fullpath
     }
-  )
+  ) ++ addCommandAlias("partest", ";createPartestFile;runPartestRunner;deletePartestFile")
 
   lazy val dotty = Project(id = "dotty", base = file("."), settings = defaults)
 
@@ -92,7 +99,7 @@ object DottyBuild extends Build {
 
 
     libraryDependencies ++= Seq("com.storm-enroute" %% "scalameter" % "0.6" % Test,
-      "com.novocode" % "junit-interface" % "0.11-RC1"),
+      "com.novocode" % "junit-interface" % "0.11"),
     testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"),
 
     // scalac options
@@ -129,4 +136,9 @@ object DottyBuild extends Build {
 
   lazy val benchmarks = Project(id = "dotty-bench", settings = benchmarkSettings,
     base = file("bench")) dependsOn(dotty % "compile->test")
+
+  lazy val createPartestFile = TaskKey[Unit]("createPartestFile", "Creates the tests/runPartest.flag file")
+  lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partests")
+  lazy val deletePartestFile = TaskKey[Unit]("deletePartestFile", "Deletes the tests/runPartest.flag file")
+
 }

test/dotc/comptest.scala

@@ -4,18 +4,21 @@ import test._
 
 object comptest extends CompilerTest {
 
+  override val generatePartestFiles = false
+  val defaultOutputDir: String = ""
+
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"
   val dotcDir = "./src/dotty/"
 
   def main(args: Array[String]) =
-    compileArgs(Array(
+    compileList("comptest", List(
       dotcDir + "tools/dotc/CompilationUnit.scala",
       dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
+      dotcDir + "tools/dotc/ast/Trees.scala"), List(
       "#runs", "2",
       "-Ylog:frontend",
       "-Xprompt"))(Nil)
 
-//    compileDir(dotcDir + "tools/dotc/printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
+//    compileDir(dotcDir + "tools/dotc/", "printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
 }

test/dotc/tests.scala

@@ -1,7 +1,10 @@
 package dotc
 
-import org.junit.Test
 import test._
+import dotty.partest._
+import org.junit.Test
+import org.junit.experimental.categories._
+
 
 class tests extends CompilerTest {
 
@@ -13,12 +16,13 @@ class tests extends CompilerTest {
 //        "-Yshow-suppressed-errors",
         "-pagewidth", "160")
 
+  val defaultOutputDir = "./out/"
+
   implicit val defaultOptions = noCheckOptions ++ List(
       "-Yno-deep-subtypes", "-Yno-double-bindings",
       "-Ycheck:tailrec,resolveSuper,mixin,restoreScopes",
-      "-d", "./out/"
+      "-d", defaultOutputDir
   )
-
   val doEmitBytecode = List("-Ystop-before:terminal")
   val failedbyName = List("-Ystop-before:collectEntryPoints") // #288
   val testPickling = List("-Xprint-types", "-Ytest-pickler", "-Ystop-after:pickler")
@@ -30,19 +34,23 @@ class tests extends CompilerTest {
   val allowDeepSubtypes = defaultOptions diff List("-Yno-deep-subtypes")
   val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings")
 
-  val posDir = "./tests/pos/"
-  val posSpecialDir = "./tests/pos-special/"
-  val negDir = "./tests/neg/"
-  val newDir = "./tests/new/"
-  val dotcDir = "./src/dotty/"
-  val picklingDir = "./tests/pickling"
+  val testsDir      = "./tests/"
+  val posDir        = testsDir + "pos/"  
+  val posSpecialDir = testsDir + "pos-special/"
+  val negDir        = testsDir + "neg/"
+  val newDir        = testsDir + "new/"
 
-  @Test def pickle_pickleOK = compileDir(picklingDir, testPickling)
-  @Test def pickle_pickling = compileDir(dotcDir + "tools/dotc/core/pickling/", testPickling)
-  @Test def pickle_ast = compileDir(dotcDir + "tools/dotc/ast/", testPickling)
+  val dottyDir = "./src/dotty/"
+  val toolsDir = dottyDir + "tools/"
+  val dotcDir  = toolsDir + "dotc/"
+  val coreDir  = dotcDir + "core/"
 
-  //@Test def pickle_core = compileDir(dotcDir + "tools/dotc/core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  @Test def pickle_pickleOK = compileDir(testsDir, "pickling", testPickling)
+  @Test def pickle_pickling = compileDir(coreDir, "pickling", testPickling)
+  @Test def pickle_ast = compileDir(dotcDir, "ast", testPickling)
 
+  //@Test def pickle_core = compileDir(dotcDir, "core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  
   @Test def pos_t2168_pat = compileFile(posDir, "t2168", twice)
   @Test def pos_erasure = compileFile(posDir, "erasure", twice)
   @Test def pos_Coder() = compileFile(posDir, "Coder", twice)
@@ -66,12 +74,12 @@ class tests extends CompilerTest {
   @Test def pos_packageobject() = compileFile(posDir, "packageobject", twice)
   @Test def pos_overloaded() = compileFile(posDir, "overloaded", twice)
   @Test def pos_overrides() = compileFile(posDir, "overrides", twice)
-  @Test def pos_javaOverride() = compileDir(posDir + "java-override", twice)
+  @Test def pos_javaOverride() = compileDir(posDir, "java-override", twice)
   @Test def pos_templateParents() = compileFile(posDir, "templateParents", twice)
   @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice)
   @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice)
-  @Test def pos_tailcall = compileDir(posDir + "tailcall/", twice)
-  @Test def pos_nullarify = compileFile(posDir, "nullarify", "-Ycheck:nullarify" :: Nil)
+  @Test def pos_tailcall = compileDir(posDir, "tailcall", twice)
+  @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil)
   @Test def pos_subtyping = compileFile(posDir, "subtyping", twice)
   @Test def pos_t2613 = compileFile(posSpecialDir, "t2613")(allowDeepSubtypes)
   @Test def pos_packageObj = compileFile(posDir, "i0239", twice)
@@ -92,7 +100,7 @@ class tests extends CompilerTest {
   @Test def neg_privates() = compileFile(negDir, "privates", xerrors = 2)
   @Test def neg_rootImports = compileFile(negDir, "rootImplicits", xerrors = 2)
   @Test def neg_templateParents() = compileFile(negDir, "templateParents", xerrors = 3)
-  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", "-language:noAutoTupling" :: Nil, xerrors = 4)
+  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, xerrors = 4)
   @Test def neg_autoTupling2 = compileFile(negDir, "autoTuplingTest", xerrors = 4)
   @Test def neg_companions = compileFile(negDir, "companions", xerrors = 1)
   @Test def neg_over = compileFile(negDir, "over", xerrors = 3)
@@ -102,14 +110,18 @@ class tests extends CompilerTest {
   @Test def neg_i50_volatile = compileFile(negDir, "i50-volatile", xerrors = 6)
   @Test def neg_t0273_doubledefs = compileFile(negDir, "t0273", xerrors = 1)
   @Test def neg_zoo = compileFile(negDir, "zoo", xerrors = 12)
-  @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
-  @Test def neg_tailcall_t1672b = compileFile(negDir, "tailcall/t1672b", xerrors = 6)
-  @Test def neg_tailcall_t3275 = compileFile(negDir, "tailcall/t3275", xerrors = 1)
-  @Test def neg_tailcall_t6574 = compileFile(negDir, "tailcall/t6574", xerrors = 2)
-  @Test def neg_tailcall = compileFile(negDir, "tailcall/tailrec", xerrors = 7)
-  @Test def neg_tailcall2 = compileFile(negDir, "tailcall/tailrec-2", xerrors = 2)
-  @Test def neg_tailcall3 = compileFile(negDir, "tailcall/tailrec-3", xerrors = 2)
-  @Test def nef_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
+  // TODO: this test file doesn't exist (anymore?), remove?
+  // @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
+
+  val negTailcallDir = negDir + "tailcall/"
+  @Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", xerrors = 6)
+  @Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", xerrors = 1)
+  @Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", xerrors = 2)
+  @Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", xerrors = 7)
+  @Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", xerrors = 2)
+  @Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", xerrors = 2)
+  
+  @Test def neg_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
   @Test def neg_t1843_variances = compileFile(negDir, "t1843-variances", xerrors = 1)
   @Test def neg_t2660_ambi = compileFile(negDir, "t2660", xerrors = 2)
   @Test def neg_t2994 = compileFile(negDir, "t2994", xerrors = 2)
@@ -127,62 +139,60 @@ class tests extends CompilerTest {
   @Test def neg_moduleSubtyping = compileFile(negDir, "moduleSubtyping", xerrors = 4)
   @Test def neg_escapingRefs = compileFile(negDir, "escapingRefs", xerrors = 2)
 
-  @Test def dotc = compileDir(dotcDir + "tools/dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
-  @Test def dotc_ast = compileDir(dotcDir + "tools/dotc/ast", failedOther ++ twice)
+  @Test def dotc = compileDir(toolsDir, "dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
+  @Test def dotc_ast = compileDir(dotcDir, "ast", failedOther ++ twice)
     //similar to dotc_core_pickling but for another anon class. Still during firstTransform
-  @Test def dotc_config = compileDir(dotcDir + "tools/dotc/config")
-  @Test def dotc_core = compileDir(dotcDir + "tools/dotc/core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_config = compileDir(dotcDir, "config")
+  @Test def dotc_core = compileDir(dotcDir, "core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
     // error: error while loading ConstraintHandling$$anon$1$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/core/ConstraintHandling$$anon$1.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_core_pickling = compileDir(dotcDir + "tools/dotc/core/pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_core_pickling = compileDir(coreDir, "pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
     // exception caught when loading class ClassfileParser$$anon$1: dotty.tools.dotc.core.Denotations$NotDefinedHere:
     // demanding denotation of module class ClassfileParser$$anon$1$ at phase frontend(1) outside defined interval:
     // defined periods are Period(31..36, run = 2) Period(3..24, run = 2) Period(25..26, run = 2)
     // Period(27..28, run = 2) Period(29..29, run = 2) Period(30..30, run = 2)
     // inside FirstTransform    at dotty.tools.dotc.transform.FirstTransform.transform(FirstTransform.scala:33)
     // weird.
 
-  @Test def dotc_transform = compileDir(dotcDir + "tools/dotc/transform")// twice omitted to make tests run faster
+  @Test def dotc_transform = compileDir(dotcDir, "transform")// twice omitted to make tests run faster
 
-  @Test def dotc_parsing = compileDir(dotcDir + "tools/dotc/parsing")// twice omitted to make tests run faster
+  @Test def dotc_parsing = compileDir(dotcDir, "parsing") // twice omitted to make tests run faster
 
-  @Test def dotc_printing = compileDir(dotcDir + "tools/dotc/printing") // twice omitted to make tests run faster
+  @Test def dotc_printing = compileDir(dotcDir, "printing") // twice omitted to make tests run faster
 
-  @Test def dotc_reporting = compileDir(dotcDir + "tools/dotc/reporting") // twice omitted to make tests run faster
+  @Test def dotc_reporting = compileDir(dotcDir, "reporting") // twice omitted to make tests run faster
 
-  @Test def dotc_typer = compileDir(dotcDir + "tools/dotc/typer", failedOther) // twice omitted to make tests run faster
+  @Test def dotc_typer = compileDir(dotcDir, "typer", failedOther)// twice omitted to make tests run faster
     // error: error while loading Checking$$anon$2$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/typer/Checking$$anon$2.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_util = compileDir(dotcDir + "tools/dotc/util", failedOther ++ twice)
+  @Test def dotc_util = compileDir(dotcDir, "util", failedOther ++ twice)
     // java.lang.ClassCastException: dotty.tools.dotc.core.Types$NoType$ cannot be cast to dotty.tools.dotc.core.Types$ClassInfo
     // at dotty.tools.dotc.core.SymDenotations$ClassDenotation.classInfo(SymDenotations.scala:1026)
     // at dotty.tools.dotc.transform.ExtensionMethods.transform(ExtensionMethods.scala:38)
 
-  @Test def tools_io = compileDir(dotcDir + "tools/io", failedOther ++ twice) // inner class has symbol <none>
+  @Test def tools_io = compileDir(toolsDir, "io", failedOther ++ twice) // inner class has symbol <none>
 
   @Test def helloWorld = compileFile(posDir, "HelloWorld", twice)
   @Test def labels = compileFile(posDir, "Labels", twice)
-  //@Test def tools = compileDir(dotcDir + "tools", "-deep" :: Nil)(allowDeepSubtypes)
+  //@Test def tools = compileDir(dottyDir, "tools", "-deep" :: Nil)(allowDeepSubtypes)
 
-  @Test def testNonCyclic = compileArgs(Array(
-      dotcDir + "tools/dotc/CompilationUnit.scala",
-      dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
-      "-Xprompt"
-      ) ++ staleSymbolError ++ twice)
+  @Test def testNonCyclic = compileList("testNonCyclic", List(
+      dotcDir + "CompilationUnit.scala",
+      coreDir + "Types.scala",
+      dotcDir + "ast/Trees.scala"
+    ), List("-Xprompt") ++ staleSymbolError ++ twice)
 
-  @Test def testIssue_34 = compileArgs(Array(
-      dotcDir + "tools/dotc/config/Properties.scala",
-      dotcDir + "tools/dotc/config/PathResolver.scala",
-      //"-Ylog:frontend",
-      "-Xprompt") ++ staleSymbolError ++ twice)
+  @Test def testIssue_34 = compileList("testIssue_34", List(
+      dotcDir + "config/Properties.scala",
+      dotcDir + "config/PathResolver.scala"
+    ), List(/* "-Ylog:frontend", */ "-Xprompt") ++ staleSymbolError ++ twice)
 
   val javaDir = "./tests/pos/java-interop/"
   @Test def java_all = compileFiles(javaDir, twice)
 
-  //@Test def dotc_compilercommand = compileFile(dotcDir + "tools/dotc/config/", "CompilerCommand")
+  //@Test def dotc_compilercommand = compileFile(dotcDir + "config/", "CompilerCommand")
 }

test/dotty/partest/DPConfig.scala

@@ -0,0 +1,31 @@
+package dotty.partest
+
+import java.io.File
+import scala.collection.JavaConversions._
+
+
+/** Dotty Partest runs all tests in the provided testDirs located under
+  * testRoot. There can be several directories with pos resp. neg tests, as
+  * long as the prefix is pos/neg.
+  *
+  * Each testDir can also have a __defaultFlags.flags file, which provides
+  * compiler flags and is used unless there's a specific flags file (e.g. for
+  * test pos/A.scala, if there's a pos/A.flags file those flags are used,
+  * otherwise pos/__defaultFlags.flags are used if the file exists).
+  */
+object DPConfig {
+  val testRoot = "./tests/partest-generated"
+  lazy val testDirs = {
+    val root = new File(testRoot)
+    val dirs = if (!root.exists) Array.empty[String] else root.listFiles.filter(_.isDirectory).map(_.getName)
+    if (dirs.length > 0)
+      println(s"Partest found generated source directories in $testRoot: " + dirs.mkString(", "))
+    else
+      throw new Exception("Partest did not detect any generated sources")
+    dirs
+  }
+
+  // Tests finish faster when running in parallel, but console output is
+  // out of order and sometimes the compiler crashes
+  val runTestsInParallel = false
+}