Partest for Dotty with pos tests and neg tests with error count

Author: vsalvis

.gitignore

@@ -1,3 +1,4 @@
+*.DS_Store
 *.class
 *.log
 *~
@@ -24,3 +25,6 @@ classes/
 .idea
 .idea_modules
 /.worksheet/
+
+# Partest
+tests/partest-generated/

project/Build.scala

@@ -4,6 +4,7 @@ import sbt._
 object DottyBuild extends Build {
 
   val TRAVIS_BUILD = "dotty.travis.build"
+  val isTravisBuild = sys.props.isDefinedAt(TRAVIS_BUILD)
 
   val agentOptions = List(
     // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
@@ -32,10 +33,11 @@ object DottyBuild extends Build {
     libraryDependencies ++= Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value,
                                 "org.scala-lang.modules" %% "scala-xml" % "1.0.1",
                                 "me.d-d" % "scala-compiler" % "2.11.5-20150402-193021-0c75410da3",
+                                "org.scala-lang.modules" %% "scala-partest" % "1.0.5" % "test",
                                 "jline" % "jline" % "2.12"),
 
     // get junit onboard
-    libraryDependencies += "com.novocode" % "junit-interface" % "0.11-RC1" % "test",
+    libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
 
     // scalac options
     scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:_"),
@@ -46,7 +48,12 @@ object DottyBuild extends Build {
     incOptions := incOptions.value.withNameHashing(true),
 
     // enable verbose exception messages for JUnit
-    testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+    // JUnit from sbt only generates partest files, doesn't run tests
+    testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector",
+      "--exclude-categories=dotty.partest." + (if (isTravisBuild) "" else "Non") + "PartestTarget"),
+    // test generates files, then we run partest
+    partest <<= runTask(Test, "dotty.partest.DPConsoleRunner", "") dependsOn (test in Test),
+
     // Adjust classpath for running dotty
     mainClass in (Compile, run) := Some("dotty.tools.dotc.Main"),
     fork in run := true,
@@ -65,10 +72,7 @@ object DottyBuild extends Build {
        // System.err.println("BOOTPATH: " + fullpath)
 
        val travis_build = // propagate if this is a travis build
-         if (sys.props.isDefinedAt(TRAVIS_BUILD))
-           List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}")
-         else
-           List()
+         if (isTravisBuild) List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}") else List()
 
        val tuning =
          if (sys.props.isDefinedAt("Oshort"))
@@ -79,7 +83,9 @@ object DottyBuild extends Build {
 
       tuning ::: agentOptions ::: travis_build ::: fullpath
     }
-  )
+  ) ++ (if (isTravisBuild) Seq() else addCommandAlias("test", "partest"))
+  // alias so that "sbt test" executes partest (which dependsOn test) - no
+  // circular dependency because the setting is evaluated before
 
   lazy val dotty = Project(id = "dotty", base = file("."), settings = defaults)
 
@@ -92,7 +98,7 @@ object DottyBuild extends Build {
 
 
     libraryDependencies ++= Seq("com.storm-enroute" %% "scalameter" % "0.6" % Test,
-      "com.novocode" % "junit-interface" % "0.11-RC1"),
+      "com.novocode" % "junit-interface" % "0.11"),
     testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"),
 
     // scalac options
@@ -116,10 +122,8 @@ object DottyBuild extends Build {
       // System.err.println("BOOTPATH: " + fullpath)
 
       val travis_build = // propagate if this is a travis build
-        if (sys.props.isDefinedAt(TRAVIS_BUILD))
-          List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}")
-        else
-          List()
+        if (isTravisBuild) List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}") else List()
+
       val res = agentOptions ::: travis_build ::: fullpath
       println("Running with javaOptions: " + res)
       res
@@ -129,4 +133,6 @@ object DottyBuild extends Build {
 
   lazy val benchmarks = Project(id = "dotty-bench", settings = benchmarkSettings,
     base = file("bench")) dependsOn(dotty % "compile->test")
+
+  lazy val partest = TaskKey[Unit]("partest", "Runs test and partests")
 }

test/dotc/comptest.scala

@@ -9,13 +9,13 @@ object comptest extends CompilerTest {
   val dotcDir = "./src/dotty/"
 
   def main(args: Array[String]) =
-    compileArgs(Array(
+    compileList("comptest", List(
       dotcDir + "tools/dotc/CompilationUnit.scala",
       dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
+      dotcDir + "tools/dotc/ast/Trees.scala"), List(
       "#runs", "2",
       "-Ylog:frontend",
       "-Xprompt"))(Nil)
 
-//    compileDir(dotcDir + "tools/dotc/printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
+//    compileDir(dotcDir + "tools/dotc/", "printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
 }

test/dotc/tests.scala

@@ -1,9 +1,15 @@
 package dotc
 
-import org.junit.Test
 import test._
+import dotty.partest._
+import org.junit.Test
+import org.junit.experimental.categories._
 
-class tests extends CompilerTest {
+
+object TestOptions {
+  val outputDir     = "./out/"
+  val testsDir      = "./tests/"
+  val posDir        = testsDir + "pos/"  
 
   val noCheckOptions = List(
 //        "-verbose",
@@ -16,8 +22,16 @@ class tests extends CompilerTest {
   implicit val defaultOptions = noCheckOptions ++ List(
       "-Yno-deep-subtypes", "-Yno-double-bindings",
       "-Ycheck:tailrec,resolveSuper,mixin,restoreScopes",
-      "-d", "./out/"
+      "-d", outputDir
   )
+}
+
+// Always use this annotation together with a subclass of the test that mixes
+// in PartestVersion (see below). This allows the tests to be run by partest
+// instead of JUnit when using sbt test.
+@Category(Array(classOf[NonPartestTarget])) 
+class tests extends CompilerTest {
+  import TestOptions._
 
   val doEmitBytecode = List("-Ystop-before:terminal")
   val failedbyName = List("-Ystop-before:collectEntryPoints") // #288
@@ -30,19 +44,21 @@ class tests extends CompilerTest {
   val allowDeepSubtypes = defaultOptions diff List("-Yno-deep-subtypes")
   val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings")
 
-  val posDir = "./tests/pos/"
-  val posSpecialDir = "./tests/pos-special/"
-  val negDir = "./tests/neg/"
-  val newDir = "./tests/new/"
-  val dotcDir = "./src/dotty/"
-  val picklingDir = "./tests/pickling"
+  val posSpecialDir = testsDir + "pos-special/"
+  val negDir        = testsDir + "neg/"
+  val newDir        = testsDir + "new/"
 
-  @Test def pickle_pickleOK = compileDir(picklingDir, testPickling)
-  @Test def pickle_pickling = compileDir(dotcDir + "tools/dotc/core/pickling/", testPickling)
-  @Test def pickle_ast = compileDir(dotcDir + "tools/dotc/ast/", testPickling)
+  val dottyDir = "./src/dotty/"
+  val toolsDir = dottyDir + "tools/"
+  val dotcDir  = toolsDir + "dotc/"
+  val coreDir  = dotcDir + "core/"
 
-  //@Test def pickle_core = compileDir(dotcDir + "tools/dotc/core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  @Test def pickle_pickleOK = compileDir(testsDir, "pickling", testPickling)
+  @Test def pickle_pickling = compileDir(coreDir, "pickling", testPickling)
+  @Test def pickle_ast = compileDir(dotcDir, "ast", testPickling)
 
+  //@Test def pickle_core = compileDir(dotcDir, "core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  
   @Test def pos_t2168_pat = compileFile(posDir, "t2168", twice)
   @Test def pos_erasure = compileFile(posDir, "erasure", twice)
   @Test def pos_Coder() = compileFile(posDir, "Coder", twice)
@@ -66,12 +82,12 @@ class tests extends CompilerTest {
   @Test def pos_packageobject() = compileFile(posDir, "packageobject", twice)
   @Test def pos_overloaded() = compileFile(posDir, "overloaded", twice)
   @Test def pos_overrides() = compileFile(posDir, "overrides", twice)
-  @Test def pos_javaOverride() = compileDir(posDir + "java-override", twice)
+  @Test def pos_javaOverride() = compileDir(posDir, "java-override", twice)
   @Test def pos_templateParents() = compileFile(posDir, "templateParents", twice)
   @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice)
   @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice)
-  @Test def pos_tailcall = compileDir(posDir + "tailcall/", twice)
-  @Test def pos_nullarify = compileFile(posDir, "nullarify", "-Ycheck:nullarify" :: Nil)
+  @Test def pos_tailcall = compileDir(posDir, "tailcall", twice)
+  @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil)
   @Test def pos_subtyping = compileFile(posDir, "subtyping", twice)
   @Test def pos_t2613 = compileFile(posSpecialDir, "t2613")(allowDeepSubtypes)
   @Test def pos_packageObj = compileFile(posDir, "i0239", twice)
@@ -92,7 +108,7 @@ class tests extends CompilerTest {
   @Test def neg_privates() = compileFile(negDir, "privates", xerrors = 2)
   @Test def neg_rootImports = compileFile(negDir, "rootImplicits", xerrors = 2)
   @Test def neg_templateParents() = compileFile(negDir, "templateParents", xerrors = 3)
-  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", "-language:noAutoTupling" :: Nil, xerrors = 4)
+  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, xerrors = 4)
   @Test def neg_autoTupling2 = compileFile(negDir, "autoTuplingTest", xerrors = 4)
   @Test def neg_companions = compileFile(negDir, "companions", xerrors = 1)
   @Test def neg_over = compileFile(negDir, "over", xerrors = 3)
@@ -102,14 +118,18 @@ class tests extends CompilerTest {
   @Test def neg_i50_volatile = compileFile(negDir, "i50-volatile", xerrors = 6)
   @Test def neg_t0273_doubledefs = compileFile(negDir, "t0273", xerrors = 1)
   @Test def neg_zoo = compileFile(negDir, "zoo", xerrors = 12)
-  @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
-  @Test def neg_tailcall_t1672b = compileFile(negDir, "tailcall/t1672b", xerrors = 6)
-  @Test def neg_tailcall_t3275 = compileFile(negDir, "tailcall/t3275", xerrors = 1)
-  @Test def neg_tailcall_t6574 = compileFile(negDir, "tailcall/t6574", xerrors = 2)
-  @Test def neg_tailcall = compileFile(negDir, "tailcall/tailrec", xerrors = 7)
-  @Test def neg_tailcall2 = compileFile(negDir, "tailcall/tailrec-2", xerrors = 2)
-  @Test def neg_tailcall3 = compileFile(negDir, "tailcall/tailrec-3", xerrors = 2)
-  @Test def nef_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
+  // TODO: this test file doesn't exist (anymore?), remove?
+  // @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
+
+  val negTailcallDir = negDir + "tailcall/"
+  @Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", xerrors = 6)
+  @Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", xerrors = 1)
+  @Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", xerrors = 2)
+  @Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", xerrors = 7)
+  @Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", xerrors = 2)
+  @Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", xerrors = 2)
+  
+  @Test def neg_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
   @Test def neg_t1843_variances = compileFile(negDir, "t1843-variances", xerrors = 1)
   @Test def neg_t2660_ambi = compileFile(negDir, "t2660", xerrors = 2)
   @Test def neg_t2994 = compileFile(negDir, "t2994", xerrors = 2)
@@ -127,62 +147,75 @@ class tests extends CompilerTest {
   @Test def neg_moduleSubtyping = compileFile(negDir, "moduleSubtyping", xerrors = 4)
   @Test def neg_escapingRefs = compileFile(negDir, "escapingRefs", xerrors = 2)
 
-  @Test def dotc = compileDir(dotcDir + "tools/dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
-  @Test def dotc_ast = compileDir(dotcDir + "tools/dotc/ast", failedOther ++ twice)
+  @Test def dotc = compileDir(toolsDir, "dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
+  @Test def dotc_ast = compileDir(dotcDir, "ast", failedOther ++ twice)
     //similar to dotc_core_pickling but for another anon class. Still during firstTransform
-  @Test def dotc_config = compileDir(dotcDir + "tools/dotc/config")
-  @Test def dotc_core = compileDir(dotcDir + "tools/dotc/core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_config = compileDir(dotcDir, "config")
+  @Test def dotc_core = compileDir(dotcDir, "core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
     // error: error while loading ConstraintHandling$$anon$1$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/core/ConstraintHandling$$anon$1.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_core_pickling = compileDir(dotcDir + "tools/dotc/core/pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_core_pickling = compileDir(coreDir, "pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
     // exception caught when loading class ClassfileParser$$anon$1: dotty.tools.dotc.core.Denotations$NotDefinedHere:
     // demanding denotation of module class ClassfileParser$$anon$1$ at phase frontend(1) outside defined interval:
     // defined periods are Period(31..36, run = 2) Period(3..24, run = 2) Period(25..26, run = 2)
     // Period(27..28, run = 2) Period(29..29, run = 2) Period(30..30, run = 2)
     // inside FirstTransform    at dotty.tools.dotc.transform.FirstTransform.transform(FirstTransform.scala:33)
     // weird.
 
-  @Test def dotc_transform = compileDir(dotcDir + "tools/dotc/transform")// twice omitted to make tests run faster
+  @Test def dotc_transform = compileDir(dotcDir, "transform")// twice omitted to make tests run faster
 
-  @Test def dotc_parsing = compileDir(dotcDir + "tools/dotc/parsing")// twice omitted to make tests run faster
+  @Test def dotc_parsing = compileDir(dotcDir, "parsing") // twice omitted to make tests run faster
 
-  @Test def dotc_printing = compileDir(dotcDir + "tools/dotc/printing") // twice omitted to make tests run faster
+  @Test def dotc_printing = compileDir(dotcDir, "printing") // twice omitted to make tests run faster
 
-  @Test def dotc_reporting = compileDir(dotcDir + "tools/dotc/reporting") // twice omitted to make tests run faster
+  @Test def dotc_reporting = compileDir(dotcDir, "reporting") // twice omitted to make tests run faster
 
-  @Test def dotc_typer = compileDir(dotcDir + "tools/dotc/typer", failedOther) // twice omitted to make tests run faster
+  @Test def dotc_typer = compileDir(dotcDir, "typer", failedOther)// twice omitted to make tests run faster
     // error: error while loading Checking$$anon$2$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/typer/Checking$$anon$2.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_util = compileDir(dotcDir + "tools/dotc/util", failedOther ++ twice)
+  @Test def dotc_util = compileDir(dotcDir, "util", failedOther ++ twice)
     // java.lang.ClassCastException: dotty.tools.dotc.core.Types$NoType$ cannot be cast to dotty.tools.dotc.core.Types$ClassInfo
     // at dotty.tools.dotc.core.SymDenotations$ClassDenotation.classInfo(SymDenotations.scala:1026)
     // at dotty.tools.dotc.transform.ExtensionMethods.transform(ExtensionMethods.scala:38)
 
-  @Test def tools_io = compileDir(dotcDir + "tools/io", failedOther ++ twice) // inner class has symbol <none>
+  @Test def tools_io = compileDir(toolsDir, "io", failedOther ++ twice) // inner class has symbol <none>
 
   @Test def helloWorld = compileFile(posDir, "HelloWorld", twice)
   @Test def labels = compileFile(posDir, "Labels", twice)
-  //@Test def tools = compileDir(dotcDir + "tools", "-deep" :: Nil)(allowDeepSubtypes)
+  //@Test def tools = compileDir(dottyDir, "tools", "-deep" :: Nil)(allowDeepSubtypes)
 
-  @Test def testNonCyclic = compileArgs(Array(
-      dotcDir + "tools/dotc/CompilationUnit.scala",
-      dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
-      "-Xprompt"
-      ) ++ staleSymbolError ++ twice)
+  @Test def testNonCyclic = compileList("testNonCyclic", List(
+      dotcDir + "CompilationUnit.scala",
+      coreDir + "Types.scala",
+      dotcDir + "ast/Trees.scala"
+    ), List("-Xprompt") ++ staleSymbolError ++ twice)
 
-  @Test def testIssue_34 = compileArgs(Array(
-      dotcDir + "tools/dotc/config/Properties.scala",
-      dotcDir + "tools/dotc/config/PathResolver.scala",
-      //"-Ylog:frontend",
-      "-Xprompt") ++ staleSymbolError ++ twice)
+  @Test def testIssue_34 = compileList("testIssue_34", List(
+      dotcDir + "config/Properties.scala",
+      dotcDir + "config/PathResolver.scala"
+    ), List(/* "-Ylog:frontend", */ "-Xprompt") ++ staleSymbolError ++ twice)
 
   val javaDir = "./tests/pos/java-interop/"
   @Test def java_all = compileFiles(javaDir, twice)
 
-  //@Test def dotc_compilercommand = compileFile(dotcDir + "tools/dotc/config/", "CompilerCommand")
+  //@Test def dotc_compilercommand = compileFile(dotcDir + "config/", "CompilerCommand")
+}
+
+
+// Always mix in the PartestVersion trait when the superclass is annotated as
+// above. Both together allow the tests to be run by partest instead of JUnit
+// when using sbt test.
+@Category(Array(classOf[PartestTarget])) 
+class testsPartest extends tests with DPCompilerTest {
+  // redirect class file output to partest-generated/kind/testname-kind.obj/
+  override val defaultOutputDir = TestOptions.outputDir
+}
+
+object testsNotPartestable extends CompilerTest {
+  import TestOptions._
+  // Add tests here that should always be run by JUnit, not partest
 }