Split scala3 doc and migrate file layout

Now both project matches repository standard.
Test and dummy classes does not clutter scala3doc jar (are defined in test)

Author: romanowski

.github/workflows/scala3doc.yaml

@@ -33,43 +33,13 @@ jobs:
           java-version: 11
 
       - name: Compile and test
-        run: sbt compile test
+        run: sbt scala3doc/test
 
       - name: Locally publish self
-        run: sbt publishLocal
+        run: sbt scala3doc/publishLocal
 
       - name: Generate test documentation
-        run: sbt generateSelfDocumentation
-        
-      - name: Generate documentation for dotty library
-        run: sbt generateDottyLibDocumentation      
+        run: sbt scala3doc/generateSelfDocumentation
 
       - name: Generate documentation with SBT plugin
-        run: sbt example-project/doc
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-central-1
-
-      - name: Publish all
-        run: |
-              dest=s3://scala3doc/$([ $GITHUB_REF = "master" ] && echo master || echo pr-$(echo $GITHUB_REF | cut '-d/' -f3))
-              aws s3 rm $dest
-              aws s3 sync output $dest
-
-      - name: Update gh-pages
-        run: |
-             if [ $GITHUB_REF = "refs/heads/master" ]; then
-                git config --global user.email "$(git log -1 --format=%ae)"
-                git config --global user.name "$(git log -1 --format=%an)"
-                git subtree add --prefix=our-site origin gh-pages
-                rm -r our-site/*
-                cp -r  output/self/* our-site
-                git add our-site
-                git commit -m "$(git log -1 --pretty=%B)"
-                git subtree push --prefix=our-site origin gh-pages
-             fi
-
+        run: sbt scala3doc-example-project/doc

build.sbt

@@ -21,7 +21,8 @@ val `tasty-core` = Build.`tasty-core`
 val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
 val `tasty-core-scala2` = Build.`tasty-core-scala2`
 val scala3doc = Build.scala3doc
-val `scala3doc-example-project` = Build.`scala3doc-example-project`
+val scala3docTest = Build.`scala3doc-test`
+// val `scala3doc-example-project` = Build.`scala3doc-example-project`
 val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`

project/Build.scala

@@ -136,11 +136,6 @@ object Build {
 
   val fetchScalaJSSource = taskKey[File]("Fetch the sources of Scala.js")
 
-  // Scala3doc specific tasks
-  val buildDokkaApi = taskKey[File]("Compile dokka wrapper and put jar in lib")
-  val generateSelfDocumentation = taskKey[Unit]("Generate example documentation")
-  val generateDottyLibDocumentation = taskKey[Unit]("Generate documentation for dotty lib")
-
   lazy val SourceDeps = config("sourcedeps")
 
   // Settings shared by the build (scoped in ThisBuild). Used in build.sbt
@@ -1169,6 +1164,7 @@ object Build {
   lazy val `scala3-bench-run` = project.in(file("bench-run")).asDottyBench(Bootstrapped)
 
   lazy val `scala3doc` = project.in(file("scala3doc")).asScala3doc
+  lazy val `scala3doc-test` = project.in(file("scala3doc-test")).asScala3docTest
   lazy val `scala3doc-example-project` = project.in(file("scala3doc-example-project")).asDocExampleProject
 
   // sbt plugin to use Dotty in your own build, see
@@ -1458,83 +1454,26 @@ object Build {
       settings(commonBenchmarkSettings).
       enablePlugins(JmhPlugin)
 
-    def asScala3doc: Project = {
-      val dokkaVersion = "1.4.10.2"
-      val kotlinxVersion = "0.7.2" // upgrade when upgrading dokka
-      val flexmarkVersion = "0.42.12"
-      val jacksonVersion = "2.9.8"
-      val scalaTagsVersion = "0.9.1"
-      val dokkaSiteVersion = "0.1.9"
+    def commonScala3DocSettings = commonBootstrappedSettings ++ Seq(
+      scalaVersion := dottyVersion,
+      resolvers += Resolver.jcenterRepo,
+      resolvers += Resolver.bintrayRepo("kotlin", "kotlin-dev"),
+      resolvers += Resolver.bintrayRepo("virtuslab", "dokka"),
+      // hack, we cannot build documentation so we need this to publish locally
+      publishArtifact in (Compile, packageDoc) := false
+    )
 
+    def asScala3doc: Project = 
       project.
-        settings(commonBootstrappedSettings).
+        settings(commonScala3DocSettings).
         dependsOn(`scala3-compiler-bootstrapped`).
-        dependsOn(`scala3-tasty-inspector`).
-        settings(
-          Compile/scalaSource := baseDirectory.value / "src/main/scala",
-          Test/scalaSource := baseDirectory.value / "src/test/scala",
-          Compile/resourceDirectory := baseDirectory.value / "src/main/resources",
-          Test/resourceDirectory := baseDirectory.value / "src/test/resources",
-
-          scalaVersion := dottyVersion,
-          resolvers ++= Seq(
-            Resolver.jcenterRepo,
-            Resolver.bintrayRepo("kotlin", "kotlin-dev"),
-            Resolver.bintrayRepo("virtuslab", "dokka"),
-          ),
-          libraryDependencies ++= Seq(
-            "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion % "test",
-
-            "org.scala-lang" %% "scala3-tasty-inspector" % dottyVersion,
-
-            "org.jetbrains.dokka" % "dokka-base" % dokkaVersion,
-            "org.jetbrains.dokka" % "dokka-core" % dokkaVersion,
-            "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion,
-            "org.jetbrains.kotlinx" % "kotlinx-html-jvm" % kotlinxVersion,
-            "com.virtuslab.dokka" % "dokka-site" % dokkaSiteVersion,
-
-            "org.scala-sbt" % "io_2.13" % "1.3.4",
-
-            "com.vladsch.flexmark" % "flexmark-all" % flexmarkVersion,
-            "com.lihaoyi" % "scalatags_2.13" % scalaTagsVersion,
-            "nl.big-o" % "liqp" % "0.6.7",
-            "args4j" % "args4j" % "2.33",
-          ),
-
-          run / fork := true,
-          Test / fork  := true,
-          Test / parallelExecution := false,
-          Compile / mainClass := Some("dotty.dokka.Main"),
-
-          Test / envVars := Map(
-            "scala3doc.classroot" -> (Compile/target/classDirectory).value.getAbsolutePath.toString,
-          ),
-
-          // hack, we cannot build documentation so we need this to publish locally
-          Compile / packageDoc / publishArtifact := false,
-
-          generateSelfDocumentation := Def.taskDyn {
-            val classdir = (Compile/target/classDirectory).value
-            (Compile/run).toTask(s" -o output/self -t ${classdir} -d documentation -n scala3doc -s src/main/scala=https://github.com/lampepfl/scala3doc/tree/master/src/main/scala#L")
-          }.value,
-
-          generateDottyLibDocumentation := Def.taskDyn {
-            val dottyLib = (Compile/fullClasspath).value.find{ a =>
-              val info = a.get(moduleID.key)
-              info.nonEmpty &&
-              info.get.organization == "ch.epfl.lamp" &&
-              info.get.name.startsWith("dotty-library")
-            }
-            if (dottyLib.isEmpty) Def.task {
-              streams.value.log.error("Dotty lib wasn't found")
-            } else Def.task {
-              (Compile/run).toTask(s" -o output/stdLib -t ${dottyLib.get.data} -d dotty-docs/docs -n dotty-lib -s library/src=https://github.com/lampepfl/dotty/tree/master/library/src#L").value
-            }
-          }.value,
-
-        )
-    }
+        dependsOn(`scala3-tasty-inspector`)
 
+    def asScala3docTest: Project = 
+      project.
+        settings(commonScala3DocSettings).
+        dependsOn(`scala3doc`)
+    
     def asDocExampleProject: Project = project.
       settings(commonBootstrappedSettings0).
       dependsOn(`scala3-compiler-bootstrapped`).

scala3doc-test/build.sbt

@@ -0,0 +1,30 @@
+val dokkaVersion = "1.4.10.2"
+
+libraryDependencies ++= Seq(
+  "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion % "test", // TODO move testing utils to dokka-site
+  "com.novocode" % "junit-interface" % "0.11" % "test",
+)
+
+val testOuputDir = taskKey[String]("Root directory where tests classses are generated")
+testOuputDir := (Compile/target/classDirectory).value.getAbsolutePath.toString
+
+val testSourceRoot = taskKey[String]("Root directory where tests sources are generated")
+testSourceRoot := (baseDirectory.value / "src").getAbsolutePath.toString
+
+
+buildInfoKeys in Test := Seq[BuildInfoKey](
+  testOuputDir,
+  testSourceRoot,
+)
+buildInfoPackage in Test := "dotty.dokka"
+BuildInfoPlugin.buildInfoScopedSettings(Test)
+BuildInfoPlugin.buildInfoDefaultSettings
+
+// Uncomment to debug dokka processing (require to run debug in listen mode on 5005 port)
+// javaOptions.in(run) += "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"
+
+fork.in(run) := true
+
+// There is a bug in dokka that prevents parallel tests withing the same jvm
+fork.in(test) := true
+Test / parallelExecution := false

scala3doc/src/main/scala/example/Documentation2.scala renamed to scala3doc-test/src/example/Documentation2.scala

@@ -55,6 +55,7 @@ abstract class DottyAbstractCoreTest extends AbstractCoreTest:
         }
 
         def listTastyFiles(f: File): Seq[File] = 
+            assertTrue(s"Tasty root dir does not exisits: $f", f.isDirectory())
             val (files, dirs) = f.listFiles().partition(_.isFile)
             files.toIndexedSeq.filter(_.getName.endsWith(".tasty")) ++ dirs.flatMap(listTastyFiles)
 

scala3doc/src/main/scala/example/Inheritance.scala renamed to scala3doc-test/src/example/Inheritance.scala

@@ -51,15 +51,15 @@ abstract class MultipleFileTest(val sourceFiles: List[String], val tastyFolders:
         def cleanup(s: String) = s.replace("\n", " ").replaceAll(" +", " ")
 
         val allFromSource = sourceFiles.map{ file => 
-            val all = signaturesFromSource(Source.fromFile(s"src/main/scala/tests/$file.scala"))
+            val all = signaturesFromSource(Source.fromFile(s"${BuildInfo.testSourceRoot}/tests/$file.scala"))
             (all.expected, all.unexpected)
         }
 
         val expectedFromSource = allFromSource.map(_._1).flatten.filter(extractSymbolName(_) != "NULL").map(cleanup)
         val unexpectedFromSource = allFromSource.map(_._2).flatten.filter(extractSymbolName(_) != "NULL").map(cleanup)
         val unexpectedSignatureSymbolNames = unexpectedFromSource.map(extractSymbolName)
 
-        val allFromDocumentation = tastyFolders.flatMap(folder => signaturesFromDocumentation(s"target/scala-0.27/classes/tests/$folder"))
+        val allFromDocumentation = tastyFolders.flatMap(folder => signaturesFromDocumentation(s"${BuildInfo.testOuputDir}/tests/$folder"))
         val fromDocumentation = allFromDocumentation.filter(extractSymbolName(_) != "NULL").map(cleanup)
 
         val documentedSignatures = fromDocumentation.flatMap(matchSignature(_, expectedFromSource)).toSet

scala3doc/src/main/scala/example/level2/Documentation.scala renamed to scala3doc-test/src/example/level2/Documentation.scala

@@ -4,6 +4,7 @@ import scala.tasty.Reflection
 
 import org.junit.{Test, Rule}
 import org.junit.Assert.{assertSame, assertTrue}
+import dotty.dokka.BuildInfo
 
 class LookupTestCases[R <: Reflection](val r: R) {
 
@@ -122,7 +123,7 @@ class MemberLookupTests {
     import java.io.File
     import scala.collection.mutable.ListBuffer
 
-    val classRoot = File("target/scala-0.27/classes")
+    val classRoot = new File(BuildInfo.testOuputDir)
 
     def go(bld: ListBuffer[String])(file: File): Unit =
       file.listFiles.foreach { f =>

scala3doc/src/main/scala/example/level2/SameLevelTypeLinking.scala renamed to scala3doc-test/src/example/level2/SameLevelTypeLinking.scala

@@ -1,59 +1,33 @@
-// re-expose subproject settings
-val `example-project` = ExampleProject.`example-project`
-
-val dottyVersion = "3.0.0-M1-bin-SNAPSHOT"
-val dokkaVersion = "1.4.10.2"
 val flexmarkVersion = "0.42.12"
 val jacksonVersion = "2.9.8"
 val scalaTagsVersion = "0.9.1"
 val dokkaSiteVersion = "0.1.9"
 
+
 libraryDependencies ++= Seq(
-  "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion % "test", // TODO move testing utils to dokka-site
-  "com.virtuslab.dokka" % "dokka-site" % dokkaSiteVersion,
+  "org.scala-lang" %% "scala3-tasty-inspector" % scalaVersion.value,
 
-  "org.scala-lang" %% "scala3-tasty-inspector" % dottyVersion,
-  "org.scala-lang" %% "scala3-compiler" % dottyVersion,
-  "org.scala-sbt" % "io_2.13" % "1.3.4",
 
+  "com.virtuslab.dokka" % "dokka-site" % dokkaSiteVersion,
+  "org.scala-sbt" % "io_2.13" % "1.3.4",
   "com.vladsch.flexmark" % "flexmark-all" % flexmarkVersion,
   "com.lihaoyi" % "scalatags_2.13" % scalaTagsVersion,
   "nl.big-o" % "liqp" % "0.6.7",
   "args4j" % "args4j" % "2.33",
-  "com.novocode" % "junit-interface" % "0.11" % "test",
-)
-
-resolvers += Resolver.jcenterRepo
-resolvers += Resolver.bintrayRepo("kotlin", "kotlin-dev")
-resolvers += Resolver.bintrayRepo("virtuslab", "dokka")
-
-lazy val root = project
-  .in(file("."))
-  .settings(
-    name := "scala3doc",
-    version := "0.1.1-SNAPSHOT",
-    scalaVersion := dottyVersion
-  )
+) //a a
 
 val generateSelfDocumentation = inputKey[Unit]("Generate example documentation")
 generateSelfDocumentation := {
   run.in(Compile).fullInput(" -o output/self -t target/scala-0.27/classes -d documentation -n scala3doc -s src/main/scala=https://github.com/lampepfl/scala3doc/tree/master/src/main/scala#L").evaluated // TODO #35 proper sbt integration
 }
 
+
 // Uncomment to debug dokka processing (require to run debug in listen mode on 5005 port)
 // javaOptions.in(run) += "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"
 
 fork.in(run) := true
-// There is a bug in dokka that prevents parallel tests withing the same jvm
-fork.in(test) := true
-Test / parallelExecution := false
-
-scalacOptions in Compile += "-language:implicitConversions"
-
 Compile / mainClass := Some("dotty.dokka.Main")
 
-// hack, we cannot build documentation so we need this to publish locally
-publishArtifact in (Compile, packageDoc) := false
 
 // TODO #35 proper sbt integration
 val generateDottyLibDocumentation = taskKey[Unit]("Generate documentation for dotty lib")