Merge commit '297eeb3' into merge-2.11-may-12

Author: lrytz

build-ant-macros.xml

@@ -91,7 +91,7 @@
 
   <macrodef name="simple-javac">
     <attribute name="project"/>
-    <!-- project: asm/forkjoin -->
+    <!-- project: forkjoin -->
     <attribute name="args" default=""/>
     <attribute name="jar" default="yes"/>
     <sequential>
@@ -371,6 +371,10 @@
         <then>
           <mkdir dir="${build-pack.dir}/${@{project}.targetdir}"/>
           <pre/>
+          <!-- can't check if a fileset is defined, so we have an additional property -->
+          <if><not><isset property="pack.@{project}.include-jars.defined"/></not><then>
+            <fileset id="pack.@{project}.include-jars" dir="." excludes="**" />
+          </then></if>
           <if>
             <not>
               <equals arg1="@{manifest}" arg2=""/>
@@ -380,6 +384,7 @@
                 <!-- update="true" makes no difference on my machine, so starting from scratch-->
                 <jar-opts/>
                 <path refid="pack.@{project}.files"/>
+                <zipgroupfileset refid="pack.@{project}.include-jars"/>
               </jar>
             </then>
             <else>

build.sbt

@@ -61,6 +61,7 @@ val partestDep = withoutScalaLang("org.scala-lang.modules" %% "scala-partest" %
 val partestInterfaceDep = withoutScalaLang("org.scala-lang.modules" %% "scala-partest-interface" % "0.5.0")
 val junitDep = "junit" % "junit" % "4.11"
 val junitIntefaceDep = "com.novocode" % "junit-interface" % "0.11" % "test"
+val asmDep = "org.scala-lang.modules" % "scala-asm" % versionProps("scala-asm.version")
 val jlineDep = "jline" % "jline" % versionProps("jline.version")
 val antDep = "org.apache.ant" % "ant" % "1.9.4"
 val scalacheckDep = withoutScalaLang("org.scalacheck" %% "scalacheck" % "1.11.4")
@@ -169,17 +170,22 @@ lazy val compiler = configureAsSubproject(project)
   .settings(generatePropertiesFileSettings: _*)
   .settings(
     name := "scala-compiler",
-    libraryDependencies += antDep,
+    libraryDependencies ++= Seq(antDep, asmDep),
     // this a way to make sure that classes from interactive and scaladoc projects
     // end up in compiler jar (that's what Ant build does)
     // we need to use LocalProject references (with strings) to deal with mutual recursion
     mappings in Compile in packageBin :=
       (mappings in Compile in packageBin).value ++
+      dependencyClasses(
+        (externalDependencyClasspath in Compile).value,
+        modules = Set(asmDep),
+        keep = "*.class" || "scala-asm.properties",
+        streams.value.cacheDirectory) ++
       (mappings in Compile in packageBin in LocalProject("interactive")).value ++
       (mappings in Compile in packageBin in LocalProject("scaladoc")).value ++
       (mappings in Compile in packageBin in LocalProject("repl")).value,
     includeFilter in unmanagedResources in Compile := compilerIncludes)
-  .dependsOn(library, reflect, asm)
+  .dependsOn(library, reflect)
 
 lazy val interactive = configureAsSubproject(project)
   .settings(disableDocsAndPublishingTasks: _*)
@@ -209,8 +215,6 @@ lazy val actors = configureAsSubproject(project)
 
 lazy val forkjoin = configureAsForkOfJavaProject(project)
 
-lazy val asm = configureAsForkOfJavaProject(project)
-
 lazy val partestExtras = configureAsSubproject(Project("partest-extras", file(".") / "src" / "partest-extras"))
   .dependsOn(repl)
   .settings(clearSourceAndResourceDirectories: _*)
@@ -231,9 +235,9 @@ lazy val junit = project.in(file("test") / "junit")
   )
 
 lazy val partestJavaAgent = (project in file(".") / "src" / "partest-javaagent").
-  dependsOn(asm).
   settings(commonSettings: _*).
   settings(
+    libraryDependencies += asmDep,
     doc := file("!!! NO DOCS !!!"),
     publishLocal := {},
     publish := {},
@@ -249,13 +253,13 @@ lazy val partestJavaAgent = (project in file(".") / "src" / "partest-javaagent")
   )
 
 lazy val test = project.
-  dependsOn(compiler, interactive, actors, repl, scalap, partestExtras, partestJavaAgent, asm, scaladoc).
+  dependsOn(compiler, interactive, actors, repl, scalap, partestExtras, partestJavaAgent, scaladoc).
   configs(IntegrationTest).
   settings(disableDocsAndPublishingTasks: _*).
   settings(commonSettings: _*).
   settings(Defaults.itSettings: _*).
   settings(
-    libraryDependencies ++= Seq(partestDep, scalaXmlDep, partestInterfaceDep, scalacheckDep),
+    libraryDependencies ++= Seq(asmDep, partestDep, scalaXmlDep, partestInterfaceDep, scalacheckDep),
     unmanagedBase in Test := baseDirectory.value / "files" / "lib",
     unmanagedJars in Test <+= (unmanagedBase) (j => Attributed.blank(j)) map(identity),
     // no main sources
@@ -279,7 +283,7 @@ lazy val test = project.
   )
 
 lazy val root = (project in file(".")).
-  aggregate(library, forkjoin, reflect, compiler, asm, interactive, repl,
+  aggregate(library, forkjoin, reflect, compiler, interactive, repl,
     scaladoc, scalap, actors, partestExtras, junit).settings(
     sources in Compile := Seq.empty,
     onLoadMessage := """|*** Welcome to the sbt build definition for Scala! ***
@@ -310,7 +314,7 @@ def configureAsSubproject(project: Project): Project = {
 
 /**
  * Configuration for subprojects that are forks of some Java projects
- * we depend on. At the moment there are just two: asm and forkjoin.
+ * we depend on. At the moment there's just forkjoin.
  *
  * We do not publish artifacts for those projects but we package their
  * binaries in a jar of other project (compiler or library).
@@ -382,6 +386,50 @@ lazy val generateVersionPropertiesFileImpl: Def.Initialize[Task[File]] = Def.tas
   propFile
 }
 
+/**
+ * Extract selected dependencies to the `cacheDirectory` and return a mapping for the content.
+ * Heavily inspired by sbt-assembly (https://github.com/sbt/sbt-assembly/blob/0.13.0/src/main/scala/sbtassembly/Assembly.scala#L157)
+ */
+def dependencyClasses(dependencies: Classpath, modules: Set[ModuleID], keep: FileFilter, cacheDirectory: File): Seq[(File, String)] = {
+  val dependencyFiles: Seq[File] = dependencies.map(_.data).toSeq
+  val toInclude = dependencyFiles.filter(f => {
+    val p = f.getCanonicalPath
+    modules.exists(m => {
+      // works for both .m2 (org/scala-lang/modules/scala-asm/5.0.3-scala-3/scala-asm-5.0.3-scala-3.jar)
+      // and .ivy2 (org.scala-lang.modules/scala-asm/5.0.3-scala-3/bundles/scala-asm.jar)
+      val nameParts = m.organization.split('.').toSet + m.name + m.revision
+      nameParts.forall(p.contains)
+    })
+  })
+  assert(toInclude.forall(sbt.classpath.ClasspathUtilities.isArchive), s"Expected JAR files as dependencies: $toInclude")
+
+  val tempDir = cacheDirectory / "unpackedDependencies"
+
+  def sha1name(f: File): String     = bytesToSha1String(f.getCanonicalPath.getBytes("UTF-8"))
+  def sha1content(f: File): String  = bytesToSha1String(IO.readBytes(f))
+  def bytesToSha1String(bytes: Array[Byte]): String = {
+    val sha1 = java.security.MessageDigest.getInstance("SHA-1")
+    val hash = sha1.digest(bytes)
+    hash map {"%02x".format(_)} mkString
+  }
+
+  val jarDirs: Seq[File] = for (jar <- toInclude) yield {
+    val jarName = jar.getName
+    val hash = sha1name(jar) + "_" + sha1content(jar)
+    val jarNamePath = tempDir / (hash + ".jarName")
+    val dest = tempDir / hash
+    if (!jarNamePath.exists || IO.read(jarNamePath) != jar.getCanonicalPath) {
+      IO.delete(dest)
+      dest.mkdir()
+      IO.unzip(jar, dest)
+      IO.write(jarNamePath, jar.getCanonicalPath, IO.utf8, append = false)
+    }
+    dest
+  }
+
+  jarDirs.flatMap(dir => dir ** keep --- dir pair relativeTo(dir))
+}
+
 // Defining these settings is somewhat redundant as we also redefine settings that depend on them.
 // However, IntelliJ's project import works better when these are set correctly.
 def clearSourceAndResourceDirectories = Seq(Compile, Test).flatMap(config => inConfig(config)(Seq(

build.xml

@@ -165,7 +165,6 @@ TODO:
   <property name="build.dir"               value="${basedir}/build"/>
   <property name="build-deps.dir"          value="${build.dir}/deps"/>
   <property name="build-libs.dir"          value="${build.dir}/libs"/>
-  <property name="build-asm.dir"           value="${build-libs.dir}"/>
   <property name="build-forkjoin.dir"      value="${build-libs.dir}"/>
   <property name="build-locker.dir"        value="${build.dir}/locker"/>
   <property name="build-quick.dir"         value="${build.dir}/quick"/>
@@ -350,6 +349,11 @@ TODO:
       <prepareCross name="partest"/>
       <prepareCross name="scalacheck"/>
 
+      <artifact:dependencies pathId="asm.classpath" filesetId="asm.fileset">
+        <dependency groupId="org.scala-lang.modules" artifactId="scala-asm" version="${scala-asm.version}"/>
+      </artifact:dependencies>
+      <copy-deps project="asm"/>
+
       <!-- TODO: delay until absolutely necessary to allow minimal build, also move out partest dependency from scaladoc -->
       <artifact:dependencies pathId="partest.classpath" filesetId="partest.fileset" versionsId="partest.versions">
         <!-- uncomment the following if you're deploying your own partest locally -->
@@ -607,9 +611,7 @@ TODO:
     </then></if>
 
     <path id="forkjoin.classpath" path="${build-forkjoin.dir}/classes/forkjoin"/>
-    <path id="asm.classpath"      path="${build-asm.dir}/classes/asm"/>
     <property name="forkjoin-classes" refid="forkjoin.classpath"/>
-    <property name="asm-classes" refid="asm.classpath"/>
 
     <!-- the following properties fully define staged-docs, staged-pack, make-bundle, copy-bundle and mvn-package for each of the projects -->
     <property name="library.description"           value="Scala Standard Library"/>
@@ -662,7 +664,7 @@ TODO:
     <property name="partest-extras.description"    value="Scala Compiler Testing Tool (compiler-specific extras)"/>
     <property name="partest-javaagent.description" value="Scala Compiler Testing Tool (compiler-specific java agent)"/>
 
-    <!-- projects without project-specific options: asm, forkjoin, manual, bin, repl -->
+    <!-- projects without project-specific options: forkjoin, manual, bin, repl -->
     <for list="compiler,interactive,scaladoc,library,parser-combinators,partest,partest-extras,partest-javaagent,reflect,scalap,swing,xml" param="project">
       <sequential>
         <!-- description is mandatory -->
@@ -836,8 +838,9 @@ TODO:
       <fileset dir="${build-quick.dir}/classes/scaladoc"/>
       <fileset dir="${build-quick.dir}/classes/interactive"/>
       <fileset dir="${build-quick.dir}/classes/repl"/>
-      <fileset dir="${asm-classes}"/>
     </path>
+    <fileset id="pack.compiler.include-jars" refid="asm.fileset"/>
+    <property name="pack.compiler.include-jars.defined" value="yeah"/>
 
     <!-- TODO modularize compiler.
     <path id="pack.scaladoc.files">   <fileset dir="${build-quick.dir}/classes/scaladoc"/> </path>
@@ -1032,9 +1035,9 @@ TODO:
 <!-- ===========================================================================
                                   CLEANLINESS
 =============================================================================-->
-  <target name="libs.clean">                        <clean build="libs"/> <clean build="asm"/>                            </target>
-  <target name="quick.clean"  depends="libs.clean">   <clean build="quick"/> <clean build="pack"/> <clean build="strap"/> </target>
-  <target name="locker.clean" depends="quick.clean">    <clean build="locker"/>                                           </target>
+  <target name="libs.clean">                         <clean build="libs"/>                                               </target>
+  <target name="quick.clean"  depends="libs.clean">  <clean build="quick"/> <clean build="pack"/> <clean build="strap"/> </target>
+  <target name="locker.clean" depends="quick.clean"> <clean build="locker"/>                                             </target>
 
   <target name="docs.clean"> <clean build="docs"/> <delete dir="${build.dir}/manmaker" includeemptydirs="yes" quiet="yes" failonerror="no"/> </target>
   <target name="dist.clean"> <delete dir="${dists.dir}" includeemptydirs="yes" quiet="yes" failonerror="no"/> </target>
@@ -1052,7 +1055,6 @@ TODO:
                                   LOCAL DEPENDENCIES
 ============================================================================ -->
 
-  <target name="asm.done"      depends="init"> <simple-javac project="asm" jar="no"/> </target>
   <target name="forkjoin.done" depends="init"> <simple-javac project="forkjoin" args="-XDignore.symbol.file" jar="no"/></target>
 
   <!-- For local development only. We only allow released versions of Scala for STARR.
@@ -1075,7 +1077,7 @@ TODO:
 <!-- ===========================================================================
                                   LOCAL REFERENCE BUILD (LOCKER)
 ============================================================================ -->
-  <target name="locker.start"  depends="asm.done, forkjoin.done">
+  <target name="locker.start"  depends="forkjoin.done">
     <condition property="locker.locked"><available file="${build-locker.dir}/locker.locked"/></condition></target>
 
   <target name="locker.lib"    depends="locker.start"     unless="locker.locked">
@@ -1140,7 +1142,7 @@ TODO:
   <target name="pack.reflect" depends="quick.reflect"> <staged-pack project="reflect"/> </target>
 
   <!-- TODO modularize compiler. Remove other quick targets when they become modules. -->
-  <target name="pack.comp"    depends="quick.comp, quick.scaladoc, quick.interactive, quick.repl, asm.done">
+  <target name="pack.comp"    depends="quick.comp, quick.scaladoc, quick.interactive, quick.repl">
     <staged-pack project="compiler" manifest="${build-pack.dir}/META-INF/MANIFEST.MF">
       <pre> <!-- TODO the files copied here do not influence actuality of this target (nor does the manifest) -->
         <copy todir="${build-pack.dir}/lib">

scripts/jobs/integrate/bootstrap

@@ -114,17 +114,18 @@ mkdir -p $baseDir/resolutionScratch_
 # repo used to publish "locker" scala to (to start the bootstrap)
 releaseTempRepoCred="private-repo"
 releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"}
+jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
 
 # Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs,
 # so we need to configure SBT to use these rather than its default, Maven Central.
 # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
 sbtRepositoryConfig="$scriptsDir/repositories-scala-release"
 cat > "$sbtRepositoryConfig" << EOF
 [repositories]
-  plugins: http://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
   private-repo: $releaseTempRepoUrl
-  typesafe-ivy-releases: http://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
-  sbt-plugin-releases: http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
+  jcenter-cache: $jcenterCacheUrl
+  typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
+  sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
   maven-central
   local
 EOF