Skip to content

Commit 16fd63b

Browse files
adriaanmadriaanm
authored
Merge pull request scala#5292 from szeiger/wip/sbt-windows
Switch Windows CI build to sbt (w/ some sbt build improvements) [ci: last-only]
2 parents efb74b7 + faa4110 commit 16fd63b

File tree

8 files changed

+102
-65
lines changed

8 files changed

+102
-65
lines changed

‎build.sbt

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -768,6 +768,32 @@ lazy val root: Project = (project in file("."))
768768
GenerateAnyVals.run(dir.getAbsoluteFile)
769769
state
770770
},
771+
testAll := {
772+
val results = ScriptCommands.sequence[Result[Unit]](List(
773+
(Keys.test in Test in junit).result,
774+
(testOnly in IntegrationTest in testP).toTask(" -- run pos neg jvm").result,
775+
(testOnly in IntegrationTest in testP).toTask(" -- res scalap specialized scalacheck").result,
776+
(testOnly in IntegrationTest in testP).toTask(" -- instrumented presentation").result,
777+
(testOnly in IntegrationTest in testP).toTask(" -- --srcpath scaladoc").result,
778+
(Keys.test in Test in osgiTestFelix).result,
779+
(Keys.test in Test in osgiTestEclipse).result,
780+
(MiMa.mima in library).result,
781+
(MiMa.mima in reflect).result,
782+
Def.task(()).dependsOn( // Run these in parallel:
783+
doc in Compile in library,
784+
doc in Compile in reflect,
785+
doc in Compile in compiler,
786+
doc in Compile in scalap
787+
).result
788+
)).value
789+
val failed = results.map(_.toEither).collect { case Left(i) => i }
790+
if(failed.nonEmpty) {
791+
val log = streams.value.log
792+
log.error(s"${failed.size} of ${results.length} test tasks failed:")
793+
failed.foreach(i => log.error(s" - $i"))
794+
throw new RuntimeException
795+
}
796+
},
771797
antStyle := false,
772798
incOptions := incOptions.value.withNameHashing(!antStyle.value).withAntStyle(antStyle.value)
773799
)
@@ -837,6 +863,7 @@ lazy val buildDirectory = settingKey[File]("The directory where all build produc
837863
lazy val mkBin = taskKey[Seq[File]]("Generate shell script (bash or Windows batch).")
838864
lazy val mkQuick = taskKey[File]("Generate a full build, including scripts, in build/quick")
839865
lazy val mkPack = taskKey[File]("Generate a full build, including scripts, in build/pack")
866+
lazy val testAll = taskKey[Unit]("Run all test tasks sequentially")
840867

841868
// Defining these settings is somewhat redundant as we also redefine settings that depend on them.
842869
// However, IntelliJ's project import works better when these are set correctly.

‎project/ScriptCommands.scala

Lines changed: 27 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,29 @@ object ScriptCommands {
77
def all = Seq(
88
setupPublishCore,
99
setupValidateTest,
10-
setupBootstrapStarr, setupBootstrapLocker, setupBootstrapQuick, setupBootstrapPublish,
11-
testAll
10+
setupBootstrapStarr, setupBootstrapLocker, setupBootstrapQuick, setupBootstrapPublish
1211
)
1312

14-
/** Set up the environment for `validate/publish-core`. The argument is the Artifactory snapshot repository URL. */
15-
def setupPublishCore = setup("setupPublishCore") { case Seq(url) =>
13+
/** Set up the environment for `validate/publish-core`.
14+
* The optional argument is the Artifactory snapshot repository URL. */
15+
def setupPublishCore = setup("setupPublishCore") { args =>
1616
Seq(
1717
baseVersionSuffix in Global := "SHA-SNAPSHOT"
18-
) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
18+
) ++ (args match {
19+
case Seq(url) => publishTarget(url)
20+
case Nil => Nil
21+
}) ++ noDocs ++ enableOptimizer
1922
}
2023

21-
/** Set up the environment for `validate/test`. The argument is the Artifactory snapshot repository URL. */
22-
def setupValidateTest = setup("setupValidateTest") { case Seq(url) =>
23-
//TODO When ant is gone, pass starr version as an argument to this command instead of using version.properties
24+
/** Set up the environment for `validate/test`.
25+
* The optional argument is the Artifactory snapshot repository URL. */
26+
def setupValidateTest = setup("setupValidateTest") { args =>
2427
Seq(
25-
resolvers in Global += "scala-pr" at url,
2628
testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
27-
) ++ enableOptimizer
29+
) ++ (args match {
30+
case Seq(url) => Seq(resolvers in Global += "scala-pr" at url)
31+
case Nil => Nil
32+
}) ++ enableOptimizer
2833
}
2934

3035
/** Set up the environment for building STARR in `validate/bootstrap`. The arguments are:
@@ -63,9 +68,7 @@ object ScriptCommands {
6368
/** Set up the environment for publishing in `validate/bootstrap`. The arguments are:
6469
* - Temporary bootstrap repository URL for resolving modules
6570
* - Version number to publish
66-
* All artifacts are published to Sonatype. GPG signing has to be configured from the
67-
* shell script after `setupBootstrapPublish` because we don't pull the GPG plugin in
68-
* by default, so we cannot reference its keys statically. */
71+
* All artifacts are published to Sonatype. */
6972
def setupBootstrapPublish = setup("setupBootstrapPublish") { case Seq(url, ver) =>
7073
// Define a copy of the setting key here in case the plugin is not part of the build
7174
val pgpPassphrase = SettingKey[Option[Array[Char]]]("pgp-passphrase", "The passphrase associated with the secret used to sign artifacts.", KeyRanks.BSetting)
@@ -95,25 +98,18 @@ object ScriptCommands {
9598
publishArtifact in (Compile, packageDoc) in ThisBuild := false
9699
)
97100

98-
private[this] def publishTarget(url: String) = Seq(
101+
private[this] def publishTarget(url: String) = {
99102
// Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
100-
publishTo in Global := Some("scala-pr-publish" at url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis)
101-
)
103+
val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis
104+
Seq(publishTo in Global := Some("scala-pr-publish" at url2))
105+
}
102106

103-
def testAll = Command.command("testAll") { state =>
104-
val cmds = Seq(
105-
"test",
106-
"partest run pos neg jvm",
107-
"partest res scalap specialized scalacheck",
108-
"partest instrumented presentation",
109-
"partest --srcpath scaladoc",
110-
"osgiTestFelix/test",
111-
"osgiTestEclipse/test",
112-
"library/mima",
113-
"reflect/mima",
114-
"doc"
115-
)
116-
state.log.info(cmds.mkString("Running all tests: \"", "\", \"", "\""))
117-
cmds ::: state
107+
/** Like `Def.sequential` but accumulate all results */
108+
def sequence[B](tasks: List[Def.Initialize[Task[B]]]): Def.Initialize[Task[List[B]]] = tasks match {
109+
case Nil => Def.task { Nil }
110+
case x :: xs => Def.taskDyn {
111+
val v = x.value
112+
sequence(xs).apply((t: Task[List[B]]) => t.map(l => v :: l))
113+
}
118114
}
119115
}

‎project/VersionUtil.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ object VersionUtil {
9494
}
9595

9696
private lazy val generateBuildCharacterPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task {
97-
writeProps(versionProperties.value.toMap, (baseDirectory in ThisBuild).value / "buildcharacter.properties")
97+
writeProps(versionProperties.value.toMap ++ versionProps, (baseDirectory in ThisBuild).value / "buildcharacter.properties")
9898
}
9999

100100
private def writeProps(m: Map[String, String], propFile: File): File = {

‎scripts/common

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,3 +154,23 @@ EOF
154154
fi
155155
popd
156156
}
157+
158+
# Generate a repositories file with all allowed repositories in our build environment.
159+
# Takes one optional argument, the private repository URL.
160+
# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
161+
function generateRepositoriesConfig() {
162+
jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
163+
sbtRepositoryConfig="$scriptsDir/sbt-repositories-config"
164+
echo > "$sbtRepositoryConfig" '[repositories]'
165+
if [ -n "$1" ]
166+
then
167+
echo >> "$sbtRepositoryConfig" " private-repo: $1"
168+
fi
169+
cat >> "$sbtRepositoryConfig" << EOF
170+
jcenter-cache: $jcenterCacheUrl
171+
typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
172+
sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
173+
maven-central
174+
local
175+
EOF
176+
}

‎scripts/jobs/integrate/bootstrap

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,8 @@
8585
moduleVersioning=${moduleVersioning-"versions.properties"}
8686

8787
publishPrivateTask=${publishPrivateTask-"publish"}
88-
publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
89-
publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
90-
publishStarrPrivateTask=${publishStarrPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built STARR before)
91-
publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before)
88+
publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"}
89+
publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"}
9290

9391
forceRebuild=${forceRebuild-no}
9492

@@ -111,21 +109,7 @@ mkdir -p $baseDir/resolutionScratch_
111109
# repo used to publish "locker" scala to (to start the bootstrap)
112110
releaseTempRepoCred="private-repo"
113111
releaseTempRepoUrl=${releaseTempRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-release-temp/"}
114-
jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
115-
116-
# Used below in sbtArgs since we use a dedicated repository to share artifcacts between jobs,
117-
# so we need to configure SBT to use these rather than its default, Maven Central.
118-
# See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
119-
sbtRepositoryConfig="$scriptsDir/repositories-scala-release"
120-
cat > "$sbtRepositoryConfig" << EOF
121-
[repositories]
122-
private-repo: $releaseTempRepoUrl
123-
jcenter-cache: $jcenterCacheUrl
124-
typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
125-
sbt-plugin-releases: https://dl.bintray.com/sbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
126-
maven-central
127-
local
128-
EOF
112+
generateRepositoriesConfig $releaseTempRepoUrl
129113

130114
##### git
131115
gfxd() {
@@ -575,7 +559,7 @@ publishSonatype() {
575559
-Dstarr.version=$SCALA_VER \
576560
${updatedModuleVersions[@]} \
577561
"setupBootstrapPublish $releaseTempRepoUrl $SCALA_VER" \
578-
publishSigned | grep -v "was too long to be displayed in the webview, and will be left out"
562+
$publishSonatypeTaskCore | grep -v "was too long to be displayed in the webview, and will be left out"
579563

580564
echo "### Publishing modules to sonatype"
581565
# build/test/publish scala core modules to sonatype (this will start a new staging repo)

‎scripts/jobs/integrate/windows

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
#!/bin/bash
22

3-
./pull-binary-libs.sh
4-
5-
export ANT_OPTS="-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M"
6-
7-
# TODO: don't hardcode these paths -- should be in scala/scala-jenkins-infra, passed in through env vars from jenkins
8-
export PATH='/cygdrive/c/Program Files/Java/jdk1.8.0_92/bin:/cygdrive/c/apache-ant-1.9.6/bin:/cygdrive/c/Program Files (x86)/Git-2.5.3/Cmd:/bin:/usr/bin:'
9-
export JAVA_HOME='C:/Program Files/Java/jdk1.8.0_92'
3+
baseDir=${WORKSPACE-`pwd`}
4+
scriptsDir="$baseDir/scripts"
5+
. $scriptsDir/common
106

117
java -version
128
javac -version
13-
ant -version
149

15-
ant test-opt
10+
generateRepositoriesConfig
11+
12+
SBT="java $JAVA_OPTS -Dsbt.ivy.home=$WORKSPACE/.ivy2 -jar $sbtLauncher -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
13+
14+
# Build locker with STARR
15+
$SBT --warn "setupPublishCore" generateBuildCharacterPropertiesFile publishLocal
16+
17+
# Build quick and run the tests
18+
parseScalaProperties buildcharacter.properties
19+
$SBT -Dstarr.version=$maven_version_number --warn "setupValidateTest" testAll | grep -v "was too long to be displayed in the webview, and will be left out"

‎scripts/jobs/validate/publish-core

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,17 @@ baseDir=${WORKSPACE-`pwd`}
99
scriptsDir="$baseDir/scripts"
1010
. $scriptsDir/common
1111

12+
generateRepositoriesConfig $prRepoUrl
13+
SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
14+
1215
case $prDryRun in
1316
yep)
1417
echo "DRY RUN"
1518
mkdir -p build/pack ; mkdir -p dists/maven/latest
1619
;;
1720
*)
1821
echo ">>> Getting Scala version number."
19-
$SBT_CMD --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile
22+
$SBT --warn "setupPublishCore $prRepoUrl" generateBuildCharacterPropertiesFile
2023
parseScalaProperties buildcharacter.properties # produce maven_version_number
2124

2225
echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl."
@@ -27,7 +30,7 @@ case $prDryRun in
2730
if $libraryAvailable && $reflectAvailable && $compilerAvailable; then
2831
echo "Scala core already built!"
2932
else
30-
$SBT_CMD --warn "setupPublishCore $prRepoUrl" publish
33+
$SBT --warn "setupPublishCore $prRepoUrl" publish
3134
fi
3235

3336
mv buildcharacter.properties jenkins.properties # parsed by the jenkins job

‎scripts/jobs/validate/test

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ baseDir=${WORKSPACE-`pwd`}
44
scriptsDir="$baseDir/scripts"
55
. $scriptsDir/common
66

7+
generateRepositoriesConfig $prRepoUrl
8+
SBT="$SBT_CMD -Dsbt.override.build.repos=true -Dsbt.repository.config=$sbtRepositoryConfig"
9+
710
case $prDryRun in
811

912
yep)
@@ -15,12 +18,12 @@ case $prDryRun in
1518
# build quick using STARR built upstream, as specified by scalaVersion
1619
# (in that sense it's locker, since it was built with starr by that upstream job);
1720
# and run JUnit tests, partest, OSGi tests, MiMa and scaladoc
18-
$SBT_CMD \
21+
$SBT \
1922
-Dstarr.version=$scalaVersion \
2023
--warn \
2124
"setupValidateTest $prRepoUrl" \
2225
$testExtraArgs \
23-
testAll
26+
testAll | grep -v "was too long to be displayed in the webview, and will be left out"
2427

2528
;;
2629

0 commit comments

Comments
 (0)