Multi user/repo/branch for jenkins & scabot.

README.md

@@ -280,7 +280,15 @@ knife vault create master scabot-keypair \
   --admins adriaan
 
 knife vault create master scabot \
-  --json $PWD/.chef/scabot.json \
+  {
+    "jenkins": {
+      "token": "..."
+    },
+    "github": {
+      "scala":    {"token": "..."}
+      "lampepfl": {"token": "..."}
+    }
+  }
   --search 'name:jenkins-master' \
   --admins adriaan
 

attributes/default.rb

@@ -121,6 +121,5 @@
   default['master']['ec2-start-stop']['url'] = 'https://dl.dropboxusercontent.com/u/12862572/ec2-start-stop.hpi'
 
   # SCABOT
-  default['scabot']['jenkins']['user']     = "scala-jenkins"
-  default['scabot']['github']['repo_user'] = "scala"
+  default['scabot']['jenkins']['user'] = "scala-jenkins"
 end

libraries/job_blurbs.rb

@@ -144,12 +144,12 @@ def scmBlurb(refspec)
       EOH
     end
 
-    def versionedJob(version, name)
-      "scala-#{version}-#{name.gsub(/\//, '-')}"
+    def versionedJob(repo, branch, name)
+      "#{repo}-#{branch}-#{name.gsub(/\//, '-')}"
     end
 
     def job(name)
-      versionedJob(@version, name)
+      versionedJob(@repo, @branch, name)
     end
 
     def jvmSelect

recipes/_master-config-scabot.rb

@@ -59,14 +59,15 @@
   revision   "master"
 end
 
-# TODO: do not use node.set for sensitive stuff!!
-node.set['scabot']['github']['token']  = chef_vault_item("master", "scabot")['github']['token']
-node.set['scabot']['jenkins']['token'] = chef_vault_item("master", "scabot")['jenkins']['token']
-
 template "#{scabotHome}/scabot.conf" do
   source    'scabot.conf.erb'
   user      scabotUser
   sensitive true
+
+  variables({
+    :scabotVault => chef_vault_item("master", "scabot")
+  })
+
 end
 
 bash 'build scabot' do

recipes/_master-jenkins-jobs.rb

@@ -21,19 +21,21 @@ class Blurbs
 end
 
 # turn template path into jenkins job name
-def templDesc(version, path)
+def templDesc(user, repo, branch, path)
   blurbs = Blurbs.new
 
-  m = path.match(/templates\/default\/jobs\/(.*)\.xml\.erb$/)
+  m = path.match(/templates\/default\/jobs\/#{user}\/(.*)\.xml\.erb$/)
   if m == nil
     []
   else
     relativePath = m.captures.first
 
-    [ { :templatePath => "jobs/#{relativePath}.xml.erb",
+    [ { :templatePath => "jobs/#{user}/#{relativePath}.xml.erb",
         :scriptName   => "jobs/#{relativePath}",
-        :jobName      => blurbs.versionedJob(version, relativePath),
-        :version      => version
+        :jobName      => blurbs.versionedJob(repo, branch, relativePath),
+        :user         => user,
+        :repo         => repo, # the main repo (we may refer to other repos under the same user in these jobs)
+        :branch       => branch,
       }
     ]
   end
@@ -55,11 +57,9 @@ def templDesc(version, path)
 #     - 9
 # core-community: sbt, ensime, modules, ide,...
 
-# create scala-$version-$jobName for every template under jobs/
-# TODO #16: add 2.12.x jobs
-%w{ 2.11.x }.each do | version |
+def expandJobTemplates(user, repo, branch)
   node.run_context.cookbook_collection["scala-jenkins-infra"].manifest[:templates]
-    .flat_map { |mani| templDesc(version, mani['path']) }
+    .flat_map { |mani| templDesc(user, repo, branch, mani['path']) }
     .each do | desc |
 
     xml = File.join(Chef::Config[:file_cache_path], "#{desc[:jobName]}.xml")
@@ -77,6 +77,14 @@ def templDesc(version, path)
   end
 end
 
+# TODO: make consistent with scabot.conf.erb by construction
+# (each github user for which we create jobs should have a corresponding top-level section in scabot.conf)
+# create scala-$branch-$jobName for every template under jobs/
+%w{ 2.11.x 2.12.x }.each do | branch |
+  expandJobTemplates("scala", "scala", branch)
+end
+
+
 # TODO #10: make a view for each top-level directory under jobs/ that lists all jobs under it (scala-2.11.x-integrate, scala-2.11.x-release, scala-2.11.x-validate)
 # https://issues.jenkins-ci.org/browse/JENKINS-8927
 def viewXML(viewPrefix)
@@ -105,4 +113,4 @@ def viewXML(viewPrefix)
       <statusFilter>true</statusFilter>
     </listView>
   EOH
-end
+end

recipes/master-init.rb

@@ -10,8 +10,11 @@
 
 include_recipe "java"
 
-# EBS -- must come before jenkins init since it mounts /var/lib/jenkins
-include_recipe "scala-jenkins-infra::_config-ebs"
+case node["platform"]
+when "amazon"
+  # EBS -- must come before jenkins init since it mounts /var/lib/jenkins
+  include_recipe "scala-jenkins-infra::_config-ebs"
+end
 
 include_recipe "scala-jenkins-infra::_master-init-jenkins"
 

recipes/worker-config.rb

@@ -12,7 +12,10 @@
 # since jenkins's home dir is mounted on ephemeral storage (see chef/userdata/ubuntu-publish-c3.xlarge)
 
 
-include_recipe "scala-jenkins-infra::_config-ebs"
+case node["platform"]
+when "amazon"
+  include_recipe "scala-jenkins-infra::_config-ebs"
+end
 
 node["jenkinsHomes"].each do |jenkinsHome, workerConfig|
   case node["platform_family"]

templates/default/jobs/integrate/bootstrap.xml.erb → templates/default/jobs/scala/integrate/bootstrap.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: %{ Stage a Scala distribution on sonatype using scripts/job/bootstrap.
     It builds locker and uses it to build and publish the required modules as well as the
     Scala compiler/library for this release, and generates an updated versions.properties

templates/default/jobs/integrate/ide.xml.erb → templates/default/jobs/scala/integrate/ide.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "PR integration testing: IDE",
   nodeRestriction: "public",
   params: [

templates/default/jobs/integrate/main.xml.erb → templates/default/jobs/scala/integrate/main.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <com.cloudbees.plugins.flow.BuildFlow plugin="[email protected]">
 <%= flowProject(
-    repoUser:    "scala",
+    repoUser:    @user,
     repoName:    "scala",
-    repoRef:     @version,
+    repoRef:     @branch,
     description: "PR integration -- verify mergeability of PR",
     params: [
       {:name => "prDryRun", :desc => ""},

templates/default/jobs/release/main.xml.erb → templates/default/jobs/scala/release/main.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <com.cloudbees.plugins.flow.BuildFlow plugin="[email protected]">
 <%= flowProject(
-    repoUser:    "scala",
-    repoRef:     @version,
-    description: "TODO: Better description. Build, package and stage a Scala release. \nNightly downloads go to http://www.scala-lang.org/files/archive/nightly/#{@version}/\nRegular ones to http://www.scala-lang.org/files/archive/.",
+    repoUser:    @user,
+    repoRef:     @branch,
+    description: "TODO: Better description. Build, package and stage a Scala release. \nNightly downloads go to http://www.scala-lang.org/files/archive/nightly/#{@branch}/\nRegular ones to http://www.scala-lang.org/files/archive/.",
     params: [
       {:name => "SCALA_VER_BASE", :desc => "When set, a release with version \"$SCALA_VER_BASE$SCALA_VER_SUFFIX\" will be built. Same effect as for tagging $repo_ref as \"v$SCALA_VER_BASE$SCALA_VER_SUFFIX\"."},
       {:name => "SCALA_VER_SUFFIX", :desc => ""}

templates/default/jobs/release/package/unix.xml.erb → templates/default/jobs/scala/release/package/unix.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala-dist",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "Run as part of the #{job("release/main")} flow.<p>" + %{Builds and uploads the universal and unix-specific artifacts for a
     Scala release.<br>
 

templates/default/jobs/release/package/windows.xml.erb → templates/default/jobs/scala/release/package/windows.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala-dist",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "Run as part of the #{job("release/main")} flow.<p>" + %{Builds and uploads the windows-specific artifacts for a Scala
     release.<br>
 

templates/default/jobs/release/smoketest.xml.erb → templates/default/jobs/scala/release/smoketest.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala-dist-smoketest",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "Run as part of the #{job("release/main")} flow." + %{ <br/> It runs the program in
     https://github.com/scala/scala-dist-smoketest/tree/${repo_ref} to make sure all JARs of the distribution
     contain classfiles. <br/> It downloads the archives for a Scala release from

templates/default/jobs/release/website/archives.xml.erb → templates/default/jobs/scala/release/website/archives.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala-dist",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "<H1>Run as part of the #{job("release/main")} flow.</H1>" +
    %{<p>It downloads the archives for a Scala release
        from http://downloads.typesafe.com/scala/$version/index.html and

templates/default/jobs/release/website/update-api.xml.erb → templates/default/jobs/scala/release/website/update-api.xml.erb

@@ -1,10 +1,10 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala-dist",
-  repoRef:     @version,
-  description: "Update scala-lang's api/#{@version} symlink to point to api/$version",
+  repoRef:     @branch,
+  description: "Update scala-lang's api/#{@branch} symlink to point to api/$version",
   nodeRestriction: "linux && publish",
   concurrent: false,
   params: [

templates/default/jobs/validate/main.xml.erb → templates/default/jobs/scala/validate/main.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <com.cloudbees.plugins.flow.BuildFlow plugin="[email protected]">
 <%= flowProject(
-    repoUser:    "scala",
+    repoUser:    @user,
     repoName:    "scala",
-    repoRef:     @version,
+    repoRef:     @branch,
     description: "PR validation -- called for every commit in every PR",
     params: [
       {:name => "prDryRun", :desc => ""},

templates/default/jobs/validate/publish-core.xml.erb → templates/default/jobs/scala/validate/publish-core.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "PR validation: publish core",
   nodeRestriction: "public",
   params: [

templates/default/jobs/validate/test.xml.erb → templates/default/jobs/scala/validate/test.xml.erb

@@ -1,9 +1,9 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
 <%= githubProject(
-  repoUser:    "scala",
+  repoUser:    @user,
   repoName:    "scala",
-  repoRef:     @version,
+  repoRef:     @branch,
   description: "PR validation: test suite",
   nodeRestriction: "public",
   params: [

templates/default/scabot.conf.erb

@@ -1,15 +1,15 @@
 scala: {
   jenkins: {
-    job:   "scala-2.11.x-validate-main"
-    jobPrefix: "scala-2.11.x-" // this will be stripped when reporting to github TODO: remove once it's computed as s"${github.repo}-${pull.base.ref}-"
+    jobSuffix: "validate-main"
     host:  "<%=node['master']['jenkinsHost']%>"
     user:  "<%=node['scabot']['jenkins']['user']%>"
-    token: "<%=node['scabot']['jenkins']['token']%>"
+    token: "<%=@scabotVault['jenkins']['token']%>"
   }
   github: {
-    user:  "<%=node['scabot']['github']['repo_user']%>"
     repo:  "scala"
+    branches: ["2.11.x", "2.12.x"]
     host:  "api.github.com"
-    token: "<%=node['scabot']['github']['token']%>"
+    user:  "scala"
+    token: "<%=@scabotVault['github']['scala']['token']%>"
   }
 }