Merge branch 'master' into ep/build-src-fix

Author: emilypgoogle

.github/workflows/build-release-artifacts.yml

@@ -16,7 +16,7 @@ jobs:
 
       - name: Perform gradle build
         run: |
-          ./gradlew  firebasePublish -PprojectsToPublish=firebase-firestore -PpublishMode=RELEASE -PincludeFireEscapeArtifacts=true
+          ./gradlew firebasePublish -PpublishConfigFilePath=release.cfg -PpublishMode=RELEASE -PincludeFireEscapeArtifacts=true
       - name: Upload generated artifacts
         uses: actions/upload-artifact@v2
         with:

.github/workflows/ci_tests.yml

@@ -122,7 +122,7 @@ jobs:
           FTL_RESULTS_DIR: ${{ github.event_name == 'pull_request' && format('pr-logs/pull/{0}/{1}/{2}/{3}_{4}/artifacts/', github.repository, github.event.pull_request.number, github.job, github.run_id, github.run_attempt) || format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt)}}
           FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
         run: |
-          ./gradlew ${{matrix.module}}:deviceCheck withErrorProne -PuseProdBackendForTests=true
+          ./gradlew ${{matrix.module}}:deviceCheck withErrorProne -PtargetBackend="prod"
 
   publish-test-results:
     name: "Publish Tests Results"

.github/workflows/create_releases.yml

@@ -7,6 +7,10 @@ on:
         description: 'Release name'
         required: true
         type: string
+      past-name:
+        description: 'Past release name'
+        required: true
+        type: string
 
 jobs:
   create-branches:
@@ -18,32 +22,24 @@ jobs:
         uses: peterjgrainger/action-create-branch@c2800a3a9edbba2218da6861fa46496cf8f3195a
         with:
           branch: 'releases/${{ inputs.name }}'
-      - name: Create release branch
-        uses: peterjgrainger/action-create-branch@c2800a3a9edbba2218da6861fa46496cf8f3195a
-        with:
-          branch: 'releases/${{ inputs.name }}.release'
 
   create-pull-request:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Create release configuration template
         run: |
-          git config user.name 'Create Release GA'
-          git config user.email '[email protected]'
-          echo "[release]" > release.cfg
-          echo "name = ${{ inputs.name }}" >> release.cfg
-          echo "mode = RELEASE" >> release.cfg
-          echo "" >> release.cfg
-          echo "[modules]" >> release.cfg
-          echo "" >> release.cfg
-          git add release.cfg
-          git commit -a -m 'Create release config'
+          ./gradlew generateReleaseConfig -PcurrentRelease=${{ inputs.name }} -PpastRelease=${{ inputs.past-name }} -PprintOutput=true
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v4
         with:
           base: 'releases/${{ inputs.name }}'
           branch: 'releases/${{ inputs.name }}.release'
+          add-paths: release.cfg
           title: '${{ inputs.name}} release'
+          body: 'Auto-generated PR for release ${{ inputs.name}}'
+          commit-message: 'Create release config for ${{ inputs.name }}'

.github/workflows/diff-javadoc.yml

@@ -31,7 +31,8 @@ jobs:
           echo "run=$(cat modules.json | sed "s/[]\"[]//g" | sed "s/,/\n/g" | xargs printf -- "%s:kotlinDoc ")" >> $GITHUB_OUTPUT
 
       - name: Build
-        run: ./gradlew ${{ steps.changed-modules.outputs.run }}
+        # Certain SDKs won't export docs, make a blank folder to diff if that's the case
+        run: mkdir build && ./gradlew ${{ steps.changed-modules.outputs.run }}
 
       - name: Move original docs
         run: mv build ~/diff/modified
@@ -41,7 +42,8 @@ jobs:
           ref: ${{ github.base_ref }}
 
       - name: Build
-        run: ./gradlew ${{ steps.changed-modules.outputs.run }}
+        # Certain SDKs won't export docs, make a blank folder to diff if that's the case
+        run: mkdir build && ./gradlew ${{ steps.changed-modules.outputs.run }}
 
       - name: Move modified docs
         run: mv build ~/diff/original
@@ -62,6 +64,7 @@ jobs:
           > diff.md
 
       - name: Add comment
+        continue-on-error: true
         uses: mshick/add-pr-comment@a65df5f64fc741e91c59b8359a4bc56e57aaf5b1
         with:
           message-path: diff.md

buildSrc/build.gradle.kts

@@ -58,6 +58,8 @@ dependencies {
     implementation("org.eclipse.aether:aether-transport-http:1.0.0.v20140518")
     implementation("org.eclipse.aether:aether-transport-wagon:1.0.0.v20140518")
     implementation("org.apache.maven:maven-aether-provider:3.1.0")
+    
+    implementation("org.eclipse.jgit:org.eclipse.jgit:6.3.0.202209071007-r")
 
     implementation("com.google.code.gson:gson:2.8.9")
     implementation("com.android.tools.build:gradle:7.2.2")

buildSrc/src/main/java/com/google/firebase/gradle/MultiProjectReleasePlugin.java

@@ -80,6 +80,8 @@ public void apply(Project project) {
               task.getDestinationDirectory().set(project.getRootDir());
             });
 
+    project.getTasks().create("generateReleaseConfig", ReleaseGenerator.class);
+
     project
         .getGradle()
         .projectsEvaluated(

buildSrc/src/main/java/com/google/firebase/gradle/ReleaseGenerator.kt

@@ -0,0 +1,125 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.firebase.gradle
+
+import java.io.File
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.api.ListBranchCommand
+import org.eclipse.jgit.api.errors.GitAPIException
+import org.eclipse.jgit.lib.Constants
+import org.eclipse.jgit.lib.ObjectId
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.TaskAction
+
+data class FirebaseLibrary(val moduleNames: List<String>, val directories: List<String>)
+
+open class ReleaseGenerator : DefaultTask() {
+  @TaskAction
+  @Throws(Exception::class)
+  fun generateReleaseConfig() {
+    val currentRelease = project.property("currentRelease").toString()
+    val pastRelease = project.property("pastRelease").toString()
+    val printReleaseConfig = project.property("printOutput").toString().toBoolean()
+    val rootDir = project.rootDir
+    val availableModules = parseSubProjects(rootDir)
+    val firebaseLibraries = extractLibraries(availableModules, rootDir)
+
+    val repo = Git.open(rootDir)
+    val headRef = repo.repository.resolve(Constants.HEAD)
+    val branchRef = getObjectRefForBranchName(repo, pastRelease)
+
+    val changedLibraries = getChangedLibraries(repo, branchRef, headRef, firebaseLibraries)
+    writeReleaseConfig(rootDir, changedLibraries, currentRelease)
+    if (printReleaseConfig) {
+      println(changedLibraries.joinToString(",", "LIBRARIES TO RELEASE: "))
+    }
+  }
+
+  private fun extractLibraries(
+    availableModules: Set<String>,
+    rootDir: File
+  ): List<FirebaseLibrary> {
+    val nonKtxModules = availableModules.filter { !it.endsWith("ktx") }.toSet()
+    return nonKtxModules
+      .map { moduleName ->
+        val ktxModuleName = "$moduleName:ktx"
+
+        val moduleNames = listOf(moduleName, ktxModuleName).filter { availableModules.contains(it) }
+        val directories = moduleNames.map { it.replace(":", "/") }
+
+        FirebaseLibrary(moduleNames, directories)
+      }
+      .filter { firebaseLibrary ->
+        firebaseLibrary.directories.first().let { File(rootDir, "$it/gradle.properties").exists() }
+      }
+  }
+
+  private fun parseSubProjects(rootDir: File) =
+    File(rootDir, "subprojects.cfg")
+      .readLines()
+      .filterNot { it.startsWith("#") || it.isEmpty() }
+      .toSet()
+
+  @Throws(GitAPIException::class)
+  private fun getObjectRefForBranchName(repo: Git, branchName: String) =
+    repo
+      .branchList()
+      .setListMode(ListBranchCommand.ListMode.REMOTE)
+      .call()
+      .firstOrNull { it.name == "refs/remotes/origin/$branchName" }
+      ?.objectId
+      ?: throw RuntimeException("Could not find branch named $branchName")
+
+  private fun getChangedLibraries(
+    repo: Git,
+    previousReleaseRef: ObjectId,
+    currentReleaseRef: ObjectId,
+    libraries: List<FirebaseLibrary>
+  ) =
+    libraries
+      .filter { library ->
+        library.directories.any { checkDirChanges(repo, previousReleaseRef, currentReleaseRef, it) }
+      }
+      .flatMap { it.moduleNames }
+
+  private fun checkDirChanges(
+    repo: Git,
+    previousReleaseRef: ObjectId,
+    currentReleaseRef: ObjectId,
+    directory: String
+  ) =
+    repo
+      .log()
+      .addPath("$directory/")
+      .addRange(previousReleaseRef, currentReleaseRef)
+      .setMaxCount(1)
+      .call()
+      .iterator()
+      .hasNext()
+
+  private fun writeReleaseConfig(configPath: File, libraries: List<String>, releaseName: String) {
+    File(configPath, "release.cfg")
+      .writeText(
+        """
+                    [release]
+                    name = $releaseName
+                    mode = RELEASE
+                    
+                    [modules]
+                    ${libraries.joinToString("\n".padEnd(21, ' '))}
+                """
+          .trimIndent()
+      )
+  }
+}

buildSrc/src/main/java/com/google/firebase/gradle/plugins/publish/PublishingPlugin.java

@@ -16,9 +16,13 @@
 
 import com.google.firebase.gradle.plugins.FirebaseLibraryExtension;
 import java.io.File;
-import java.util.Arrays;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
@@ -41,12 +45,33 @@
  * <p><strong>Prepare a release</strong>
  *
  * <pre>
+ * ./gradlew -PpublishConfigFilePath=release.cfg
+ *           -PpublishMode=(RELEASE|SNAPSHOT) \
+ *           firebasePublish
+ * </pre>
+ *
+ * <pre>
  * ./gradlew -PprojectsToPublish="firebase-inappmessaging,firebase-inappmessaging-display"\
  *           -PpublishMode=(RELEASE|SNAPSHOT) \
  *           firebasePublish
  * </pre>
  *
  * <ul>
+ *   <li>{@code publishConfigFilePath} is the path to the configuration file from which to read the
+ *       list of projects to release. The file format should be consistent with Python's
+ *       configparser, and the list of projects should be in a section called "modules". If both
+ *       this, and {@code projectsToPublish} are specified, this property takes precedence. <br>
+ *       <br>
+ *       Example config file content:
+ *       <pre>
+ *         [release]
+ *         name = M126
+ *         ...
+ *         [modules]
+ *         firebase-database
+ *         firebase-common
+ *         firebase-firestore
+ *       </pre>
  *   <li>{@code projectsToPublish} is a list of projects to release separated by {@code
  *       projectsToPublishSeparator}(default: ","), these projects will have their versions depend
  *       on the {@code publishMode} parameter.
@@ -73,6 +98,7 @@ public PublishingPlugin() {}
   public void apply(Project project) {
     String projectNamesToPublish = getPropertyOr(project, "projectsToPublish", "");
     String projectsToPublishSeparator = getPropertyOr(project, "projectsToPublishSeparator", ",");
+    String publishConfigFilePath = getPropertyOr(project, "publishConfigFilePath", "");
     Mode publishMode = Enum.valueOf(Mode.class, getPropertyOr(project, "publishMode", "SNAPSHOT"));
 
     Task publishAllToLocal = project.task("publishAllToLocal");
@@ -83,8 +109,16 @@ public void apply(Project project) {
         .getGradle()
         .projectsEvaluated(
             gradle -> {
+              List<String> projectsNames;
+              if (!publishConfigFilePath.isEmpty()) {
+                projectsNames = readReleaseConfigFile(publishConfigFilePath);
+              } else {
+                projectsNames =
+                    List.of(projectNamesToPublish.split(projectsToPublishSeparator, -1));
+              }
+
               Set<FirebaseLibraryExtension> projectsToPublish =
-                  Arrays.stream(projectNamesToPublish.split(projectsToPublishSeparator, -1))
+                  projectsNames.stream()
                       .filter(name -> !name.isEmpty())
                       .map(
                           name ->
@@ -194,6 +228,19 @@ public void apply(Project project) {
             });
   }
 
+  private List<String> readReleaseConfigFile(String publishConfigurationFilePath) {
+    try (Stream<String> stream = Files.lines(Path.of(publishConfigurationFilePath))) {
+      return stream
+          .dropWhile((line) -> !line.equals("[modules]"))
+          // We need to skip the "[modules]" line since it's not dropped
+          .skip(1)
+          .collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new IllegalArgumentException(
+          "Error reading configuration file " + publishConfigurationFilePath, e);
+    }
+  }
+
   private static String getPropertyOr(Project p, String property, String defaultValue) {
     Object value = p.findProperty(property);
     if (value != null) {