First commit, removed root files and moved configuration to BomGeneratorTask

Author: David Motsonashvili

buildSrc/build.gradle

@@ -42,11 +42,21 @@ dependencies {
     implementation "com.google.firebase:perf-plugin:$perfPluginVersion"
 
     implementation 'org.jsoup:jsoup:1.11.2'
+    implementation "com.google.auto.value:auto-value-annotations:1.6.6"
+    annotationProcessor "com.google.auto.value:auto-value:1.6.5"
     implementation 'digital.wup:android-maven-publish:3.6.2'
     implementation 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.72'
     implementation 'org.json:json:20180813'
     implementation 'io.opencensus:opencensus-api:0.18.0'
     implementation 'io.opencensus:opencensus-exporter-stats-stackdriver:0.18.0'
+    implementation "org.eclipse.aether:aether-api:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-util:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-impl:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-connector-basic:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-transport-file:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-transport-http:1.0.0.v20140518"
+    implementation "org.eclipse.aether:aether-transport-wagon:1.0.0.v20140518"
+    implementation "org.apache.maven:maven-aether-provider:3.1.0"
     runtime 'io.opencensus:opencensus-impl:0.18.0'
     implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.6'
     implementation 'org.jetbrains.dokka:dokka-android-gradle-plugin:0.9.17-g013-9b8280a'

buildSrc/src/main/java/com/google/firebase/gradle/bomgenerator/BomGeneratorTask.java

@@ -0,0 +1,285 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+package com.google.firebase.gradle.bomgenerator;
+
+import static java.util.stream.Collectors.toList;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.eclipse.aether.resolution.VersionRangeResolutionException;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.TaskAction;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import com.google.firebase.gradle.bomgenerator.model.Dependency;
+import com.google.firebase.gradle.bomgenerator.model.VersionBump;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+public class BomGeneratorTask extends DefaultTask {
+  private static final List<String> BOM_ARTIFACTS = ImmutableList.of(
+    "com.crashlytics.sdk.android:crashlytics",
+    "com.google.firebase:firebase-ads",
+    "com.google.firebase:firebase-ads-lite",
+    "com.google.firebase:firebase-analytics",
+    "com.google.firebase:firebase-analytics-ktx",
+    "com.google.firebase:firebase-appindexing",
+    "com.google.firebase:firebase-auth",
+    "com.google.firebase:firebase-auth-ktx",
+    "com.google.firebase:firebase-common",
+    "com.google.firebase:firebase-common-ktx",
+    "com.google.firebase:firebase-config",
+    "com.google.firebase:firebase-config-ktx",
+    "com.google.firebase:firebase-core",
+    "com.google.firebase:firebase-crash",
+    "com.google.firebase:firebase-crashlytics",
+    "com.google.firebase:firebase-crashlytics-ktx",
+    "com.google.firebase:firebase-crashlytics-ndk",
+    "com.google.firebase:firebase-database",
+    "com.google.firebase:firebase-database-ktx",
+    "com.google.firebase:firebase-dynamic-links",
+    "com.google.firebase:firebase-dynamic-links-ktx",
+    "com.google.firebase:firebase-encoders",
+    "com.google.firebase:firebase-firestore",
+    "com.google.firebase:firebase-firestore-ktx",
+    "com.google.firebase:firebase-functions",
+    "com.google.firebase:firebase-functions-ktx",
+    "com.google.firebase:firebase-iid",
+    "com.google.firebase:firebase-inappmessaging",
+    "com.google.firebase:firebase-inappmessaging-display",
+    "com.google.firebase:firebase-inappmessaging-display-ktx",
+    "com.google.firebase:firebase-inappmessaging-ktx",
+    "com.google.firebase:firebase-installations",
+    "com.google.firebase:firebase-installations-ktx",
+    "com.google.firebase:firebase-messaging",
+    "com.google.firebase:firebase-messaging-directboot",
+    "com.google.firebase:firebase-messaging-ktx",
+    "com.google.firebase:firebase-ml-model-interpreter",
+    "com.google.firebase:firebase-ml-vision",
+    "com.google.firebase:firebase-perf",
+    "com.google.firebase:firebase-perf-ktx",
+    "com.google.firebase:firebase-storage",
+    "com.google.firebase:firebase-storage-ktx");
+  private static final List<String> IGNORED_ARTIFACTS = ImmutableList.of(
+    "crash-plugin",
+    "firebase-abt",
+    "firebase-analytics-impl",
+    "firebase-analytics-impl-license",
+    "firebase-analytics-license",
+    "firebase-annotations",
+    "firebase-appdistribution-gradle",
+    "firebase-appindexing-license",
+    "firebase-auth-common",
+    "firebase-auth-impl",
+    "firebase-auth-interop",
+    "firebase-auth-license",
+    "firebase-encoders-json",
+    "firebase-auth-module",
+    "firebase-bom",
+    "firebase-common-license",
+    "firebase-components",
+    "firebase-config-license",
+    "firebase-crash-license",
+    "firebase-crashlytics-buildtools",
+    "firebase-crashlytics-gradle",
+    "firebase-database-collection",
+    "firebase-database-connection",
+    "firebase-database-connection-license",
+    "firebase-database-license",
+    "firebase-datatransport",
+    "firebase-dynamic-links-license",
+    "firebase-functions-license",
+    "firebase-iid-interop",
+    "firebase-iid-license",
+    "firebase-invites",
+    "firebase-measurement-connector",
+    "firebase-measurement-connector-impl",
+    "firebase-messaging-license",
+    "firebase-ml-common",
+    "firebase-ml-vision-internal-vkp",
+    "firebase-perf-license",
+    "firebase-plugins",
+    "firebase-storage-common",
+    "firebase-storage-common-license",
+    "firebase-storage-license",
+    "perf-plugin",
+    "protolite-well-known-types",
+    "testlab-instr-lib",
+    "firebase-installations-interop",
+    "google-services",
+    "gradle",
+    "firebase-ml-vision-automl",
+    "firebase-ml-vision-barcode-model",
+    "firebase-ml-vision-face-model",
+    "firebase-ml-vision-image-label-model",
+    "firebase-ml-vision-object-detection-model",
+    "firebase-ml-natural-language",
+    "firebase-ml-natural-language-language-id-model",
+    "firebase-ml-natural-language-smart-reply",
+    "firebase-ml-natural-language-smart-reply-model",
+    "firebase-ml-natural-language-translate",
+    "firebase-ml-natural-language-translate-model");
+  private static final List<String> IMPORTANT_NON_FIREBASE_LIBRARIES = ImmutableList.of(
+    "com.google.gms:google-services", 
+    "com.android.tools.build:gradle",
+    "com.google.firebase:perf-plugin",
+    "com.google.firebase:firebase-crashlytics-gradle",
+    "com.google.firebase:firebase-appdistribution-gradle");
+
+  private Set<String> ignoredFirebaseArtifacts;
+  private Set<String> bomArtifacts;
+  private Set<String> allFirebaseArtifacts;
+
+  public Map<String, String> versionOverrides = new HashMap<>();
+
+  /**
+   * This task generates a current Bill of Materials (BoM) based on the latest versions of
+   * everything in gMaven. This is meant to be a post-release task so that the BoM contains the most
+   * recent versions of all artifacts.
+   *
+   * <p>Version overrides may be given to this task in a map like so: versionOverrides =
+   * ["com.google.firebase:firebase-firestore": "17.0.1"]
+   */
+  @TaskAction
+  public void generateBom() throws Exception {
+    // Repo Access Setup
+    RepositoryClient depPopulator = new RepositoryClient();
+
+    // Prepare script by pulling the state of the world (checking configuration files and gMaven artifacts)
+    bomArtifacts = new HashSet(BOM_ARTIFACTS);
+    ignoredFirebaseArtifacts = new HashSet(IGNORED_ARTIFACTS);
+    allFirebaseArtifacts = depPopulator.getAllFirebaseArtifacts();
+    allFirebaseArtifacts.addAll(IMPORTANT_NON_FIREBASE_LIBRARIES);
+
+    // Find version for BoM artifact. First version released should be 15.0.0
+    String currentVersion =
+            depPopulator
+                    .getLastPublishedVersion(Dependency.create("com.google.firebase", "firebase-bom"))
+                    .orElse("15.0.0");
+
+    // We need to get the content of the current BoM to compute version bumps.
+    Map<String, String> previousBomVersions = getBomMap(currentVersion);
+
+    // Generate list of firebase libraries, ping gmaven for current versions, and override as needed
+    // from local settings
+    List<Dependency> allFirebaseDependencies = buildVersionedDependencyList(depPopulator, previousBomVersions);
+
+    List<Dependency> bomDependencies = allFirebaseDependencies.stream().filter(dep -> bomArtifacts.contains(dep.fullArtifactId())).collect(toList());
+
+    // Sanity check that there are no unaccounted for artifacts that we might want in the BoM
+    Set<String> bomArtifactIds = bomArtifacts.stream().map(x -> x.split(":")[1]).collect(Collectors.toSet());
+    Set<String> allFirebaseArtifactIds = allFirebaseArtifacts.stream().map(x -> x.split(":")[1]).collect(Collectors.toSet());
+    Set<String> invalidArtifacts = Sets.difference(Sets.difference(allFirebaseArtifactIds, bomArtifactIds), ignoredFirebaseArtifacts);
+
+    if (!invalidArtifacts.isEmpty()) {
+      throw new RuntimeException("Some dependencies are unaccounted for, add to BomGeneratorTask#IGNORED_ARTIFACTS or " +
+              "BomGeneratorTask#BOM_ARTIFACTS. Unaccounted for dependencies: " + invalidArtifacts.toString());
+    }
+    String version = findArtifactVersion(bomDependencies, currentVersion, previousBomVersions);
+
+
+    // Surface generated pom for sanity checking and testing, and then write it.
+    Path projectRootDir = this.getProject().getRootDir().toPath();
+    PomXmlWriter xmlWriter = new PomXmlWriter(bomDependencies, version, projectRootDir);
+    MarkdownDocumentationWriter documentationWriter = new MarkdownDocumentationWriter(bomDependencies, version, previousBomVersions, currentVersion);
+    RecipeVersionWriter recipeWriter = new RecipeVersionWriter(allFirebaseDependencies);
+    Document outputXmlDoc = xmlWriter.generatePomXml();
+    String outputDocumentation = documentationWriter.generateDocumentation();
+    String outputRecipe = recipeWriter.generateVersionUpdate();
+    xmlWriter.writeXmlDocument(outputXmlDoc);
+    documentationWriter.writeDocumentation(outputDocumentation);
+    recipeWriter.writeVersionUpdate(outputRecipe);
+  }
+
+  // Finds the version for the BoM artifact.
+  private String findArtifactVersion(List<Dependency> firebaseDependencies, String currentVersion, Map<String, String> previousBomVersions)
+      throws VersionRangeResolutionException {
+    Optional<VersionBump> bump =
+        firebaseDependencies.stream().map(Dependency::versionBump).distinct().sorted().findFirst();
+
+    if (firebaseDependencies.size() < previousBomVersions.size()) {
+      bump = Optional.of(VersionBump.MAJOR);
+    }
+
+    return bump.map(x -> VersionBump.bumpVersionBy(currentVersion, x))
+        .orElseThrow(() -> new RuntimeException("Could not figure out how to bump version"));
+  }
+
+  private Dependency overrideVersion(Dependency dep) {
+    if (versionOverrides.containsKey(dep.fullArtifactId())) {
+      return Dependency.create(
+          dep.groupId(),
+          dep.artifactId(),
+          versionOverrides.get(dep.fullArtifactId()),
+          VersionBump.PATCH);
+    } else {
+      return dep;
+    }
+  }
+
+  private List<Dependency> buildVersionedDependencyList(RepositoryClient depPopulator, Map<String, String> previousBomVersions) {
+    return allFirebaseArtifacts.stream().map(dep -> {
+            String[] splitDep = dep.split(":");
+            return Dependency.create(splitDep[0], splitDep[1]);
+          }).map(dep -> depPopulator.populateDependencyVersion(dep, previousBomVersions))
+            .map(this::overrideVersion)
+            .collect(toList());
+  }
+
+  private Map<String, String> getBomMap(String bomVersion) {
+    String bomUrl = "https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-bom/"
+            + bomVersion + "/firebase-bom-" + bomVersion + ".pom";
+    try(InputStream index = new URL(bomUrl).openStream()) {
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      factory.setValidating(true);
+      factory.setIgnoringElementContentWhitespace(true);
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      Document doc = builder.parse(index);
+      NodeList dependencyList = doc.getElementsByTagName("dependency");
+      ImmutableMap.Builder<String, String> outputBuilder = ImmutableMap.builder();
+      for (int i = 0; i < dependencyList.getLength(); i++) {
+        Element artifact = (Element) dependencyList.item(i);
+        String groupId = artifact.getElementsByTagName("groupId").item(0).getTextContent();
+        String artifactId = artifact.getElementsByTagName("artifactId").item(0).getTextContent();
+        String version = artifact.getElementsByTagName("version").item(0).getTextContent();
+        outputBuilder.put(groupId + ":" + artifactId, version);
+      }
+      return outputBuilder.build();
+    } catch (SAXException | IOException | ParserConfigurationException e) {
+      throw new RuntimeException("Failed to get contents of BoM version " + bomVersion, e);
+    }
+  }
+}