Add workflow to automatically update iOS and Android dependencies.

.github/workflows/lint.yml

@@ -22,6 +22,10 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
+      - name: Install prerequisites
+        run: |
+          cd firebase
+          python scripts/gha/install_prereqs_desktop.py
       - name: Install prerequisites
         run: |
           python3 -m pip install unidiff

.github/workflows/update-dependencies.yml

@@ -1,13 +1,137 @@
-name: Update Android and iOS dependencies (placeholder)
+name: Update Android and iOS dependencies
 on:
   workflow_dispatch:
     inputs:
+      updateAndroid:
+        description: 'update Android dependencies?'
+        default: 1
+      updateiOS:
+        description: 'update iOS dependencies?'
+        default: 1
+      triggerTests:
+        description: 'trigger tests on PR?'
+        default: 1
+      baseBranch:
+        description: 'create the new branch from this base'
+        default: 'main'
+
+env:
+  branchPrefix: "workflow/auto-update-deps-"
+  triggerTestsLabel: "tests-requested: quick"
 
 jobs:
   update_dependencies:
-    name: update-dependencies
+    name: update-deps
     runs-on: ubuntu-latest
     steps:
-      - name: Placeholder step
+      - name: Get token for firebase-workflow-trigger
+        uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
+          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Check out base branch
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.baseBranch }}
+
+      - name: Install prerequisites
+        run: |
+          python scripts/gha/install_prereqs_desktop.py
+          python -m pip install requests
+
+      - name: Name new branch
+        run: |
+          date_str=$(date "+%Y%m%d-%H%M%S")
+          echo "NEW_BRANCH=${{env.branchPrefix}}${{github.run_number}}-${date_str}" >> $GITHUB_ENV
+
+      - name: Create new branch
         run: |
-          true
+          git remote update
+          git checkout -b "${NEW_BRANCH}"
+          echo "UPDATE_LOGFILE=update_log.txt" >> $GITHUB_ENV
+
+      - name: Run update script
+        run: |
+          if [[ ${{ github.event.inputs.updateiOS }} -eq 1 ]]; then
+            if [[ ${{ github.event.inputs.updateAndroid }} -eq 1 ]]; then
+              # Update both
+              echo "Updating all dependencies"
+              python scripts/update_android_ios_dependencies.py --logfile=${UPDATE_LOGFILE}
+              echo "CHOSEN_DEPS=mobile" >> $GITHUB_ENV
+            else
+              # Update iOS only
+              echo "Updating iOS dependencies only"
+              python scripts/update_android_ios_dependencies.py --skip_android --logfile=${UPDATE_LOGFILE}
+              echo "CHOSEN_DEPS=iOS" >> $GITHUB_ENV
+            fi
+            # iOS: Update Firestore external version to match Firestore Cocoapod version.
+            firestore_version=$(grep "pod 'Firebase/Firestore'" ios_pod/Podfile | sed "s/.*'\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)'.*/\1/")
+            sed -i~ "s/^set(version [^)]*)/set(version CocoaPods-${firestore_version})/i" cmake/external/firestore.cmake
+          elif [[ ${{ github.event.inputs.updateAndroid }} -eq 1 ]]; then
+            # Update Android only
+            echo "Updating Android dependencies only"
+            python scripts/update_android_ios_dependencies.py --skip_ios --logfile=${UPDATE_LOGFILE}
+            echo "CHOSEN_DEPS=Android" >> $GITHUB_ENV
+          else
+            echo "::error ::Neither Android nor iOS selected. Exiting."
+            exit 1
+          fi
+
+      - name: Push branch if there are changes
+        id: push-branch
+        run: |
+          if ! git update-index --refresh; then
+            # Do a bit of post-processing on the update log to split it by platform.
+            UPDATE_LOGFILE_PROCESSED=update_log_processed.txt
+            if grep -q ^Android: "${UPDATE_LOGFILE}"; then
+              echo "### Android" >> "${UPDATE_LOGFILE_PROCESSED}"
+              echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+              sed 's/^Android: /- /' ${UPDATE_LOGFILE} >> ${UPDATE_LOGFILE_PROCESSED}
+              echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+            fi
+            if grep -q ^iOS: "${UPDATE_LOGFILE}"; then
+              echo "### iOS" >> "${UPDATE_LOGFILE_PROCESSED}"
+              echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+              sed 's/^iOS: /- /' ${UPDATE_LOGFILE} >> ${UPDATE_LOGFILE_PROCESSED}
+              echo "" >> "${UPDATE_LOGFILE_PROCESSED}"
+            fi
+
+            date_str=$(date "+%a %b %d %Y")
+            commit_title="Update ${CHOSEN_DEPS} dependencies - ${date_str}"
+            commit_body="$(cat ${UPDATE_LOGFILE_PROCESSED})
+
+          > Created by [${{github.workflow}} workflow]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+            git config user.email "[email protected]"
+            git config user.name "firebase-workflow-trigger-bot"
+            git config core.commentChar "%"  # so we can use # in git commit messages
+            git commit -a -m "${commit_title}
+
+          ${commit_body}"
+            echo "::set-output name=branch_pushed::1"
+            # Show changes in git log
+            git diff
+            # Push branch
+            git push --set-upstream origin "${NEW_BRANCH}"
+            # Create pull request
+            pr_number=$(python scripts/gha/create_pull_request.py --token ${{ steps.generate-token.outputs.token }} --head "${NEW_BRANCH}" --base "${{ github.event.inputs.baseBranch }}" --title "${commit_title}" --body "${commit_body}")
+            echo "::set-output name=created_pr_number::${pr_number}"
+          else
+            echo "::warning ::No changes detected, won't create pull request."
+            echo "::set-output name=branch_pushed::0"
+          fi
+
+      - name: Set test trigger label.
+        uses: actions-ecosystem/action-add-labels@v1
+        if: ${{ github.event.inputs.triggerTests == 1 && steps.push-branch.outputs.branch_pushed == 1 }}
+        with:
+          github_token: ${{ steps.generate-token.outputs.token }}
+          number: ${{ steps.push-branch.outputs.created_pr_number }}
+          labels: "${{ env.triggerTestsLabel }}"

scripts/gha/create_pull_request.py

@@ -0,0 +1,75 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A utility to create pull requests.
+
+USAGE:
+  python scripts/gha/create_pull_request.py \
+    --token ${{github.token}} \
+    --head pr_branch \
+    --base main \
+    --title 'Title of the pull request' \
+    [--body 'Body text for the pull request']
+
+  Creates the pull request, and outputs the new PR number to stdout.
+"""
+
+import datetime
+import shutil
+
+from absl import app
+from absl import flags
+from absl import logging
+
+import github
+
+FLAGS = flags.FLAGS
+_DEFAULT_MESSAGE = "Creating pull request."
+
+flags.DEFINE_string(
+    "token", None,
+    "github.token: A token to authenticate on your repository.")
+
+flags.DEFINE_string(
+    "head", None,
+    "Head branch name.")
+
+flags.DEFINE_string(
+    "base", "main",
+    "Base branch name.")
+
+flags.DEFINE_string(
+    "title", None,
+    "Title for the pull request.")
+
+flags.DEFINE_string(
+    "body", "",
+    "Body text for the pull request.")
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError("Too many command-line arguments.")
+  if github.create_pull_request(FLAGS.token, FLAGS.head, FLAGS.base, FLAGS.title, FLAGS.body, True):
+    # Find the most recent pull_request with the given base and head, that's ours.
+    pull_requests = github.list_pull_requests(FLAGS.token, "open", FLAGS.head, FLAGS.base)
+    print(pull_requests[0]['number'])
+  else:
+    exit(1)
+
+
+if __name__ == "__main__":
+  flags.mark_flag_as_required("token")
+  flags.mark_flag_as_required("head")
+  flags.mark_flag_as_required("title")
+  app.run(main)

scripts/gha/github.py

@@ -211,3 +211,39 @@ def workflow_dispatch(token, workflow_id, ref, inputs):
   with requests.post(url, headers=headers, data=json.dumps(data),
                     stream=True, timeout=TIMEOUT) as response:
     logging.info("workflow_dispatch: %s response: %s", url, response)
+
+
+def create_pull_request(token, head, base, title, body, maintainer_can_modify):
+  """https://docs.github.com/en/rest/reference/pulls#create-a-pull-request"""
+  url = f'{FIREBASE_URL}/pulls'
+  headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+  data = {'head': head, 'base': base, 'title': title, 'body': body,
+          'maintainer_can_modify': maintainer_can_modify}
+  with requests.post(url, headers=headers, data=json.dumps(data),
+                    stream=True, timeout=TIMEOUT) as response:
+    logging.info("create_pull_request: %s response: %s", head, response)
+    return True if response.status_code == 201 else False
+
+def list_pull_requests(token, state, head, base):
+  """https://docs.github.com/en/rest/reference/pulls#list-pull-requests"""
+  url = f'{FIREBASE_URL}/pulls'
+  headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+  page = 1
+  per_page = 100
+  results = []
+  keep_going = True
+  while keep_going:
+    params = {'per_page': per_page, 'page': page}
+    if state: params.update({'state': state})
+    if head: params.update({'head': head})
+    if base: params.update({'base': base})
+    page = page + 1
+    keep_going = False
+    with requests_retry_session().get(url, headers=headers, params=params,
+                      stream=True, timeout=TIMEOUT) as response:
+      logging.info("get_reviews: %s response: %s", url, response)
+      results = results + response.json()
+      # If exactly per_page results were retrieved, read the next page.
+      keep_going = (len(response.json()) == per_page)
+  return results
+

scripts/update_android_ios_dependencies.py

@@ -62,6 +62,9 @@
 --depfiles
 --readmefiles
 
+Log updated version numbers to a text file:
+--logfile=my_log_filename.txt
+
 These "files" flags can take a list of paths (files and directories).
 If directories are provided, they are scanned for known file types.
 """
@@ -321,6 +324,7 @@ def modify_pod_file(pod_file, pod_version_map, dryrun=True):
       dryrun (bool, optional): Just print the substitutions.
                                Do not write to file. Defaults to True.
   """
+  global logfile_lines
   to_update = False
   existing_lines = []
   with open(pod_file, "r") as podfile:
@@ -345,6 +349,7 @@ def modify_pod_file(pod_file, pod_version_map, dryrun=True):
           substituted_pairs.append((line, substituted_line))
           existing_lines[idx] = substituted_line
           to_update = True
+          logfile_lines.add('iOS: %s → %s' % (pod_name, latest_version))
 
   if to_update:
     print('Updating contents of {0}'.format(pod_file))
@@ -491,6 +496,7 @@ def modify_dependency_file(dependency_filepath, version_map, dryrun=True):
     dryrun (bool, optional): Just print the substitutions.
       Do not write to file. Defaults to True.
   """
+  global logfile_lines
   logging.debug('Reading dependency file: {0}'.format(dependency_filepath))
   lines = None
   with open(dependency_filepath, "r") as dependency_file:
@@ -523,6 +529,9 @@ def replace_dependency(m):
     if substituted_line != line:
       substituted_pairs.append((line, substituted_line))
       to_update = True
+      log_match = re.search(RE_GENERIC_DEPENDENCY_MODULE, line)
+      log_pkg = log_match.group('pkg').replace('-', '_').replace(':', '.')
+      logfile_lines.add('Android: %s → %s' % (log_pkg, version_map[log_pkg]))
 
   if to_update:
     print('Updating contents of {0}'.format(dependency_filepath))
@@ -705,6 +714,7 @@ def parse_cmdline_args():
             default=('release_build_files/readme.md',),
             help= 'List of release readme markdown files or directories'
                   'containing them.')
+  parser.add_argument('--logfile', help='Log to text file')
 
   args = parser.parse_args()
 
@@ -725,6 +735,7 @@ def parse_cmdline_args():
   logging.getLogger(__name__)
   return args
 
+logfile_lines = set()
 
 def main():
   args = parse_cmdline_args()
@@ -761,5 +772,9 @@ def main():
     for gradle_file in gradle_files:
       modify_gradle_file(gradle_file, latest_android_versions_map, args.dryrun)
 
+    if args.logfile:
+      with open(args.logfile, 'w') as logfile_file:
+        logfile_file.write("\n".join(sorted(list(logfile_lines))) + "\n")
+
 if __name__ == '__main__':
   main()