Add workflow definition for health metric tests (#3954)

yifanyang · web-flow · commit ba6f410c160e · 2022-08-15T10:49:09.000-07:00
diff --git a/.github/workflows/health-metrics.yml b/.github/workflows/health-metrics.yml
@@ -0,0 +1,89 @@
+name: Health Metrics
+
+on: [ pull_request, push ]
+
+env:
+  GITHUB_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+
+jobs:
+  coverage:
+    name: Coverage
+    if: (github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: temurin
+          cache: gradle
+      - name: Set up NDK 21.4.7075529
+        run: |
+          ANDROID_ROOT=/usr/local/lib/android
+          ANDROID_SDK_ROOT=${ANDROID_ROOT}/sdk
+          ANDROID_NDK_ROOT=${ANDROID_SDK_ROOT}/ndk-bundle
+          SDKMANAGER=${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager
+          echo "y" | $SDKMANAGER "ndk;21.4.7075529"
+          ln -sfn ${ANDROID_SDK_ROOT}/ndk/21.4.7075529 ${ANDROID_NDK_ROOT}
+          echo "ANDROID_NDK_HOME=${ANDROID_NDK_ROOT}" >> $GITHUB_ENV
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}'
+      - uses: google-github-actions/setup-gcloud@v0
+      - name: Set up fireci
+        run: pip3 install -e ci/fireci
+      - name: Run coverage tests (presubmit)
+        if: ${{ github.event_name == 'pull_request' }}
+        run: fireci coverage --pull-request
+      - name: Run coverage tests (post-submit)
+        if: ${{ github.event_name == 'push' }}
+        run: fireci coverage
+
+  size:
+    name: Size
+    if: (github.repository == 'Firebase/firebase-android-sdk' && github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: temurin
+          cache: gradle
+      - name: Set up NDK 21.4.7075529
+        run: |
+          ANDROID_ROOT=/usr/local/lib/android
+          ANDROID_SDK_ROOT=${ANDROID_ROOT}/sdk
+          ANDROID_NDK_ROOT=${ANDROID_SDK_ROOT}/ndk-bundle
+          SDKMANAGER=${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager
+          echo "y" | $SDKMANAGER "ndk;21.4.7075529"
+          ln -sfn ${ANDROID_SDK_ROOT}/ndk/21.4.7075529 ${ANDROID_NDK_ROOT}
+          echo "ANDROID_NDK_HOME=${ANDROID_NDK_ROOT}" >> $GITHUB_ENV
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: google-github-actions/auth@v0
+        with:
+          credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}'
+      - uses: google-github-actions/setup-gcloud@v0
+      - name: Set up fireci
+        run: pip3 install -e ci/fireci
+      - name: Run size tests (presubmit)
+        if: ${{ github.event_name == 'pull_request' }}
+        run: fireci binary_size --pull-request
+      - name: Run size tests (post-submit)
+        if: ${{ github.event_name == 'push' }}
+        run: fireci binary_size
diff --git a/buildSrc/src/main/java/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.java b/buildSrc/src/main/java/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.java
@@ -68,10 +68,10 @@ Set<Project> find() {
 
   private static Set<String> changedPaths(File workDir) {
     try {
-      // works on Prow only.
+      // works on CI only.
       Process process =
           Runtime.getRuntime()
-              .exec("git diff --name-only --submodule=diff HEAD@{0} HEAD@{1}", null, workDir);
+              .exec("git diff --name-only --submodule=diff HEAD^1 HEAD", null, workDir);
       try {
         return ImmutableSet.copyOf(
             CharStreams.readLines(new InputStreamReader(process.getInputStream())));
diff --git a/ci/fireci/fireci/ci_utils.py b/ci/fireci/fireci/ci_utils.py
@@ -16,7 +16,25 @@
 import os
 import subprocess
 
-_logger = logging.getLogger('fireci.prow_utils')
+_logger = logging.getLogger('fireci.ci_utils')
+
+
+def ci_log_link():
+  """Returns the link to the log of the current CI test."""
+
+  if os.getenv('GITHUB_ACTIONS'):
+    return github_actions_run_log_link()
+  elif os.getenv('PROW_JOB_ID'):
+    return prow_job_log_link()
+
+
+def github_actions_run_log_link():
+  """Returns the link to the log of the current GitHub Actions run."""
+
+  repo = os.getenv('GITHUB_REPOSITORY')
+  run_id = os.getenv('GITHUB_RUN_ID')
+
+  return f'https://github.com/{repo}/actions/runs/{run_id}'
 
 
 def prow_job_log_link():
diff --git a/ci/fireci/fireci/uploader.py b/ci/fireci/fireci/uploader.py
@@ -19,15 +19,19 @@
 import subprocess
 import urllib.parse
 
-from . import prow_utils
 
 _logger = logging.getLogger('fireci.uploader')
 
 
-def post_report(test_report, metrics_service_url, access_token, metric='reports', note=''):
+def post_report(test_report, metrics_service_url, access_token, metric_type):
   """Post a report to the metrics service backend."""
 
-  endpoint = _construct_request_endpoint(metric, note)
+  endpoint = ''
+  if os.getenv('GITHUB_ACTIONS'):
+    endpoint = _construct_request_endpoint_for_github_actions(metric_type)
+  elif os.getenv('PROW_JOB_ID'):
+    endpoint = _construct_request_endpoint_for_prow(metric_type)
+
   headers = {'Authorization': f'Bearer {access_token}', 'Content-Type': 'application/json'}
   data = json.dumps(test_report)
 
@@ -41,25 +45,36 @@ def post_report(test_report, metrics_service_url, access_token, metric='reports'
   _logger.info(f'Response: {result.text}')
 
 
-def _construct_request_endpoint(metric, note):
+def _construct_request_endpoint_for_github_actions(metric_type):
+  repo = os.getenv('GITHUB_REPOSITORY')
+  commit = os.getenv('GITHUB_SHA')
+  event_name = os.getenv('GITHUB_EVENT_NAME')
+
+  endpoint = f'/repos/{repo}/commits/{commit}/{metric_type}'
+  if event_name == 'pull_request':
+    pull_request = os.getenv('GITHUB_PULL_REQUEST_NUMBER')
+    base_commit = _get_commit_hash('HEAD^1')
+    head_commit = _get_commit_hash('HEAD^2')
+    endpoint += f'?pull_request={pull_request}&base_commit={base_commit}&head_commit={head_commit}'
+  else:
+    branch = os.getenv('GITHUB_REF_NAME')
+    endpoint += f'?branch={branch}'
+
+  return endpoint
+
+def _construct_request_endpoint_for_prow(metric_type):
   repo_owner = os.getenv('REPO_OWNER')
   repo_name = os.getenv('REPO_NAME')
-  branch = os.getenv('PULL_BASE_REF')
-  pull_request = os.getenv('PULL_NUMBER')
-
   commit = _get_commit_hash('HEAD@{0}')
-  log = prow_utils.prow_job_log_link()
+  pull_request = os.getenv('PULL_NUMBER')
 
-  endpoint = f'/repos/{repo_owner}/{repo_name}/commits/{commit}/{metric}?log={log}'
+  endpoint = f'/repos/{repo_owner}/{repo_name}/commits/{commit}/{metric_type}'
   if pull_request:
     base_commit = os.getenv('PULL_BASE_SHA')
     head_commit = os.getenv('PULL_PULL_SHA')
     endpoint += f'&pull_request={pull_request}&base_commit={base_commit}&head_commit={head_commit}'
-
-    commit_note = _get_prow_commit_note('HEAD@{0}')
-    note += f'\n{commit_note}\n'
-    endpoint += f'&note={urllib.parse.quote(note)}'
   else:
+    branch = os.getenv('PULL_BASE_REF')
     endpoint += f'&branch={branch}'
 
   return endpoint
@@ -68,13 +83,3 @@ def _construct_request_endpoint(metric, note):
 def _get_commit_hash(revision):
   result = subprocess.run(['git', 'rev-parse', revision], stdout=subprocess.PIPE, check=True)
   return result.stdout.decode('utf-8').strip()
-
-
-def _get_prow_commit_note(revision):
-  template = 'Head commit (%h) is created by Prow via merging commits: %p.'
-  result = subprocess.run(
-    ['git', 'show', revision, f'--format={template}', '-s'],
-    stdout=subprocess.PIPE,
-    check=True,
-  )
-  return result.stdout.decode('utf-8').strip()
diff --git a/ci/fireci/fireciplugins/binary_size.py b/ci/fireci/fireciplugins/binary_size.py
@@ -17,11 +17,10 @@
 import json
 import logging
 import os
-import re
 
 from fireci import ci_command
+from fireci import ci_utils
 from fireci import gradle
-from fireci import prow_utils
 from fireci import uploader
 
 _logger = logging.getLogger('fireci.binary_size')
@@ -34,17 +33,17 @@
 )
 @click.option(
   '--log',
-  default=prow_utils.prow_job_log_link,
-  help='The link to the log of the prow job, which runs this coverage check.'
+  default=ci_utils.ci_log_link,
+  help='The link to the log of the current job, which runs this size test.'
 )
 @click.option(
   '--metrics-service-url',
-  envvar='METRICS_SERVICE_URL',
+  default='https://api.firebase-sdk-health-metrics.com',
   help='The URL to the metrics service, which persists data and calculates diff.'
 )
 @click.option(
   '--access-token',
-  default=prow_utils.gcloud_identity_token,
+  default=ci_utils.gcloud_identity_token,
   help='The access token, used to authorize http requests to the metrics service.'
 )
 @ci_command()
@@ -58,12 +57,17 @@ def binary_size(pull_request, log, metrics_service_url, access_token):
   sdks = ','.join(artifacts)
 
   workdir = 'health-metrics/apk-size'
-  gradle.run('assemble', '--continue', gradle.P('sdks', sdks), workdir=workdir, check=False)
+  process = gradle.run('assemble', '--continue', gradle.P('sdks', sdks), workdir=workdir, check=False)
 
   test_results = _measure_aar_sizes(artifacts) + _measure_apk_sizes()
-  test_report = {'metric': 'BinarySize', 'results': test_results, 'log': log}
+  test_report = {'sizes': test_results, 'log': log}
 
-  uploader.post_report(test_report, metrics_service_url, access_token)
+  access_token = ci_utils.gcloud_identity_token()
+  uploader.post_report(test_report, metrics_service_url, access_token, 'size')
+
+  if process.returncode != 0:
+    _logger.error(f'{process.args} failed with error code: {process.returncode}.')
+    raise click.ClickException('Binary size test failed with above errors.')
 
 
 def _measure_aar_sizes(artifacts):
@@ -75,7 +79,7 @@ def _measure_aar_sizes(artifacts):
 
     if aar_files:
       aar_size = os.path.getsize(aar_files[0])
-      test_results.append({'sdk': artifact_id, 'type': 'aar', 'value': aar_size})
+      test_results.append({'sdk': artifact_id, 'type': 'aar', 'size': aar_size})
 
   return test_results
 
@@ -91,7 +95,7 @@ def _measure_apk_sizes():
     apk_type = build_type if abi == 'universal' else f'{build_type} / {abi}'
     apk_size = os.path.getsize(apk_file)
 
-    test_results.append({'sdk': artifact, 'type': f'apk ({apk_type})', 'value': apk_size})
+    test_results.append({'sdk': artifact, 'type': f'apk ({apk_type})', 'size': apk_size})
 
   return test_results
 
diff --git a/ci/fireci/fireciplugins/coverage.py b/ci/fireci/fireciplugins/coverage.py
@@ -19,8 +19,8 @@
 import xml.etree.ElementTree as ElementTree
 
 from fireci import ci_command
+from fireci import ci_utils
 from fireci import gradle
-from fireci import prow_utils
 from fireci import uploader
 
 _logger = logging.getLogger('fireci.coverage')
@@ -33,35 +33,35 @@
 )
 @click.option(
   '--log',
-  default=prow_utils.prow_job_log_link,
-  help='The link to the log of the prow job, which runs this coverage check.'
+  default=ci_utils.ci_log_link,
+  help='The link to the log of the current job, which runs this coverage check.'
 )
 @click.option(
   '--metrics-service-url',
-  envvar='METRICS_SERVICE_URL',
+  default='https://api.firebase-sdk-health-metrics.com',
   help='The URL to the metrics service, which persists data and calculates diff.'
 )
 @click.option(
   '--access-token',
-  default=prow_utils.gcloud_identity_token,
+  default=ci_utils.gcloud_identity_token,
   help='The access token, used to authorize http requests to the metrics service.'
 )
 @ci_command()
 def coverage(pull_request, log, metrics_service_url, access_token):
   """Produces and uploads code coverage reports."""
 
   coverage_task = 'checkCoverageChanged' if pull_request else 'checkCoverage'
-  gradle.run(coverage_task, '--continue', check=False)
+  process = gradle.run(coverage_task, '--continue', check=False)
 
   test_results = _parse_xml_reports()
-  test_report = {'metric': 'Coverage', 'results': test_results, 'log': log}
+  test_report = {'coverages': test_results, 'log': log}
 
-  note = '''
-HTML coverage reports can be produced locally with `./gradlew <product>:checkCoverage`.
-Report files are located at `<product-build-dir>/reports/jacoco/`.
-'''
+  access_token = ci_utils.gcloud_identity_token()
+  uploader.post_report(test_report, metrics_service_url, access_token, 'coverage')
 
-  uploader.post_report(test_report, metrics_service_url, access_token, note=note)
+  if process.returncode != 0:
+    _logger.error(f'{process.args} failed with error code: {process.returncode}.')
+    raise click.ClickException('Coverage test failed with above errors.')
 
 
 def _parse_xml_reports():
@@ -74,12 +74,12 @@ def _parse_xml_reports():
     sdk = re.search(r'([^/]*)\.xml', xml_report).group(1)
     report = ElementTree.parse(xml_report).getroot()
     sdk_coverage = _calculate_coverage(report)
-    test_results.append({'sdk': sdk, 'type': '', 'value': sdk_coverage})
+    test_results.append({'sdk': sdk, 'filename': '', 'coverage': sdk_coverage})
 
     for source_file in report.findall('.//sourcefile'):
       file_name = source_file.attrib['name']
       file_coverage = _calculate_coverage(source_file)
-      test_results.append({'sdk': sdk, 'type': file_name, 'value': file_coverage})
+      test_results.append({'sdk': sdk, 'filename': file_name, 'coverage': file_coverage})
 
   return test_results
 
diff --git a/ci/fireci/fireciplugins/macrobenchmark.py b/ci/fireci/fireciplugins/macrobenchmark.py
@@ -30,8 +30,8 @@
 from google.cloud import storage
 
 from fireci import ci_command
+from fireci import ci_utils
 from fireci.dir_utils import chdir
-from fireci import prow_utils
 from fireci import uploader
 
 _logger = logging.getLogger('fireci.macrobenchmark')
@@ -113,9 +113,12 @@ async def _post_processing(results):
     if not isinstance(result, Exception):
       measurements.extend(result)
 
-  metrics_service_url = os.getenv('METRICS_SERVICE_URL')
-  access_token = prow_utils.gcloud_identity_token()
-  uploader.post_report(measurements, metrics_service_url, access_token, metric='macrobenchmark')
+  log = ci_utils.ci_log_link()
+  test_report = {'benchmarks': measurements, 'log': log}
+
+  metrics_service_url = 'https://api.firebase-sdk-health-metrics.com'
+  access_token = ci_utils.gcloud_identity_token()
+  uploader.post_report(test_report, metrics_service_url, access_token, 'macrobenchmark')
 
   # Raise exceptions for failed measurements
   if any(map(lambda x: isinstance(x, Exception), results)):
diff --git a/health-metrics/apk-size/gradle.properties b/health-metrics/apk-size/gradle.properties
@@ -15,4 +15,4 @@
 android.useAndroidX=true
 android.enableJetifier=true
 
-org.gradle.jvmargs=-Xmx8g -XX:MaxPermSize=8g
+org.gradle.jvmargs=-Xmx4g -XX:MaxPermSize=4g
