Add test apps template and fireci commands to measure sdk startup times.

Author: yifanyang

ci/fireci/fireciplugins/startup_time.py

@@ -0,0 +1,145 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import glob
+import json
+import logging
+import os
+import pystache
+import shutil
+import subprocess
+import yaml
+
+from fireci import ci_command
+from fireci import gradle
+
+_logger = logging.getLogger('fireci.startup_time')
+
+
+@ci_command()
+def startup_time():
+  """Measures app startup times for Firebase SDKs."""
+
+  gradle.run('assembleAllForSmokeTests')
+  artifact_versions = _parse_artifacts()
+
+  os.chdir('macrobenchmark')
+
+  config = _parse_config_yaml()
+  _populate_test_apps(config, artifact_versions)
+
+  gradle.run('assemble', 'assembleAndroidTest')
+
+  _run_all_benchmark_tests()
+
+
+def _parse_artifacts():
+  with open('build/m2repository/changed-artifacts.json', 'r') as json_file:
+    artifacts = json.load(json_file)
+    return dict(_artifact_key_version(x) for x in artifacts['headGit'])
+
+
+def _artifact_key_version(artifact):
+  group_id, artifact_id, version = artifact.split(':')
+  return f'{group_id}:{artifact_id}', version
+
+
+def _parse_config_yaml():
+  with open('config.yaml', 'r') as yaml_file:
+    return yaml.safe_load(yaml_file)
+
+
+def _populate_test_apps(config, artifact_versions):
+  for sdk, test_app_configs in config.items():
+    _logger.info(f'Creating test apps for "{sdk}" ...')
+    for cfg in test_app_configs:
+        _create_test_app(cfg, artifact_versions)
+
+
+def _create_test_app(cfg, artifact_versions):
+  _logger.info(f'Creating test app "{cfg["name"]}" with app-id "{cfg["application-id"]}" ...')
+
+  mustache_context = {
+    'application-id': cfg['application-id'],
+    'plugins': cfg['plugins'] if 'plugins' in cfg else False,
+    'dependencies': [
+      {
+        'key': x,
+        'version': artifact_versions[x]
+      } for x in cfg['dependencies']
+    ] if 'dependencies' in cfg else False,
+  }
+
+  output_path = f'test-apps/{cfg["name"]}'
+
+  shutil.copytree('template', output_path)
+
+  with chdir(output_path):
+    renderer = pystache.Renderer()
+    mustaches = glob.glob('**/*.mustache', recursive=True)
+    for mustache in mustaches:
+      result = renderer.render_path(mustache, mustache_context)
+      # TODO(yifany): mustache.removesuffix('.mustache') with python 3.9
+      original_name = mustache[:-9]
+      with open(original_name, 'w') as file:
+        file.write(result)
+
+
+def _run_all_benchmark_tests():
+  test_apps = os.listdir('test-apps')
+  benchmark_tests = [
+    {
+      'name': test_app,
+      'app_apk_path': glob.glob(f'test-apps/{test_app}/app/**/*.apk', recursive=True)[0],
+      'test_apk_path': glob.glob(f'test-apps/{test_app}/benchmark/**/*.apk', recursive=True)[0],
+    } for test_app in test_apps
+  ]
+  for test in benchmark_tests:
+    _logger.info(f'Running macrobenchmark test for "{test["name"]}" ...')
+    _run_on_ftl(test['app_apk_path'], test['test_apk_path'])
+
+
+def _run_on_ftl(app_apk_path, test_apk_path, results_bucket=None):
+  ftl_environment_variables = [
+      'clearPackageData=true',
+      'additionalTestOutputDir=/sdcard/Download',
+      'no-isolated-storage=true',
+  ]
+  commands = ['gcloud', 'firebase', 'test', 'android', 'run']
+  commands += ['--type', 'instrumentation']
+  commands += ['--app', app_apk_path]
+  commands += ['--test', test_apk_path]
+  commands += ['--device', 'model=flame,version=30,locale=en,orientation=portrait']
+  commands += ['--directories-to-pull', '/sdcard/Download']
+  commands += ['--results-bucket', results_bucket] if results_bucket else []
+  commands += ['--environment-variables', ','.join(ftl_environment_variables)]
+  commands += ['--timeout', '30m']
+  commands += ['--project', 'fireescape-c4819']
+  return subprocess.run(commands, stdout=subprocess.PIPE, check=True)
+
+
+# TODO(yifany): same as the one in fierperf.py
+@contextlib.contextmanager
+def chdir(directory):
+  """Change working dir to `directory` and restore to original afterwards."""
+  original_dir = os.getcwd()
+  abs_directory = os.path.join(original_dir, directory)
+  _logger.debug(f'Changing directory from {original_dir} to: {abs_directory} ...')
+  os.chdir(directory)
+  try:
+    yield
+  finally:
+    _logger.debug(f'Restoring directory to: {original_dir} ...')
+    os.chdir(original_dir)

ci/fireci/setup.py

@@ -27,7 +27,9 @@
     install_requires=[
         'click==7.0',
         'PyGithub==1.43.8',
+        'pystache==0.5.4',
         'requests==2.23.0',
+        'PyYAML==5.4.1',
     ],
     packages=find_packages(exclude=['tests']),
     entry_points={

macrobenchmark/.gitignore

@@ -0,0 +1,16 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties
+test-apps

macrobenchmark/README.md

@@ -0,0 +1,10 @@
+# Macrobenchmark
+
+This directory contains infrastructure to measure startup times for Firebase SDKs.
+
+Run to invoke the startup time tests:
+```shell
+fireci startup_time
+```
+
+See more details on fireci [here](../ci/README.md).

macrobenchmark/build.gradle

@@ -0,0 +1,49 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+buildscript {
+    ext.kotlin_version = "1.4.32"
+    repositories {
+        google()
+        mavenCentral()
+    }
+    dependencies {
+        classpath "com.android.tools.build:gradle:4.2.0-rc01"
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+
+        classpath 'com.google.gms:google-services:4.3.5'
+
+        // Add the Crashlytics Gradle plugin (be sure to add version
+        // 2.0.0 or later if you built your app with Android Studio 4.1).
+        classpath 'com.google.firebase:firebase-crashlytics-gradle:2.5.2'
+
+        // Add the dependency for the Performance Monitoring plugin
+        classpath 'com.google.firebase:perf-plugin:1.3.5'  // Performance Monitoring plugin
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        mavenCentral()
+
+        // snapshot repository for androidx - not necessary once alpha is public
+        maven {
+            url uri('https://androidx.dev/snapshots/builds/7205390/artifacts/repository')
+        }
+
+        maven {
+            url rootProject.file('../build/m2repository')
+        }
+    }
+}
+
+task cleanTestApps(type: Delete) {
+    delete 'test-apps'
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+    dependsOn cleanTestApps
+}

macrobenchmark/config.yaml

@@ -0,0 +1,79 @@
+baseline:
+  - name: baseline
+    application-id: com.google.firebase.benchmark.baseline
+
+firebase-config:
+  - name: config
+    application-id: com.google.firebase.benchmark.config
+    dependencies:
+      - com.google.firebase:firebase-config-ktx
+
+firebase-crashlytics:
+  - name: crash
+    application-id: com.google.firebase.benchmark.crash
+    dependencies:
+      - com.google.firebase:firebase-crashlytics-ktx
+    plugins:
+      - com.google.firebase.crashlytics
+
+firebase-database:
+  - name: database
+    application-id: com.google.firebase.benchmark.database
+    dependencies:
+      - com.google.firebase:firebase-database-ktx
+
+firebase-dynamic-links:
+  - name: dynamiclinks
+    application-id: com.google.firebase.benchmark.dynamiclinks
+    dependencies:
+      - com.google.firebase:firebase-dynamic-links-ktx
+
+firebase-firestore:
+  - name: firestore
+    application-id: com.google.firebase.benchmark.firestore
+    dependencies:
+      - com.google.firebase:firebase-firestore-ktx
+
+firebase-functions:
+  - name: functions
+    application-id: com.google.firebase.benchmark.functions
+    dependencies:
+      - com.google.firebase:firebase-functions-ktx
+
+firebase-inappmessaging-display:
+  - name: inappmessaging
+    application-id: com.google.firebase.benchmark.inappmessaging
+    dependencies:
+      - com.google.firebase:firebase-inappmessaging-ktx
+      - com.google.firebase:firebase-inappmessaging-display-ktx
+
+firebase-messaging:
+  - name: messaging
+    application-id: com.google.firebase.benchmark.messaging
+    dependencies:
+      - com.google.firebase:firebase-messaging-ktx
+
+firebase-perf:
+  - name: perf
+    application-id: com.google.firebase.benchmark.perf
+    dependencies:
+      - com.google.firebase:firebase-perf-ktx
+    plugins:
+      - com.google.firebase.firebase-perf
+
+firebase-storage:
+  - name: storage
+    application-id: com.google.firebase.benchmark.storage
+    dependencies:
+      - com.google.firebase:firebase-storage-ktx
+
+
+
+# TODO(yifany): google3 sdks, customizing FTL devices
+# auth
+# analytics
+# combined
+#   - crashlytics + analytics
+#   - crashlytics + fireperf
+#   - auth + firestore
+#   - ...

macrobenchmark/gradle.properties

@@ -0,0 +1,19 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+org.gradle.parallel=true
+# AndroidX package structure to make it clearer which packages are bundled with the
+# Android operating system, and which are packaged with your app"s APK
+# https://developer.android.com/topic/libraries/support-library/androidx-rn
+android.useAndroidX=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official

macrobenchmark/gradle/wrapper/gradle-wrapper.jar

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip
+distributionPath=wrapper/dists
+zipStorePath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME