wip

Author: yifanyang

ci/fireci/fireciplugins/macrobenchmark.py

@@ -41,7 +41,7 @@
 @click.option(
   '--build-only/--no-build-only',
   default=False,
-  help='Whether to build tracing test apps only or to also submit to FTL afterwards'
+  help='Whether to only build tracing test apps or to also run them on FTL afterwards'
 )
 @ci_command()
 def macrobenchmark(build_only):
@@ -62,12 +62,18 @@ async def _launch_macrobenchmark_test(build_only):
   _logger.info(f'Processed yaml configurations: {config}')
 
   tests = [MacrobenchmarkTest(app, artifact_versions, os.getcwd(), test_dir) for app in config['test-apps']]
-  _logger.info(f'Building {len(tests)} macrobenchmark tests...')
 
-  if build_only:
-    await asyncio.gather(*[x.run_build_only() for x in tests])
-  else:
-    results = await asyncio.gather(*[x.run() for x in tests], return_exceptions=True)
+  _logger.info(f'Building {len(tests)} macrobenchmark test apps...')
+  # TODO(yifany): investigate why it runs significantly slower
+  #   - on corp workstations than M1 macbook pro
+  #   - with gradle 7.5.1 than gradle 6.9.2
+  # await asyncio.gather(*[x.build() for x in tests])
+  for test in tests:
+    await test.build()
+
+  if not build_only:
+    _logger.info(f'Submitting {len(tests)} tests to Firebase Test Lab...')
+    results = await asyncio.gather(*[x.test() for x in tests], return_exceptions=True)
     await _post_processing(results)
 
   _logger.info('Macrobenchmark test finished.')
@@ -152,18 +158,16 @@ def __init__(
     self.test_results_dir = str(uuid.uuid4())
     self.gcs_client = storage.Client()
 
-  async def run(self):
-    """Starts the workflow of src creation, apks assembly, FTL testing and results upload."""
+  async def build(self):
+    """Creates test app project and assembles app and test apks."""
     await self._create_benchmark_projects()
     await self._assemble_benchmark_apks()
+
+  async def test(self):
+    """Runs benchmark tests on FTL and fetches FTL results from GCS."""
     await self._execute_benchmark_tests()
     return await self._aggregate_benchmark_results()
 
-  async def run_build_only(self):
-    """Populate test app src and assemble apks."""
-    await self._create_benchmark_projects()
-    await self._assemble_benchmark_apks()
-
   async def _create_benchmark_projects(self):
     app_name = self.test_app_config['name']
     self.logger.info(f'Creating test app "{app_name}"...')
@@ -176,6 +180,7 @@ async def _create_benchmark_projects(self):
       renderer = pystache.Renderer()
       mustaches = glob.glob('**/*.mustache', recursive=True)
       for mustache in mustaches:
+        self.logger.info(f'Processing mustache template: {mustache}...')
         result = renderer.render_path(mustache, mustache_context)
         original_name = mustache.removesuffix('.mustache')
         with open(original_name, 'w') as file:

health-metrics/README.md

@@ -12,4 +12,14 @@ Refer to [README.md](apk-size/README.md) in the subdirectory `apk-size` for more
 
 ## App startup time
 
-**TODO(yifany)**: Add more details once the measurement tools and infrastructure is ready.
+Firebase runs during different
+[app lifecycle](https://d.android.com/guide/components/activities/process-lifecycle)
+phases, and contributes to the overall
+[app startup time](https://d.android.com/topic/performance/vitals/launch-time)
+in many ways.
+
+We are currently using
+[benchmarking](https://d.android.com/topic/performance/benchmarking/benchmarking-overview)
+and [tracing](https://d.android.com/topic/performance/tracing) to measure its
+latency impact. Refer to [README.md](benchmark/README.md) in the subdirectory
+`benchmark` for more details.

health-metrics/benchmark/README.md

@@ -0,0 +1,32 @@
+# Benchmark
+
+This directory contains the benchmark test apps used for measuring latency for
+different Firebase Android SDKs during app startup.
+
+## Usage
+
+### Prerequisite
+
+- `fireci` CLI tool
+
+  Refer to the [readme](../../ci/fireci/README.md) for how to install it.
+
+- `google-services.json`
+
+  Download it from Firebase project
+  [`fireescape-integ-tests`](https://firebase.corp.google.com/u/0/project/fireescape-integ-tests)
+  to the directory `firebase-android-sdk/health-metrics/benchmark/template/app`.
+
+### Running benchmark tests locally
+
+1. Under the root directory `firebase-android-sdk`, run
+
+   ```
+   fireci macrobenchmark --build-only
+   ```
+
+   to create all benchmark test apps based on the [configuration](config.yaml).
+
+1. [Connect an Android device to the computer](https://d.android.com/studio/run/device)
+
+1.