Replace test runner with JUnit runner

Author: allisonbm92

smoke-tests/README.md

@@ -15,11 +15,15 @@ is a work in progress, and the following list shows what is complete:
 # Project Structure
 
 This Gradle project is split into flavors for each Firebase product. Test code
-lives in these flavors. Each flavor has a single activity with the test methods.
-Test methods are annotated with `SmokeTest`. The Android plugin adds additional,
-compound flavors, such as `androidTestDatabase`. These should only contain a
-test class that extends `SmokeTestBase`. The are no formal test methods in these
-classes.
+lives in these flavors. Each flavor has one or more test classes in the app APK.
+These are regular JUnit tests.
+
+There is only one testing variant, `androidTest`. This contains the
+`SmokeTestSuite` JUnit test runner. This custom runner finds the test classes by
+searching for metadata in the application's Android manifest. Therefore, a
+metadata tag named `com.google.firebase.testing.classes` is needed in each
+application variant with a comma-separated list of test class names. The
+`androidTest` variant does not need to be modified by test authors.
 
 # Test Workflow
 
@@ -28,8 +32,7 @@ As we want to test building as a third-party consumer, the test code and
 Firebase dependencies all belong in the application APK. This can be optionally
 obfsucated like a real application.
 
-To simplify the tests, all test code is one source tree. The test source tree is
-presently required for bootstrapping the testing procedure and does not house
-anything related to the test methods. The test class reflectively finds and
-executes the test methods from the application APK. This may be simplified in
-the future.
+The Android instrumentation runner loads one test class, `SmokeTests`. This is
+executed by the `SmokeTestSuite`. `SmokeTests` is simply a placeholder class to
+bootstrap the logic contained in the suite runner, which then delegates to the
+test runners for the classes referenced by the Android manifest.

smoke-tests/build.gradle

@@ -29,11 +29,6 @@ apply plugin: "com.android.application"
 android {
   compileSdkVersion 24
 
-  compileOptions {
-    sourceCompatibility JavaVersion.VERSION_1_8
-    targetCompatibility JavaVersion.VERSION_1_8
-  }
-
   defaultConfig {
     minSdkVersion 16
     multiDexEnabled true
@@ -58,18 +53,18 @@ repositories {
 dependencies {
   // Common
 
-  implementation "com.google.firebase:firebase-common:16.0.7"
+  api "androidx.test:runner:1.1.0"
+  api "junit:junit:4.12"
 
-  androidTestImplementation "androidx.test:runner:1.1.0"
+  implementation "androidx.test:rules:1.1.0"
+  implementation "com.google.firebase:firebase-common:16.0.7"
 
   // Firestore
   firestoreImplementation "com.google.android.gms:play-services-tasks:16.0.1"
   firestoreImplementation "com.google.firebase:firebase-auth:16.1.0"
   firestoreImplementation "com.google.firebase:firebase-firestore:18.1.0"
   firestoreImplementation "com.google.truth:truth:0.43"
 
-  androidTestFirestoreImplementation "androidx.test:rules:1.1.0"
-  androidTestFirestoreImplementation "junit:junit:4.12"
 }
 
 apply plugin: "com.google.gms.google-services"

smoke-tests/src/androidTest/java/com/google/firebase/testing/common/SmokeTestRunner.java

@@ -0,0 +1,108 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.testing.common;
+
+import android.content.Context;
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageManager;
+import androidx.test.InstrumentationRegistry;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.runner.Description;
+import org.junit.runner.Runner;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.ParentRunner;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * A runner for smoke tests.
+ *
+ * <p>This is a JUnit runner inspired by the {@code Suite} runner from JUnit. That runner uses an
+ * annotation to determine the classes in the suite. This runner, on the other hand, uses metadata
+ * in the application's Android manifest. Therefore, a single test class can be reused with many
+ * different test suites by simply changing the manifest.
+ *
+ * <p>The application's manifest must include metadata with the name {@code
+ * com.google.firebase.testing.classes} or else this run with fail to run. This item is expected to
+ * be a comma-separated list of class names. These class names must be valid, loadable JUnit test
+ * classes in the application APK.
+ */
+public class SmokeTestSuite extends ParentRunner<Runner> {
+
+  private static final String TAG = "SmokeTestSuite";
+  private static final String TEST_CLASSES_KEY = "com.google.firebase.testing.classes";
+
+  private final List<Runner> runners;
+
+  public SmokeTestSuite(Class<?> clazz, RunnerBuilder builder) throws InitializationError {
+    super(clazz);
+    this.runners = builder.runners(clazz, getTestClasses());
+  }
+
+  @Override
+  protected Description describeChild(Runner runner) {
+    return runner.getDescription();
+  }
+
+  @Override
+  protected List<Runner> getChildren() {
+    return runners;
+  }
+
+  @Override
+  protected void runChild(Runner runner, RunNotifier notifier) {
+    runner.run(notifier);
+  }
+
+  private static List<Class<?>> getTestClasses() throws InitializationError {
+    Context ctx = InstrumentationRegistry.getTargetContext();
+    return getTestClasses(ctx);
+  }
+
+  private static List<Class<?>> getTestClasses(Context ctx) throws InitializationError {
+    List<String> names = getTestClassNames(ctx);
+    ArrayList<Class<?>> classes = new ArrayList<>(names.size());
+
+    try {
+      for (String name : names) {
+        Class<?> c = Class.forName(name);
+        classes.add(c);
+      }
+    } catch (ClassNotFoundException ex) {
+      throw new InitializationError(ex);
+    }
+
+    return classes;
+  }
+
+  private static List<String> getTestClassNames(Context ctx) throws InitializationError {
+    String name = ctx.getPackageName();
+    try {
+      PackageManager pm = ctx.getPackageManager();
+      ApplicationInfo ai = pm.getApplicationInfo(name, PackageManager.GET_META_DATA);
+      String names = ai.metaData.getString(TEST_CLASSES_KEY);
+
+      if (names == null) {
+        throw new InitializationError("No test classes found in Application Manifest");
+      }
+
+      return Arrays.asList(names.split(","));
+    } catch (PackageManager.NameNotFoundException ex) {
+      throw new InitializationError(ex);
+    }
+  }
+}

smoke-tests/src/androidTest/java/com/google/firebase/testing/common/SmokeTestSuite.java

@@ -14,12 +14,8 @@
 
 package com.google.firebase.testing.common;
 
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
+import org.junit.runner.RunWith;
 
-/** Marks a method as a smoke test. */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.METHOD)
-public @interface SmokeTest {}
+/** An empty class that serves as the entrypoint for the smoke tests. */
+@RunWith(SmokeTestSuite.class)
+public final class SmokeTests {}

smoke-tests/src/main/java/com/google/firebase/testing/common/SmokeTest.java renamed to smoke-tests/src/androidTest/java/com/google/firebase/testing/common/SmokeTests.java

@@ -4,11 +4,14 @@
   <uses-permission android:name="android.permission.INTERNET" />
 
   <application>
-    <activity android:name="com.google.firebase.testing.firestore.FirestoreActivity">
+    <activity android:name="android.app.Activity">
       <intent-filter>
         <action android:name="android.intent.action.MAIN" />
         <category android:name="android.intent.category.LAUNCHER" />
       </intent-filter>
     </activity>
+
+    <meta-data android:name="com.google.firebase.testing.classes"
+      android:value="com.google.firebase.testing.firestore.FirestoreTest" />
   </application>
 </manifest>

smoke-tests/src/androidTestFirestore/java/com/google/firebase/testing/firestore/FirestoreTest.java

@@ -17,6 +17,8 @@
 import static com.google.common.truth.Truth.assertThat;
 
 import android.app.Activity;
+import androidx.test.rule.ActivityTestRule;
+import androidx.test.runner.AndroidJUnit4;
 import com.google.android.gms.tasks.Task;
 import com.google.android.gms.tasks.TaskCompletionSource;
 import com.google.firebase.auth.FirebaseAuth;
@@ -26,14 +28,19 @@
 import com.google.firebase.firestore.FirebaseFirestore;
 import com.google.firebase.firestore.FirebaseFirestoreException;
 import com.google.firebase.firestore.ListenerRegistration;
-import com.google.firebase.testing.common.SmokeTest;
 import com.google.firebase.testing.common.Tasks2;
 import java.util.HashMap;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 
-/** An empty activity with Firestore smoke tests. */
-public final class FirestoreActivity extends Activity {
+/** Firestore smoke tests. */
+@RunWith(AndroidJUnit4.class)
+public final class FirestoreTest {
 
-  @SmokeTest
+  @Rule public final ActivityTestRule<Activity> activity = new ActivityTestRule<>(Activity.class);
+
+  @Test
   public void listenForUpdate() throws Exception {
     FirebaseAuth auth = FirebaseAuth.getInstance();
     FirebaseFirestore firestore = FirebaseFirestore.getInstance();