Add new smoke tests (Firestore).

This commit adds a new smoke test for Firestore and some infrastructure
for the other smoke tests that be added soon.

Author: allisonbm92

smoke-tests/README.md

@@ -0,0 +1,35 @@
+# Firebase Smoke Test Suite
+
+This directory contains smoke tests for Firebase on Android. These tests are
+intended to verify the integrations between different components and versions of
+Firebase. The tests should not include additional overhead, such as user
+interfaces. However, the tests should strive to remain similar to real use
+cases. As such, these tests run on devices or emulators (no Robolectric). This
+is a work in progress, and the following list shows what is complete:
+
+- [ ] Create first set of tests to replace old test apps.
+- [ ] Reliably run smoke tests on CI.
+- [ ] Support version matrices.
+- [ ] Extend to collect system health metrics.
+
+# Project Structure
+
+This Gradle project is split into flavors for each Firebase product. Test code
+lives in these flavors. Each flavor has a single activity with the test methods.
+Test methods are annotated with `SmokeTest`. The Android plugin adds additional,
+compound flavors, such as `androidTestDatabase`. These should only contain a
+test class that extends `SmokeTestBase`. The are no formal test methods in these
+classes.
+
+# Test Workflow
+
+Android instrumentation uses two APKs: one for the app and the other for tests.
+As we want to test building as a third-party consumer, the test code and
+Firebase dependencies all belong in the application APK. This can be optionally
+obfsucated like a real application.
+
+To simplify the tests, all test code is one source tree. The test source tree is
+presently required for bootstrapping the testing procedure and does not house
+anything related to the test methods. The test class reflectively finds and
+executes the test methods from the application APK. This may be simplified in
+the future.

smoke-tests/build.gradle

@@ -0,0 +1,77 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+buildscript {
+  repositories {
+    google()
+    jcenter()
+  }
+
+  dependencies {
+    classpath "com.android.tools.build:gradle:3.3.2"
+    classpath "com.google.gms:google-services:4.0.0"
+  }
+}
+
+apply plugin: "com.android.application"
+
+android {
+  compileSdkVersion 24
+
+  compileOptions {
+    sourceCompatibility JavaVersion.VERSION_1_8
+    targetCompatibility JavaVersion.VERSION_1_8
+  }
+
+  defaultConfig {
+    minSdkVersion 16
+    multiDexEnabled true
+    testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+  }
+
+  flavorDimensions "systemUnderTest"
+
+  productFlavors {
+    firestore {
+      dimension "systemUnderTest"
+      applicationId "com.google.firebase.testing.firestore"
+    }
+  }
+}
+
+repositories {
+  google()
+  jcenter()
+}
+
+dependencies {
+  // Common
+
+  implementation "com.google.firebase:firebase-common:16.0.7"
+
+  androidTestImplementation "androidx.test.espresso:espresso-core:3.1.0"
+  androidTestImplementation "androidx.test.espresso:espresso-idling-resource:3.1.0"
+  androidTestImplementation "androidx.test:runner:1.1.0"
+
+  // Firestore
+  firestoreImplementation "com.google.android.gms:play-services-tasks:16.0.1"
+  firestoreImplementation "com.google.firebase:firebase-auth:16.1.0"
+  firestoreImplementation "com.google.firebase:firebase-firestore:18.1.0"
+  firestoreImplementation "com.google.truth:truth:0.43"
+
+  androidTestFirestoreImplementation "androidx.test:rules:1.1.0"
+  androidTestFirestoreImplementation "junit:junit:4.12"
+}
+
+apply plugin: "com.google.gms.google-services"

smoke-tests/src/androidTest/java/com/google/firebase/testing/common/SmokeTestBase.java

@@ -0,0 +1,78 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.testing.common;
+
+import android.app.Activity;
+import android.util.Log;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+/**
+ * A base class for smoke tests.
+ *
+ * <p>All smoke tests should extend this class. It contains the implementation for finding tests in
+ * the app APK and running them. Test methods must be annotated with {@link
+ * com.google.firebase.testing.common.SmokeTest} and must be in the test's primary activity.
+ *
+ * <p>This design enables test code to be built, obfuscated, and run like any other application's
+ * code. Test methods may be fully defined in one source tree, instead of splitting them across the
+ * two APKs used during testing.
+ */
+public abstract class SmokeTestBase {
+
+  private static final String TAG = "SmokeTestBase";
+
+  /**
+   * Runs all smoke tests in the provided activity.
+   *
+   * <p>This method reflectively obtains all methods defined in the activity with the {@link
+   * com.google.firebase.testing.common.SmokeTest} annotation. These are executed sequentially. Any
+   * exception thrown by the test methods is rethrown here without wrapping.
+   */
+  protected void runSmokeTests(Activity instance) throws Exception {
+    try {
+      Method[] methods = instance.getClass().getMethods();
+      Log.d(TAG, "Searching for SmokeTest methods.");
+      for (Method method : methods) {
+        if (method.getAnnotation(SmokeTest.class) == null) {
+          Log.d(TAG, "Skipping non-SmokeTest method, " + method.getName() + ".");
+          continue;
+        }
+
+        Log.d(TAG, "Preparing to run method, " + method.getName() + ".");
+        try {
+          method.invoke(instance);
+          Log.d(TAG, "Test method complete.");
+        } catch (InvocationTargetException ex) {
+          Throwable t = ex.getCause();
+          if (t instanceof Exception) {
+            throw (Exception) t;
+          } else if (t instanceof Error) {
+            throw (Error) t;
+          }
+
+          throw new IllegalStateException("Test threw unexpected Throwable", t);
+        }
+      }
+    } catch (Exception ex) {
+      // TODO(allisonbm92): Catch more specific exceptions to provide better user experience.
+      // An exception should only be caught here if there was a problem reflectively accessing the
+      // test methods. This corresponds to a user configuration error.
+      throw new IllegalStateException("Test framework error", ex);
+    }
+
+    Log.d(TAG, "Finished test runs.");
+  }
+}

smoke-tests/src/androidTest/java/com/google/firebase/testing/common/TestApkUtil.java

@@ -0,0 +1,73 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.testing.common;
+
+import android.util.Log;
+import androidx.test.espresso.Espresso;
+import androidx.test.espresso.IdlingRegistry;
+import androidx.test.espresso.idling.CountingIdlingResource;
+import com.google.android.gms.tasks.RuntimeExecutionException;
+import com.google.android.gms.tasks.Task;
+
+/**
+ * Smoke test utilities in the test APK.
+ *
+ * <p>The purpose of this class is to avoid adding testing libraries to the application APK that
+ * contains Firebase code and will be obfuscated. The {@code TestUtil} class in the application APK
+ * calls the implementations in this class reflectively.
+ */
+public final class TestApkUtil {
+
+  private static final String TAG = "TestApkUtil";
+
+  private TestApkUtil() {}
+
+  /**
+   * Waits for the task to complete successfully.
+   *
+   * <p>This method will block the current thread and return the result of the task. It will rethrow
+   * any expection thrown by the task without wrapping.
+   */
+  public static <T> T waitForSuccess(Task<T> task) throws Exception {
+    Log.d(TAG, "Begin waiting for task, " + task + ".");
+    CountingIdlingResource idler = new CountingIdlingResource("Task");
+
+    idler.increment();
+    IdlingRegistry.getInstance().register(idler);
+    task.addOnCompleteListener(
+        t -> {
+          Log.d(TAG, "Task completed. Success: " + t.isSuccessful() + ".");
+          idler.decrement();
+        });
+
+    Log.d(TAG, "Wait now.");
+    Espresso.onIdle();
+    Log.d(TAG, "End wait.");
+
+    IdlingRegistry.getInstance().unregister(idler);
+    try {
+      return task.getResult();
+    } catch (RuntimeExecutionException ex) {
+      Throwable t = ex.getCause();
+      if (t instanceof Exception) {
+        throw (Exception) t;
+      } else if (t instanceof Error) {
+        throw (Error) t;
+      }
+
+      throw new IllegalStateException("Task threw unexpected Throwable", t);
+    }
+  }
+}

smoke-tests/src/androidTestFirestore/java/com/google/firebase/testing/firestore/FirestoreTest.java

@@ -0,0 +1,42 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.testing.firestore;
+
+import android.util.Log;
+import androidx.test.rule.ActivityTestRule;
+import androidx.test.runner.AndroidJUnit4;
+import com.google.firebase.testing.common.SmokeTestBase;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(AndroidJUnit4.class)
+public final class FirestoreTest extends SmokeTestBase {
+
+  private static final String TAG = "FirestoreTest";
+
+  @Rule
+  public final ActivityTestRule<FirestoreActivity> atr =
+      new ActivityTestRule<>(FirestoreActivity.class);
+
+  @Test
+  public void runSmokeTests() throws Exception {
+    Log.d(TAG, "Initializing activity.");
+    FirestoreActivity instance = atr.getActivity();
+
+    // Delegate to base class.
+    runSmokeTests(instance);
+  }
+}

smoke-tests/src/firestore/AndroidManifest.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.google.firebase.testing">
+
+  <uses-permission android:name="android.permission.INTERNET" />
+
+  <application>
+    <activity android:name="com.google.firebase.testing.firestore.FirestoreActivity">
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN" />
+        <category android:name="android.intent.category.LAUNCHER" />
+      </intent-filter>
+    </activity>
+  </application>
+</manifest>