Random changes for Indexing (#3169)

Author: schmidt-sebastian

firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java

@@ -49,10 +49,10 @@ public void tearDown() {
   }
 
   @Test
-  public void testCanConfigureIndices() throws ExecutionException, InterruptedException {
+  public void testCanConfigureIndexes() throws ExecutionException, InterruptedException {
     FirebaseFirestore db = testFirestore();
     Task<Void> indexTask =
-        db.configureIndices(
+        db.setIndexConfiguration(
             "{\n"
                 + "  \"indexes\": [\n"
                 + "    {\n"
@@ -89,7 +89,7 @@ public void testCanConfigureIndices() throws ExecutionException, InterruptedExce
   public void testBadJsonDoesNotCrashClient() {
     FirebaseFirestore db = testFirestore();
 
-    Assert.assertThrows(IllegalArgumentException.class, () -> db.configureIndices("{,"));
+    Assert.assertThrows(IllegalArgumentException.class, () -> db.setIndexConfiguration("{,"));
   }
 
   @Test
@@ -98,7 +98,7 @@ public void testBadIndexDoesNotCrashClient() {
     Assert.assertThrows(
         IllegalArgumentException.class,
         () ->
-            db.configureIndices(
+            db.setIndexConfiguration(
                 "{\n"
                     + "  \"indexes\": [\n"
                     + "    {\n"

firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java

@@ -297,13 +297,14 @@ public FirebaseApp getApp() {
    * automatically start using the index once the index entries have been written.
    *
    * <p>The method accepts the JSON format exported by the Firebase CLI (`firebase
-   * firestore:indexes`). If the JSON format is invalid, this method rejects the returned task.
+   * firestore:indexes`). If the JSON format is invalid, this method throws an exception.
    *
    * @param json The JSON format exported by the Firebase CLI.
    * @return A task that resolves once all indices are successfully configured.
+   * @throws IllegalArgumentException if the JSON format is invalid
    */
   @VisibleForTesting
-  Task<Void> configureIndices(String json) {
+  Task<Void> setIndexConfiguration(String json) {
     ensureClientConfigured();
 
     // Preconditions.checkState(BuildConfig.ENABLE_INDEXING, "Indexing support is not yet

firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexBackfiller.java

@@ -26,6 +26,7 @@
 import com.google.firebase.firestore.model.ResourcePath;
 import com.google.firebase.firestore.model.SnapshotVersion;
 import com.google.firebase.firestore.util.AsyncQueue;
+import com.google.firebase.firestore.util.Logger;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -37,6 +38,8 @@
 
 /** Implements the steps for backfilling indexes. */
 public class IndexBackfiller {
+  private static final String LOG_TAG = "IndexBackfiller";
+
   /** How long we wait to try running index backfill after SDK initialization. */
   private static final long INITIAL_BACKFILL_DELAY_MS = TimeUnit.SECONDS.toMillis(15);
   /** Minimum amount of time between backfill checks, after the first one. */
@@ -65,29 +68,6 @@ public void setIndexManager(IndexManager indexManager) {
     this.indexManager = indexManager;
   }
 
-  public static class Results {
-    private final boolean hasRun;
-
-    private final int documentsProcessed;
-
-    static IndexBackfiller.Results DidNotRun() {
-      return new IndexBackfiller.Results(/* hasRun= */ false, 0);
-    }
-
-    Results(boolean hasRun, int documentsProcessed) {
-      this.hasRun = hasRun;
-      this.documentsProcessed = documentsProcessed;
-    }
-
-    public boolean hasRun() {
-      return hasRun;
-    }
-
-    public int getDocumentsProcessed() {
-      return documentsProcessed;
-    }
-  }
-
   public class Scheduler implements com.google.firebase.firestore.local.Scheduler {
     private boolean hasRun = false;
     @Nullable private AsyncQueue.DelayedTask backfillTask;
@@ -118,7 +98,8 @@ private void scheduleBackfill() {
               AsyncQueue.TimerId.INDEX_BACKFILL,
               delay,
               () -> {
-                backfill();
+                int documentsProcessed = backfill();
+                Logger.debug(LOG_TAG, "Documents written: %s", documentsProcessed);
                 hasRun = true;
                 scheduleBackfill();
               });
@@ -129,15 +110,12 @@ public Scheduler getScheduler() {
     return scheduler;
   }
 
-  public Results backfill() {
+  /** Runs a single backfill operation and returns the number of documents processed. */
+  public int backfill() {
     hardAssert(localDocumentsView != null, "setLocalDocumentsView() not called");
     hardAssert(indexManager != null, "setIndexManager() not called");
     return persistence.runTransaction(
-        "Backfill Indexes",
-        () -> {
-          int documentsProcessed = writeIndexEntries(localDocumentsView);
-          return new Results(/* hasRun= */ true, documentsProcessed);
-        });
+        "Backfill Indexes", () -> writeIndexEntries(localDocumentsView));
   }
 
   /** Writes index entries until the cap is reached. Returns the number of documents processed. */
@@ -149,6 +127,7 @@ private int writeIndexEntries(LocalDocumentsView localDocumentsView) {
       if (collectionGroup == null || processedCollectionGroups.contains(collectionGroup)) {
         break;
       }
+      Logger.debug(LOG_TAG, "Processing collection: %s", collectionGroup);
       documentsRemaining -=
           writeEntriesForCollectionGroup(localDocumentsView, collectionGroup, documentsRemaining);
       processedCollectionGroups.add(collectionGroup);

firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java

@@ -130,7 +130,7 @@ public final class LocalStore implements BundleCallback {
   private LocalDocumentsView localDocuments;
 
   /** Performs queries over the localDocuments (and potentially maintains indexes). */
-  private QueryEngine queryEngine;
+  private final QueryEngine queryEngine;
 
   /** The set of document references maintained by any local views. */
   private final ReferenceSet localViewReferences;
@@ -170,6 +170,7 @@ public LocalStore(
     this.queryEngine = queryEngine;
     this.indexBackfiller = indexBackfiller;
     queryEngine.setLocalDocumentsView(localDocuments);
+    queryEngine.setIndexManager(indexManager);
 
     localViewReferences = new ReferenceSet();
     persistence.getReferenceDelegate().setInMemoryPins(localViewReferences);

firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java

@@ -358,9 +358,11 @@ private void createV1RemoteDocumentCache() {
    * <p>The `index_configuration` table holds the configuration for all indices. Entries in this
    * table apply for all users. It is not possible to only enable indices for a subset of users.
    *
+   * <p>The `index_state` table holds backfill information on each index that stores when it was
+   * last updated.
+   *
    * <p>The `index_entries` table holds the index values themselves. An index value is created for
-   * each field combination that matches a configured index. If there are pending mutations that
-   * affect an indexed field, an additional index entry is created per mutated field.
+   * each field combination that matches a configured index.
    */
   private void createFieldIndex() {
     ifTablesDontExist(

firebase-firestore/src/test/java/com/google/firebase/firestore/local/IndexBackfillerTest.java

@@ -99,8 +99,8 @@ public void testBackfillWritesLatestReadTimeToFieldIndexOnCompletion() {
     addDoc("coll1/docA", "foo", version(10));
     addDoc("coll2/docA", "bar", version(20));
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(2, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(2, documentsProcessed);
 
     FieldIndex fieldIndex1 = indexManager.getFieldIndexes("coll1").iterator().next();
     FieldIndex fieldIndex2 = indexManager.getFieldIndexes("coll2").iterator().next();
@@ -112,8 +112,8 @@ public void testBackfillWritesLatestReadTimeToFieldIndexOnCompletion() {
     addDoc("coll2/docB", "bar", version(60));
     addDoc("coll2/docC", "bar", version(60, 10));
 
-    results = backfiller.backfill();
-    assertEquals(4, results.getDocumentsProcessed());
+    documentsProcessed = backfiller.backfill();
+    assertEquals(4, documentsProcessed);
 
     fieldIndex1 = indexManager.getFieldIndexes("coll1").iterator().next();
     fieldIndex2 = indexManager.getFieldIndexes("coll2").iterator().next();
@@ -127,17 +127,17 @@ public void testBackfillFetchesDocumentsAfterEarliestReadTime() {
 
     // Documents before earliest read time should not be fetched.
     addDoc("coll1/docA", "foo", version(9));
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(0, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(0, documentsProcessed);
 
     // Read time of index should not change.
     Iterator<FieldIndex> it = indexManager.getFieldIndexes("coll1").iterator();
     assertEquals(version(10), it.next().getIndexState().getReadTime());
 
     // Documents that are after the earliest read time but before field index read time are fetched.
     addDoc("coll1/docB", "boo", version(19));
-    results = backfiller.backfill();
-    assertEquals(1, results.getDocumentsProcessed());
+    documentsProcessed = backfiller.backfill();
+    assertEquals(1, documentsProcessed);
 
     // Field indexes should now hold the latest read time
     it = indexManager.getFieldIndexes("coll1").iterator();
@@ -153,8 +153,8 @@ public void testBackfillWritesIndexEntries() {
     addDoc("coll2/docA", "bar", version(10));
     addDoc("coll2/docB", "car", version(10));
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(4, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(4, documentsProcessed);
   }
 
   @Test
@@ -167,13 +167,13 @@ public void testBackfillWritesOldestDocumentFirst() {
     addDoc("coll1/docB", "foo", version(3));
     addDoc("coll1/docC", "foo", version(10));
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(2, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(2, documentsProcessed);
 
     verifyQueryResults("coll1", "coll1/docA", "coll1/docB");
 
-    results = backfiller.backfill();
-    assertEquals(1, results.getDocumentsProcessed());
+    documentsProcessed = backfiller.backfill();
+    assertEquals(1, documentsProcessed);
 
     verifyQueryResults("coll1", "coll1/docA", "coll1/docB", "coll1/docC");
   }
@@ -192,8 +192,8 @@ public void testBackfillUpdatesCollectionGroups() {
     String collectionGroup = indexManager.getNextCollectionGroupToUpdate();
     assertEquals("coll1", collectionGroup);
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(2, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(2, documentsProcessed);
 
     // Check that coll1 was backfilled and that coll2 is next
     collectionGroup = indexManager.getNextCollectionGroupToUpdate();
@@ -217,8 +217,8 @@ public void testBackfillPrioritizesNewCollectionGroups() {
     // Check that coll3 is the next collection ID the backfiller should update
     assertEquals("coll3", indexManager.getNextCollectionGroupToUpdate());
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(1, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(1, documentsProcessed);
 
     verifyQueryResults("coll3", "coll3/doc");
   }
@@ -233,8 +233,8 @@ public void testBackfillWritesUntilCap() {
     addDoc("coll2/docA", "foo", version(30));
     addDoc("coll2/docA", "foo", version(40));
 
-    IndexBackfiller.Results results = backfiller.backfill();
-    assertEquals(3, results.getDocumentsProcessed());
+    int documentsProcessed = backfiller.backfill();
+    assertEquals(3, documentsProcessed);
 
     verifyQueryResults("coll1", "coll1/docA", "coll1/docB");
     verifyQueryResults("coll2", "coll2/docA");