Initial base classes for index backfill (#2950)

Author: Brian Chen

firebase-firestore/src/main/java/com/google/firebase/firestore/core/ComponentProvider.java

@@ -18,9 +18,9 @@
 import androidx.annotation.Nullable;
 import com.google.firebase.firestore.FirebaseFirestoreSettings;
 import com.google.firebase.firestore.auth.User;
-import com.google.firebase.firestore.local.GarbageCollectionScheduler;
 import com.google.firebase.firestore.local.LocalStore;
 import com.google.firebase.firestore.local.Persistence;
+import com.google.firebase.firestore.local.Scheduler;
 import com.google.firebase.firestore.remote.ConnectivityMonitor;
 import com.google.firebase.firestore.remote.Datastore;
 import com.google.firebase.firestore.remote.RemoteStore;
@@ -38,8 +38,9 @@ public abstract class ComponentProvider {
   private SyncEngine syncEngine;
   private RemoteStore remoteStore;
   private EventManager eventManager;
-  private ConnectivityMonitor connectityMonitor;
-  @Nullable private GarbageCollectionScheduler gargabeCollectionScheduler;
+  private ConnectivityMonitor connectivityMonitor;
+  @Nullable private Scheduler garbageCollectionScheduler;
+  @Nullable private Scheduler indexBackfillScheduler;
 
   /** Configuration options for the component provider. */
   public static class Configuration {
@@ -103,8 +104,13 @@ public Persistence getPersistence() {
   }
 
   @Nullable
-  public GarbageCollectionScheduler getGargabeCollectionScheduler() {
-    return gargabeCollectionScheduler;
+  public Scheduler getGarbageCollectionScheduler() {
+    return garbageCollectionScheduler;
+  }
+
+  @Nullable
+  public Scheduler getIndexBackfillScheduler() {
+    return indexBackfillScheduler;
   }
 
   public LocalStore getLocalStore() {
@@ -124,24 +130,26 @@ public EventManager getEventManager() {
   }
 
   protected ConnectivityMonitor getConnectivityMonitor() {
-    return connectityMonitor;
+    return connectivityMonitor;
   }
 
   public void initialize(Configuration configuration) {
     persistence = createPersistence(configuration);
     persistence.start();
     localStore = createLocalStore(configuration);
-    connectityMonitor = createConnectivityMonitor(configuration);
+    connectivityMonitor = createConnectivityMonitor(configuration);
     remoteStore = createRemoteStore(configuration);
     syncEngine = createSyncEngine(configuration);
     eventManager = createEventManager(configuration);
     localStore.start();
     remoteStore.start();
-    gargabeCollectionScheduler = createGarbageCollectionScheduler(configuration);
+    garbageCollectionScheduler = createGarbageCollectionScheduler(configuration);
+    indexBackfillScheduler = createIndexBackfillScheduler(configuration);
   }
 
-  protected abstract GarbageCollectionScheduler createGarbageCollectionScheduler(
-      Configuration configuration);
+  protected abstract Scheduler createGarbageCollectionScheduler(Configuration configuration);
+
+  protected abstract Scheduler createIndexBackfillScheduler(Configuration configuration);
 
   protected abstract EventManager createEventManager(Configuration configuration);
 

firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java

@@ -32,10 +32,10 @@
 import com.google.firebase.firestore.bundle.BundleSerializer;
 import com.google.firebase.firestore.bundle.NamedQuery;
 import com.google.firebase.firestore.core.EventManager.ListenOptions;
-import com.google.firebase.firestore.local.GarbageCollectionScheduler;
 import com.google.firebase.firestore.local.LocalStore;
 import com.google.firebase.firestore.local.Persistence;
 import com.google.firebase.firestore.local.QueryResult;
+import com.google.firebase.firestore.local.Scheduler;
 import com.google.firebase.firestore.model.Document;
 import com.google.firebase.firestore.model.DocumentKey;
 import com.google.firebase.firestore.model.FieldIndex;
@@ -74,7 +74,9 @@ public final class FirestoreClient {
   private EventManager eventManager;
 
   // LRU-related
-  @Nullable private GarbageCollectionScheduler gcScheduler;
+  @Nullable private Scheduler gcScheduler;
+
+  @Nullable private Scheduler indexBackfillScheduler;
 
   public FirestoreClient(
       final Context context,
@@ -143,6 +145,10 @@ public Task<Void> terminate() {
           if (gcScheduler != null) {
             gcScheduler.stop();
           }
+
+          if (indexBackfillScheduler != null) {
+            indexBackfillScheduler.stop();
+          }
         });
   }
 
@@ -255,7 +261,7 @@ private void initialize(Context context, User user, FirebaseFirestoreSettings se
             : new MemoryComponentProvider();
     provider.initialize(configuration);
     persistence = provider.getPersistence();
-    gcScheduler = provider.getGargabeCollectionScheduler();
+    gcScheduler = provider.getGarbageCollectionScheduler();
     localStore = provider.getLocalStore();
     remoteStore = provider.getRemoteStore();
     syncEngine = provider.getSyncEngine();
@@ -264,6 +270,12 @@ private void initialize(Context context, User user, FirebaseFirestoreSettings se
     if (gcScheduler != null) {
       gcScheduler.start();
     }
+
+    if (Persistence.INDEXING_SUPPORT_ENABLED && settings.isPersistenceEnabled()) {
+      indexBackfillScheduler = provider.getIndexBackfillScheduler();
+      hardAssert(indexBackfillScheduler != null, "Index backfill scheduler should not be null.");
+      indexBackfillScheduler.start();
+    }
   }
 
   public void addSnapshotsInSyncListener(EventListener<Void> listener) {

firebase-firestore/src/main/java/com/google/firebase/firestore/core/MemoryComponentProvider.java

@@ -17,10 +17,10 @@
 import androidx.annotation.Nullable;
 import com.google.firebase.database.collection.ImmutableSortedSet;
 import com.google.firebase.firestore.local.DefaultQueryEngine;
-import com.google.firebase.firestore.local.GarbageCollectionScheduler;
 import com.google.firebase.firestore.local.LocalStore;
 import com.google.firebase.firestore.local.MemoryPersistence;
 import com.google.firebase.firestore.local.Persistence;
+import com.google.firebase.firestore.local.Scheduler;
 import com.google.firebase.firestore.model.DocumentKey;
 import com.google.firebase.firestore.model.mutation.MutationBatchResult;
 import com.google.firebase.firestore.remote.AndroidConnectivityMonitor;
@@ -36,8 +36,13 @@ public class MemoryComponentProvider extends ComponentProvider {
 
   @Override
   @Nullable
-  protected GarbageCollectionScheduler createGarbageCollectionScheduler(
-      Configuration configuration) {
+  protected Scheduler createGarbageCollectionScheduler(Configuration configuration) {
+    return null;
+  }
+
+  @Override
+  @Nullable
+  protected Scheduler createIndexBackfillScheduler(Configuration configuration) {
     return null;
   }
 

firebase-firestore/src/main/java/com/google/firebase/firestore/core/SQLiteComponentProvider.java

@@ -14,25 +14,31 @@
 
 package com.google.firebase.firestore.core;
 
-import com.google.firebase.firestore.local.GarbageCollectionScheduler;
+import com.google.firebase.firestore.local.IndexBackfiller;
 import com.google.firebase.firestore.local.LocalSerializer;
 import com.google.firebase.firestore.local.LruDelegate;
 import com.google.firebase.firestore.local.LruGarbageCollector;
 import com.google.firebase.firestore.local.Persistence;
 import com.google.firebase.firestore.local.SQLitePersistence;
+import com.google.firebase.firestore.local.Scheduler;
 import com.google.firebase.firestore.remote.RemoteSerializer;
 
 /** Provides all components needed for Firestore with SQLite persistence. */
 public class SQLiteComponentProvider extends MemoryComponentProvider {
 
   @Override
-  protected GarbageCollectionScheduler createGarbageCollectionScheduler(
-      Configuration configuration) {
+  protected Scheduler createGarbageCollectionScheduler(Configuration configuration) {
     LruDelegate lruDelegate = ((SQLitePersistence) getPersistence()).getReferenceDelegate();
     LruGarbageCollector gc = lruDelegate.getGarbageCollector();
     return gc.newScheduler(configuration.getAsyncQueue(), getLocalStore());
   }
 
+  @Override
+  protected Scheduler createIndexBackfillScheduler(Configuration configuration) {
+    IndexBackfiller indexBackfiller = ((SQLitePersistence) getPersistence()).getIndexBackfiller();
+    return indexBackfiller.newScheduler(configuration.getAsyncQueue(), getLocalStore());
+  }
+
   @Override
   protected Persistence createPersistence(Configuration configuration) {
     LocalSerializer serializer =

firebase-firestore/src/main/java/com/google/firebase/firestore/index/IndexEntry.java

@@ -0,0 +1,50 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore.index;
+
+/**
+ * Represents an index entry saved by the SDK in the local storage. Temporary placeholder, since
+ * we'll probably serialize the indexValue right away rather than store it.
+ */
+// TODO(indexing)
+public class IndexEntry {
+  private final int indexId;
+  private final byte[] indexValue;
+  private final String uid;
+  private final String documentId;
+
+  public IndexEntry(int indexId, byte[] indexValue, String uid, String documentId) {
+    this.indexId = indexId;
+    this.indexValue = indexValue;
+    this.uid = uid;
+    this.documentId = documentId;
+  }
+
+  public int getIndexId() {
+    return indexId;
+  }
+
+  public byte[] getIndexValue() {
+    return indexValue;
+  }
+
+  public String getUid() {
+    return uid;
+  }
+
+  public String getDocumentId() {
+    return documentId;
+  }
+}

firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexBackfiller.java

@@ -0,0 +1,152 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.firestore.local;
+
+import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
+import com.google.firebase.firestore.index.IndexEntry;
+import com.google.firebase.firestore.util.AsyncQueue;
+import java.util.concurrent.TimeUnit;
+
+/** Implements the steps for backfilling indexes. */
+public class IndexBackfiller {
+  /** How long we wait to try running index backfill after SDK initialization. */
+  private static final long INITIAL_BACKFILL_DELAY_MS = TimeUnit.SECONDS.toMillis(15);
+  /** Minimum amount of time between backfill checks, after the first one. */
+  private static final long REGULAR_BACKFILL_DELAY_MS = TimeUnit.MINUTES.toMillis(1);
+
+  private final SQLitePersistence persistence;
+
+  public IndexBackfiller(SQLitePersistence sqLitePersistence) {
+    this.persistence = sqLitePersistence;
+  }
+
+  public static class Results {
+    private final boolean hasRun;
+
+    private final int entriesAdded;
+    private final int entriesRemoved;
+
+    static IndexBackfiller.Results DidNotRun() {
+      return new IndexBackfiller.Results(/* hasRun= */ false, 0, 0);
+    }
+
+    Results(boolean hasRun, int entriesAdded, int entriesRemoved) {
+      this.hasRun = hasRun;
+      this.entriesAdded = entriesAdded;
+      this.entriesRemoved = entriesRemoved;
+    }
+
+    public boolean hasRun() {
+      return hasRun;
+    }
+
+    public int getEntriesAdded() {
+      return entriesAdded;
+    }
+
+    public int getEntriesRemoved() {
+      return entriesRemoved;
+    }
+  }
+
+  public class BackfillScheduler implements Scheduler {
+    private final AsyncQueue asyncQueue;
+    private final LocalStore localStore;
+    private boolean hasRun = false;
+    @Nullable private AsyncQueue.DelayedTask backfillTask;
+
+    public BackfillScheduler(AsyncQueue asyncQueue, LocalStore localStore) {
+      this.asyncQueue = asyncQueue;
+      this.localStore = localStore;
+    }
+
+    @Override
+    public void start() {
+      scheduleBackfill();
+    }
+
+    @Override
+    public void stop() {
+      if (backfillTask != null) {
+        backfillTask.cancel();
+      }
+    }
+
+    private void scheduleBackfill() {
+      long delay = hasRun ? REGULAR_BACKFILL_DELAY_MS : INITIAL_BACKFILL_DELAY_MS;
+      backfillTask =
+          asyncQueue.enqueueAfterDelay(
+              AsyncQueue.TimerId.INDEX_BACKFILL,
+              delay,
+              () -> {
+                localStore.backfillIndexes(IndexBackfiller.this);
+                hasRun = true;
+                scheduleBackfill();
+              });
+    }
+  }
+
+  public BackfillScheduler newScheduler(AsyncQueue asyncQueue, LocalStore localStore) {
+    return new BackfillScheduler(asyncQueue, localStore);
+  }
+
+  // TODO(indexing): Figure out which index entries to backfill.
+  public Results backfill() {
+    int numIndexesWritten = 0;
+    int numIndexesRemoved = 0;
+    return new Results(/* hasRun= */ true, numIndexesWritten, numIndexesRemoved);
+  }
+
+  @VisibleForTesting
+  void addIndexEntry(IndexEntry entry) {
+    persistence.execute(
+        "INSERT OR IGNORE INTO index_entries ("
+            + "index_id, "
+            + "index_value, "
+            + "uid, "
+            + "document_id) VALUES(?, ?, ?, ?)",
+        entry.getIndexId(),
+        entry.getIndexValue(),
+        entry.getUid(),
+        entry.getDocumentId());
+  }
+
+  @VisibleForTesting
+  void removeIndexEntry(int indexId, String uid, String documentId) {
+    persistence.execute(
+        "DELETE FROM index_entries "
+            + "WHERE index_id = ? "
+            + "AND uid = ?"
+            + "AND document_id = ?",
+        indexId,
+        uid,
+        documentId);
+    ;
+  }
+
+  @Nullable
+  @VisibleForTesting
+  IndexEntry getIndexEntry(int indexId) {
+    return persistence
+        .query("SELECT index_value, uid, document_id FROM index_entries WHERE index_id = ?")
+        .binding(indexId)
+        .firstValue(
+            row ->
+                row == null
+                    ? null
+                    : new IndexEntry(indexId, row.getBlob(0), row.getString(1), row.getString(2)));
+  }
+}