Merge

Author: schmidt-sebastian

packages/firestore/CHANGELOG.md

@@ -1,4 +1,9 @@
 # Unreleased
+- [changed] Improved how Firestore handles idle queries to reduce the cost of
+  re-listening within 30 minutes.
+- [changed] Improved offline performance with many outstanding writes.
+
+# 0.6.0
 - [fixed] Fixed an issue where queries returned fewer results than they should,
   caused by documents that were cached as deleted when they should not have
   been (firebase/firebase-ios-sdk#1548). Because some cache data is cleared,

packages/firestore/src/core/sync_engine.ts

@@ -45,6 +45,7 @@ import { Query } from './query';
 import { SnapshotVersion } from './snapshot_version';
 import { TargetIdGenerator } from './target_id_generator';
 import { Transaction } from './transaction';
+<<<<<<< HEAD
 import {
   BatchId,
   MutationBatchState,
@@ -53,6 +54,9 @@ import {
   ProtoByteString,
   TargetId
 } from './types';
+=======
+import { BatchId, OnlineState, TargetId } from './types';
+>>>>>>> master
 import {
   AddedLimboDocument,
   LimboDocumentChange,
@@ -88,12 +92,6 @@ class QueryView {
      * stream to identify this query.
      */
     public targetId: TargetId,
-    /**
-     * An identifier from the datastore backend that indicates the last state
-     * of the results that was received. This can be used to indicate where
-     * to continue receiving new doc changes for the query.
-     */
-    public resumeToken: ProtoByteString,
     /**
      * The view is responsible for computing the final merged truth of what
      * docs are in the query. It gets notified of local and remote changes,
@@ -274,6 +272,7 @@ export class SyncEngine implements RemoteSyncer, SharedClientStateSyncer {
             'applyChanges for new view should always return a snapshot'
           );
 
+<<<<<<< HEAD
           const data = new QueryView(
             query,
             queryData.targetId,
@@ -310,6 +309,17 @@ export class SyncEngine implements RemoteSyncer, SharedClientStateSyncer {
             );
           }
           return viewSnapshot;
+=======
+              const data = new QueryView(query, queryData.targetId, view);
+              this.queryViewsByQuery.set(query, data);
+              this.queryViewsByTarget[queryData.targetId] = data;
+              this.viewHandler!([viewChange.snapshot!]);
+              this.remoteStore.listen(queryData);
+            });
+        })
+        .then(() => {
+          return queryData.targetId;
+>>>>>>> master
         });
     });
   }

packages/firestore/src/local/indexeddb_mutation_queue.ts

@@ -18,6 +18,7 @@ import { Timestamp } from '../api/timestamp';
 import { User } from '../auth/user';
 import { Query } from '../core/query';
 import { BatchId, ProtoByteString } from '../core/types';
+import { DocumentKeySet } from '../model/collections';
 import { DocumentKey } from '../model/document_key';
 import { Mutation } from '../model/mutation';
 import { BATCHID_UNKNOWN, MutationBatch } from '../model/mutation_batch';
@@ -40,8 +41,13 @@ import { LocalSerializer } from './local_serializer';
 import { MutationQueue } from './mutation_queue';
 import { PersistenceTransaction } from './persistence';
 import { PersistencePromise } from './persistence_promise';
+<<<<<<< HEAD
 import { SimpleDb, SimpleDbStore } from './simple_db';
 import { DocumentKeySet } from '../model/collections';
+=======
+import { SimpleDbStore } from './simple_db';
+import { IndexedDbPersistence } from './indexeddb_persistence';
+>>>>>>> master
 
 /** A mutation queue for a specific user, backed by IndexedDB. */
 export class IndexedDbMutationQueue implements MutationQueue {
@@ -342,6 +348,50 @@ export class IndexedDbMutationQueue implements MutationQueue {
       .next(() => results);
   }
 
+  getAllMutationBatchesAffectingDocumentKeys(
+    transaction: PersistenceTransaction,
+    documentKeys: DocumentKeySet
+  ): PersistencePromise<MutationBatch[]> {
+    let uniqueBatchIDs = new SortedSet<BatchId>(primitiveComparator);
+
+    const promises: Array<PersistencePromise<void>> = [];
+    documentKeys.forEach(documentKey => {
+      const indexStart = DbDocumentMutation.prefixForPath(
+        this.userId,
+        documentKey.path
+      );
+      const range = IDBKeyRange.lowerBound(indexStart);
+
+      const promise = documentMutationsStore(transaction).iterate(
+        { range },
+        (indexKey, _, control) => {
+          const [userID, encodedPath, batchID] = indexKey;
+
+          // Only consider rows matching exactly the specific key of
+          // interest. Note that because we order by path first, and we
+          // order terminators before path separators, we'll encounter all
+          // the index rows for documentKey contiguously. In particular, all
+          // the rows for documentKey will occur before any rows for
+          // documents nested in a subcollection beneath documentKey so we
+          // can stop as soon as we hit any such row.
+          const path = EncodedResourcePath.decode(encodedPath);
+          if (userID !== this.userId || !documentKey.path.isEqual(path)) {
+            control.done();
+            return;
+          }
+
+          uniqueBatchIDs = uniqueBatchIDs.add(batchID);
+        }
+      );
+
+      promises.push(promise);
+    });
+
+    return PersistencePromise.waitFor(promises).next(() =>
+      this.lookupMutationBatches(transaction, uniqueBatchIDs)
+    );
+  }
+
   getAllMutationBatchesAffectingQuery(
     transaction: PersistenceTransaction,
     query: Query
@@ -393,6 +443,7 @@ export class IndexedDbMutationQueue implements MutationQueue {
         }
         uniqueBatchIDs = uniqueBatchIDs.add(batchID);
       })
+<<<<<<< HEAD
       .next(() => {
         const results: MutationBatch[] = [];
         const promises: Array<PersistencePromise<void>> = [];
@@ -421,6 +472,36 @@ export class IndexedDbMutationQueue implements MutationQueue {
         });
         return PersistencePromise.waitFor(promises).next(() => results);
       });
+=======
+      .next(() => this.lookupMutationBatches(transaction, uniqueBatchIDs));
+  }
+
+  private lookupMutationBatches(
+    transaction: PersistenceTransaction,
+    batchIDs: SortedSet<BatchId>
+  ): PersistencePromise<MutationBatch[]> {
+    const results: MutationBatch[] = [];
+    const promises: Array<PersistencePromise<void>> = [];
+    // TODO(rockwood): Implement this using iterate.
+    batchIDs.forEach(batchID => {
+      const mutationKey = this.keyForBatchId(batchID);
+      promises.push(
+        mutationsStore(transaction)
+          .get(mutationKey)
+          .next(mutation => {
+            if (mutation === null) {
+              fail(
+                'Dangling document-mutation reference found, ' +
+                  'which points to ' +
+                  mutationKey
+              );
+            }
+            results.push(this.serializer.fromDbMutationBatch(mutation!));
+          })
+      );
+    });
+    return PersistencePromise.waitFor(promises).next(() => results);
+>>>>>>> master
   }
 
   removeMutationBatches(
@@ -567,7 +648,7 @@ function convertStreamToken(token: ProtoByteString): string {
 function mutationsStore(
   txn: PersistenceTransaction
 ): SimpleDbStore<DbMutationBatchKey, DbMutationBatch> {
-  return SimpleDb.getStore<DbMutationBatchKey, DbMutationBatch>(
+  return IndexedDbPersistence.getStore<DbMutationBatchKey, DbMutationBatch>(
     txn,
     DbMutationBatch.store
   );
@@ -579,10 +660,10 @@ function mutationsStore(
 function documentMutationsStore(
   txn: PersistenceTransaction
 ): SimpleDbStore<DbDocumentMutationKey, DbDocumentMutation> {
-  return SimpleDb.getStore<DbDocumentMutationKey, DbDocumentMutation>(
-    txn,
-    DbDocumentMutation.store
-  );
+  return IndexedDbPersistence.getStore<
+    DbDocumentMutationKey,
+    DbDocumentMutation
+  >(txn, DbDocumentMutation.store);
 }
 
 /**
@@ -591,7 +672,7 @@ function documentMutationsStore(
 function mutationQueuesStore(
   txn: PersistenceTransaction
 ): SimpleDbStore<DbMutationQueueKey, DbMutationQueue> {
-  return SimpleDb.getStore<DbMutationQueueKey, DbMutationQueue>(
+  return IndexedDbPersistence.getStore<DbMutationQueueKey, DbMutationQueue>(
     txn,
     DbMutationQueue.store
   );

packages/firestore/src/local/indexeddb_persistence.ts

@@ -17,7 +17,7 @@
 import { User } from '../auth/user';
 import { DatabaseInfo } from '../core/database_info';
 import { JsonProtoSerializer } from '../remote/serializer';
-import { assert } from '../util/assert';
+import { assert, fail } from '../util/assert';
 import { Code, FirestoreError } from '../util/error';
 import * as log from '../util/log';
 
@@ -35,19 +35,26 @@ import {
 } from './indexeddb_schema';
 import { LocalSerializer } from './local_serializer';
 import { MutationQueue } from './mutation_queue';
+<<<<<<< HEAD
 import {
   Persistence,
   PersistenceTransaction,
   PrimaryStateListener
 } from './persistence';
+=======
+import { Persistence, PersistenceTransaction } from './persistence';
+>>>>>>> master
 import { PersistencePromise } from './persistence_promise';
 import { QueryCache } from './query_cache';
 import { RemoteDocumentCache } from './remote_document_cache';
 import { SimpleDb, SimpleDbStore, SimpleDbTransaction } from './simple_db';
+<<<<<<< HEAD
 import { Platform } from '../platform/platform';
 import { AsyncQueue, TimerId } from '../util/async_queue';
 import { ClientId } from './shared_client_state';
 import { CancelablePromise } from '../util/promise';
+=======
+>>>>>>> master
 
 const LOG_TAG = 'IndexedDbPersistence';
 
@@ -79,9 +86,17 @@ const UNSUPPORTED_PLATFORM_ERROR_MSG =
   ' IndexedDB or is known to have an incomplete implementation. Offline' +
   ' persistence has been disabled.';
 
+<<<<<<< HEAD
 // The format of the LocalStorage key that stores zombied client is:
 //     firestore_zombie_<persistence_prefix>_<instance_key>
 const ZOMBIED_CLIENTS_KEY_PREFIX = 'firestore_zombie';
+=======
+export class IndexedDbTransaction extends PersistenceTransaction {
+  constructor(readonly simpleDbTransaction: SimpleDbTransaction) {
+    super();
+  }
+}
+>>>>>>> master
 
 /**
  * An IndexedDB-backed instance of Persistence. Data is stored persistently
@@ -115,6 +130,17 @@ const ZOMBIED_CLIENTS_KEY_PREFIX = 'firestore_zombie';
  * TODO(multitab): Update this comment with multi-tab changes.
  */
 export class IndexedDbPersistence implements Persistence {
+  static getStore<Key extends IDBValidKey, Value>(
+    txn: PersistenceTransaction,
+    store: string
+  ): SimpleDbStore<Key, Value> {
+    if (txn instanceof IndexedDbTransaction) {
+      return SimpleDb.getStore<Key, Value>(txn.simpleDbTransaction, store);
+    } else {
+      fail('IndexedDbPersistence must use instances of IndexedDbTransaction');
+    }
+  }
+
   /**
    * The name of the main (and currently only) IndexedDB database. this name is
    * appended to the prefix provided to the IndexedDbPersistence constructor.
@@ -468,10 +494,14 @@ export class IndexedDbPersistence implements Persistence {
 
   runTransaction<T>(
     action: string,
+<<<<<<< HEAD
     requirePrimaryLease: boolean,
     transactionOperation: (
       transaction: PersistenceTransaction
     ) => PersistencePromise<T>
+=======
+    operation: (transaction: IndexedDbTransaction) => PersistencePromise<T>
+>>>>>>> master
   ): Promise<T> {
     // TODO(multitab): Consider removing `requirePrimaryLease` and exposing
     // three different write modes (readonly, readwrite, readwrite_primary).
@@ -483,6 +513,7 @@ export class IndexedDbPersistence implements Persistence {
 
     // Do all transactions as readwrite against all object stores, since we
     // are the only reader/writer.
+<<<<<<< HEAD
     return this.simpleDb.runTransaction('readwrite', ALL_STORES, txn => {
       if (requirePrimaryLease) {
         // While we merely verify that we have (or can acquire) the lease
@@ -516,6 +547,18 @@ export class IndexedDbPersistence implements Persistence {
         );
       }
     });
+=======
+    return this.simpleDb.runTransaction(
+      'readwrite',
+      ALL_STORES,
+      simpleDbTxn => {
+        // Verify that we still have the owner lease as part of every transaction.
+        return this.ensureOwnerLease(simpleDbTxn).next(() =>
+          operation(new IndexedDbTransaction(simpleDbTxn))
+        );
+      }
+    );
+>>>>>>> master
   }
 
   /**
@@ -616,6 +659,7 @@ export class IndexedDbPersistence implements Persistence {
     return true;
   }
 
+<<<<<<< HEAD
   private attachVisibilityHandler(): void {
     if (
       this.document !== null &&
@@ -627,6 +671,27 @@ export class IndexedDbPersistence implements Persistence {
           return this.updateClientMetadataAndTryBecomePrimary();
         });
       };
+=======
+  /**
+   * Schedules a recurring timer to update the owner lease timestamp to prevent
+   * other tabs from taking the lease.
+   */
+  private scheduleOwnerLeaseRefreshes(): void {
+    // NOTE: This doesn't need to be scheduled on the async queue and doing so
+    // would increase the chances of us not refreshing on time if the queue is
+    // backed up for some reason.
+    this.ownerLeaseRefreshHandle = setInterval(() => {
+      const txResult = this.simpleDb.runTransaction(
+        'readwrite',
+        ALL_STORES,
+        txn => {
+          // NOTE: We don't need to validate the current owner contents, since
+          // runTransaction does that automatically.
+          const store = txn.store<DbOwnerKey, DbOwner>(DbOwner.store);
+          return store.put('owner', new DbOwner(this.ownerId, Date.now()));
+        }
+      );
+>>>>>>> master
 
       this.document.addEventListener(
         'visibilitychange',