Merge branch 'master' into log

Mocha version in branch has updated to 5.2.0.

Author: yifanyang

.github/CODEOWNERS

@@ -19,9 +19,9 @@ packages/storage  @schmidt-sebastian
 packages/storage-types  @schmidt-sebastian
 
 # Messaging Code
-packages/messaging  @gauntface @pinarx @mmermerkaya @dwoffinden
-packages/messaging-types  @gauntface @pinarx @mmermerkaya @dwoffinden
-integration/messaging  @gauntface @pinarx @mmermerkaya @dwoffinden
+packages/messaging  @pinarx @mmermerkaya @dwoffinden
+packages/messaging-types  @pinarx @mmermerkaya @dwoffinden
+integration/messaging  @pinarx @mmermerkaya @dwoffinden
 
 # Auth Code
 packages/auth  @bojeil-google @wti806

integration/browserify/package.json

@@ -22,6 +22,6 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
     "mkdirp": "0.5.1",
-    "mocha": "5.0.5"
+    "mocha": "5.2.0"
   }
 }

integration/firestore/package.json

@@ -21,7 +21,7 @@
     "karma-firefox-launcher": "1.1.0",
     "karma-mocha": "1.3.0",
     "karma-spec-reporter": "0.0.32",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "ts-loader": "3.5.0",
     "typescript": "2.8.1",
     "webpack": "3.11.0",

integration/messaging/package.json

@@ -15,7 +15,7 @@
     "chromedriver": "2.37.0",
     "express": "4.16.3",
     "geckodriver": "1.11.0",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "node-fetch": "2.1.2",
     "selenium-assistant": "5.3.0",
     "sinon": "4.5.0"

integration/typescript/package.json

@@ -23,7 +23,7 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
     "karma-typescript": "3.0.12",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "typescript": "2.8.1"
   }

integration/webpack/package.json

@@ -20,7 +20,7 @@
     "karma-mocha": "1.3.0",
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "webpack": "3.11.0"
   }
 }

packages/app/package.json

@@ -42,7 +42,7 @@
     "karma-sourcemap-loader": "0.3.7",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",

packages/database/package.json

@@ -44,7 +44,7 @@
     "karma-sourcemap-loader": "0.3.7",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",

packages/database/test/helpers/EventAccumulator.ts

@@ -17,7 +17,7 @@
 export const EventAccumulatorFactory = {
   waitsForCount: maxCount => {
     let count = 0;
-    const condition = () => ea.eventData.length >= count;
+    const condition = () => count >= maxCount;
     const ea = new EventAccumulator(condition);
     ea.onReset(() => {
       count = 0;

packages/firestore/package.json

@@ -55,7 +55,7 @@
     "karma-webpack": "2.0.9",
     "long": "3.2.0",
     "mkdirp": "0.5.1",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "prettier": "1.12.0",

packages/firestore/src/local/indexeddb_schema.ts

@@ -23,6 +23,7 @@ import { SnapshotVersion } from '../core/snapshot_version';
 import { BATCHID_UNKNOWN } from '../model/mutation_batch';
 import { encode, EncodedResourcePath } from './encoded_resource_path';
 import { removeMutationBatch } from './indexeddb_mutation_queue';
+import { getHighestListenSequenceNumber } from './indexeddb_query_cache';
 import { dbDocumentSize } from './indexeddb_remote_document_cache';
 import { LocalSerializer } from './local_serializer';
 import { PersistencePromise } from './persistence_promise';
@@ -39,8 +40,10 @@ import { SimpleDbSchemaConverter, SimpleDbTransaction } from './simple_db';
  *    https://github.com/firebase/firebase-ios-sdk/issues/1548
  * 4. Multi-Tab Support.
  * 5. Removal of held write acks.
+ * 6. Create document global for tracking document cache size.
+ * 7. Ensure every cached document has a sentinel row with a sequence number.
  */
-export const SCHEMA_VERSION = 6;
+export const SCHEMA_VERSION = 7;
 
 /** Performs database creation and schema upgrades. */
 export class SchemaConverter implements SimpleDbSchemaConverter {
@@ -115,6 +118,10 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
       });
     }
 
+    if (fromVersion < 7 && toVersion >= 7) {
+      p = p.next(() => this.ensureSequenceNumbers(txn));
+    }
+
     return p;
   }
 
@@ -172,6 +179,53 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
       });
     });
   }
+
+  /**
+   * Ensures that every document in the remote document cache has a corresponding sentinel row
+   * with a sequence number. Missing rows are given the most recently used sequence number.
+   */
+  private ensureSequenceNumbers(
+    txn: SimpleDbTransaction
+  ): PersistencePromise<void> {
+    const documentTargetStore = txn.store<
+      DbTargetDocumentKey,
+      DbTargetDocument
+    >(DbTargetDocument.store);
+    const documentsStore = txn.store<DbRemoteDocumentKey, DbRemoteDocument>(
+      DbRemoteDocument.store
+    );
+
+    return getHighestListenSequenceNumber(txn).next(currentSequenceNumber => {
+      const writeSentinelKey = (
+        path: ResourcePath
+      ): PersistencePromise<void> => {
+        return documentTargetStore.put(
+          new DbTargetDocument(0, encode(path), currentSequenceNumber)
+        );
+      };
+
+      const promises: Array<PersistencePromise<void>> = [];
+      return documentsStore
+        .iterate((key, doc) => {
+          const path = new ResourcePath(key);
+          const docSentinelKey = sentinelKey(path);
+          promises.push(
+            documentTargetStore.get(docSentinelKey).next(maybeSentinel => {
+              if (!maybeSentinel) {
+                return writeSentinelKey(path);
+              } else {
+                return PersistencePromise.resolve();
+              }
+            })
+          );
+        })
+        .next(() => PersistencePromise.waitFor(promises));
+    });
+  }
+}
+
+function sentinelKey(path: ResourcePath): DbTargetDocumentKey {
+  return [0, encode(path)];
 }
 
 // TODO(mikelehen): Get rid of "as any" if/when TypeScript fixes their types.

packages/firestore/src/platform_node/grpc_connection.ts

@@ -113,7 +113,17 @@ export class GrpcConnection implements Connection {
         ? grpc.credentials.createSsl()
         : grpc.credentials.createInsecure();
       this.cachedStub = {
-        stub: new this.firestore.Firestore(this.databaseInfo.host, credentials),
+        stub: new this.firestore.Firestore(
+          this.databaseInfo.host,
+          credentials,
+          {
+            // We do our own connection backoff (that for example is aware of whether or
+            // not a write stream error is permanent or not) so we don't want gRPC to do
+            // backoff on top of that. 100ms is the minimum value that gRPC allows.
+            'grpc.initial_reconnect_backoff_ms': 100,
+            'grpc.max_reconnect_backoff_ms': 100
+          }
+        ),
         token
       };
     }

packages/firestore/test/unit/local/indexeddb_persistence.test.ts

@@ -17,6 +17,7 @@
 import { expect } from 'chai';
 import { PersistenceSettings } from '../../../src/api/database';
 import { SnapshotVersion } from '../../../src/core/snapshot_version';
+import { decode, encode } from '../../../src/local/encoded_resource_path';
 import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence';
 import {
   DbDocumentMutation,
@@ -32,6 +33,8 @@ import {
   DbRemoteDocumentGlobalKey,
   DbRemoteDocumentKey,
   DbTarget,
+  DbTargetDocument,
+  DbTargetDocumentKey,
   DbTargetGlobal,
   DbTargetGlobalKey,
   DbTargetKey,
@@ -521,6 +524,92 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => {
       });
     });
   });
+
+  it('can upgrade from version 6 to 7', async () => {
+    const oldSequenceNumber = 1;
+    // Set the highest sequence number to this value so that untagged documents
+    // will pick up this value.
+    const newSequenceNumber = 2;
+    await withDb(6, db => {
+      const serializer = TEST_SERIALIZER;
+
+      const sdb = new SimpleDb(db);
+      return sdb.runTransaction('readwrite', V6_STORES, txn => {
+        const targetGlobalStore = txn.store<DbTargetGlobalKey, DbTargetGlobal>(
+          DbTargetGlobal.store
+        );
+        const remoteDocumentStore = txn.store<
+          DbRemoteDocumentKey,
+          DbRemoteDocument
+        >(DbRemoteDocument.store);
+        const targetDocumentStore = txn.store<
+          DbTargetDocumentKey,
+          DbTargetDocument
+        >(DbTargetDocument.store);
+        return targetGlobalStore
+          .get(DbTargetGlobal.key)
+          .next(metadata => {
+            expect(metadata).to.not.be.null;
+            metadata!.highestListenSequenceNumber = newSequenceNumber;
+            return targetGlobalStore.put(DbTargetGlobal.key, metadata!);
+          })
+          .next(() => {
+            // Set up some documents (we only need the keys)
+            // For the odd ones, add sentinel rows.
+            const promises: Array<PersistencePromise<void>> = [];
+            for (let i = 0; i < 10; i++) {
+              const document = doc('docs/doc_' + i, 1, { foo: 'bar' });
+              promises.push(
+                remoteDocumentStore.put(
+                  document.key.path.toArray(),
+                  serializer.toDbRemoteDocument(document)
+                )
+              );
+              if (i % 2 === 1) {
+                promises.push(
+                  targetDocumentStore.put(
+                    new DbTargetDocument(
+                      0,
+                      encode(document.key.path),
+                      oldSequenceNumber
+                    )
+                  )
+                );
+              }
+            }
+            return PersistencePromise.waitFor(promises);
+          });
+      });
+    });
+
+    // Now run the migration and verify
+    await withDb(7, db => {
+      const sdb = new SimpleDb(db);
+      return sdb.runTransaction('readonly', V6_STORES, txn => {
+        const targetDocumentStore = txn.store<
+          DbTargetDocumentKey,
+          DbTargetDocument
+        >(DbTargetDocument.store);
+        const range = IDBKeyRange.bound(
+          [0],
+          [1],
+          /*lowerOpen=*/ false,
+          /*upperOpen=*/ true
+        );
+        return targetDocumentStore.iterate(
+          { range },
+          ([_, path], targetDocument) => {
+            const decoded = decode(path);
+            const lastSegment = decoded.lastSegment();
+            const docNum = +lastSegment.split('_')[1];
+            const expected =
+              docNum % 2 === 1 ? oldSequenceNumber : newSequenceNumber;
+            expect(targetDocument.sequenceNumber).to.equal(expected);
+          }
+        );
+      });
+    });
+  });
 });
 
 describe('IndexedDb: canActAsPrimary', () => {

packages/functions/package.json

@@ -39,7 +39,7 @@
     "karma-sourcemap-loader": "0.3.7",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",

packages/logger/package.json

@@ -31,7 +31,7 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",

packages/messaging/package.json

@@ -43,7 +43,7 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-sourcemap-loader": "0.3.7",
     "karma-spec-reporter": "0.0.32",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "rollup": "0.57.1",
     "rollup-plugin-commonjs": "9.1.0",

packages/rxfire/docs/database.md

@@ -80,8 +80,8 @@ list(ref).subscribe(changes => {
 list(ref)
   .pipe(
     map(changes => changes.map(c => { 
-      return { _key: c.snapshot.key, event: c.event, ...c.snapshot.val(); };
-    ))
+      return { _key: c.snapshot.key, event: c.event, ...c.snapshot.val() }
+    })
   )
   .subscribe(users => { console.log(users); })
 

packages/rxfire/package.json

@@ -57,7 +57,7 @@
     "karma-sourcemap-loader": "0.3.7",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "sinon": "4.5.0",

packages/template/package.json

@@ -40,7 +40,7 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",

packages/testing/index.ts

@@ -24,8 +24,10 @@ export {
   apps,
   assertFails,
   assertSucceeds,
+  database,
+  firestore,
   initializeAdminApp,
   initializeTestApp,
   loadDatabaseRules,
-  loadFirestoreRules
+  loadFirestoreRules,
 } from './src/api';

packages/testing/package.json

@@ -25,7 +25,7 @@
   "devDependencies": {
     "chai": "4.1.2",
     "chai-as-promised": "7.1.1",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "nyc": "13.1.0",
     "rollup": "0.57.1",
     "rollup-plugin-commonjs": "9.1.0",

packages/util/package.json

@@ -34,7 +34,7 @@
     "karma-sauce-launcher": "1.2.0",
     "karma-spec-reporter": "0.0.32",
     "karma-webpack": "2.0.9",
-    "mocha": "5.0.5",
+    "mocha": "5.2.0",
     "npm-run-all": "4.1.2",
     "nyc": "11.6.0",
     "rollup": "0.57.1",