Move bundle integration tests under api/ and complete exp API (#3962)

* Rolls a node app building bundles.

* Build bundle files for given list of project IDs.

* Build the bundle json map and save it for integration tests.

* Add emulator_settings.ts to gulp

* Move bundle.test.ts to api/

* Bundles passes all tests and expose in classic API

* Add CI project ID to bundles.

* Adhoc string replacement and length re-calculation

* Fix lint errors.

* Delete old changes from make node app

* Address comments

* Manually prepares the bundle strings.

* More fixes.

Author: wu-hui

packages/firebase/index.d.ts

@@ -8255,12 +8255,45 @@ declare namespace firebase.firestore {
      */
     terminate(): Promise<void>;
 
+    loadBundle(
+      bundleData: ArrayBuffer | ReadableStream<ArrayBuffer> | string
+    ): LoadBundleTask;
+
+    namedQuery(name: string): Promise<Query<DocumentData> | null>;
+
     /**
      * @hidden
      */
     INTERNAL: { delete: () => Promise<void> };
   }
 
+  export interface LoadBundleTask {
+    onProgress(
+      next?: (progress: LoadBundleTaskProgress) => any,
+      error?: (error: Error) => any,
+      complete?: () => void
+    ): void;
+
+    then<T, R>(
+      onFulfilled?: (a: LoadBundleTaskProgress) => T | PromiseLike<T>,
+      onRejected?: (a: Error) => R | PromiseLike<R>
+    ): Promise<T | R>;
+
+    catch<R>(
+      onRejected: (a: Error) => R | PromiseLike<R>
+    ): Promise<R | LoadBundleTaskProgress>;
+  }
+
+  export interface LoadBundleTaskProgress {
+    documentsLoaded: number;
+    totalDocuments: number;
+    bytesLoaded: number;
+    totalBytes: number;
+    taskState: TaskState;
+  }
+
+  export type TaskState = 'Error' | 'Running' | 'Success';
+
   /**
    * An immutable object representing a geo point in Firestore. The geo point
    * is represented as latitude/longitude pair.

packages/firestore-types/index.d.ts

@@ -96,6 +96,12 @@ export class FirebaseFirestore {
 
   terminate(): Promise<void>;
 
+  loadBundle(
+    bundleData: ArrayBuffer | ReadableStream<ArrayBuffer> | string
+  ): LoadBundleTask;
+
+  namedQuery(name: string): Promise<Query<DocumentData> | null>;
+
   INTERNAL: { delete: () => Promise<void> };
 }
 

packages/firestore/exp/src/api/database.ts

@@ -42,7 +42,7 @@ import {
 import { Code, FirestoreError } from '../../../src/util/error';
 import { Deferred } from '../../../src/util/promise';
 import { LruParams } from '../../../src/local/lru_garbage_collector';
-import { CACHE_SIZE_UNLIMITED, Query } from '../../../src/api/database';
+import { CACHE_SIZE_UNLIMITED } from '../../../src/api/database';
 import {
   indexedDbClearPersistence,
   indexedDbStoragePrefix
@@ -74,6 +74,7 @@ import {
 import { PersistenceSettings } from '../../../exp-types';
 import { getNamedQuery } from '../../../src/local/local_store';
 import { newSerializer } from '../../../src/platform/serializer';
+import { Query } from '../../../lite/src/api/reference';
 
 const LOG_TAG = 'Firestore';
 
@@ -578,14 +579,15 @@ export function namedQuery(
   firestore: FirebaseFirestore,
   name: string
 ): Promise<Query | null> {
-  return getLocalStore(firestore).then(localStore => {
-    return getNamedQuery(localStore, name).then(namedQuery => {
-      if (!namedQuery) {
-        return null;
-      }
+  return firestore._queue.enqueue(() => {
+    return getLocalStore(firestore).then(localStore => {
+      return getNamedQuery(localStore, name).then(namedQuery => {
+        if (!namedQuery) {
+          return null;
+        }
 
-      return null;
-      // return new Query(namedQuery.query, firestore, null);
+        return new Query(firestore, null, namedQuery.query);
+      });
     });
   });
 }

packages/firestore/exp/test/shim.ts

@@ -42,6 +42,7 @@ import {
   getDocsFromServer,
   initializeFirestore,
   loadBundle,
+  namedQuery,
   onSnapshot,
   onSnapshotsInSync,
   query,
@@ -175,8 +176,8 @@ export class FirebaseFirestore
   }
 
   async namedQuery(name: string): Promise<Query | null> {
-    return null;
-    // return namedQuery(this._delegate, name);
+    const query = await namedQuery(this._delegate, name);
+    return !!query ? new Query<legacy.DocumentData>(this, query) : null;
   }
 
   INTERNAL = {

packages/firestore/src/api/database.ts

@@ -111,10 +111,7 @@ import {
   valueDescription,
   validateIsNotUsedTogether
 } from '../util/input_validation';
-import {
-  setLogLevel as setClientLogLevel,
-  logWarn
-} from '../util/log';
+import { setLogLevel as setClientLogLevel, logWarn } from '../util/log';
 import { AutoId } from '../util/misc';
 import { Deferred } from '../util/promise';
 import { FieldPath as ExternalFieldPath } from './field_path';
@@ -627,7 +624,7 @@ export class Firestore implements PublicFirestore, FirebaseService {
     }
   }
 
-  _loadBundle(
+  loadBundle(
     bundleData: ArrayBuffer | ReadableStream<Uint8Array> | string
   ): LoadBundleTask {
     this.ensureClientConfigured();
@@ -636,7 +633,7 @@ export class Firestore implements PublicFirestore, FirebaseService {
     return resultTask;
   }
 
-  _namedQuery(name: string): Promise<PublicQuery | null> {
+  namedQuery(name: string): Promise<PublicQuery | null> {
     this.ensureClientConfigured();
     return this._firestoreClient!.getNamedQuery(name).then(namedQuery => {
       if (!namedQuery) {

packages/firestore/src/core/firestore_client.ts

@@ -38,7 +38,7 @@ import {
 } from '../remote/remote_store';
 import { AsyncQueue, wrapInUserErrorIfRecoverable } from '../util/async_queue';
 import { Code, FirestoreError } from '../util/error';
-import { logDebug, logWarn } from '../util/log';
+import { logDebug } from '../util/log';
 import { Deferred } from '../util/promise';
 import {
   addSnapshotsInSyncListener,
@@ -535,15 +535,14 @@ export class FirestoreClient {
     );
     this.asyncQueue.enqueueAndForget(async () => {
       loadBundle(this.syncEngine, reader, resultTask);
-      return resultTask.catch(e => {
-        logWarn(LOG_TAG, `Loading bundle failed with ${e}`);
-      });
     });
   }
 
   getNamedQuery(queryName: string): Promise<NamedQuery | undefined> {
     this.verifyNotTerminated();
-    return getNamedQuery(this.localStore, queryName);
+    return this.asyncQueue.enqueue(() =>
+      getNamedQuery(this.localStore, queryName)
+    );
   }
 }
 

packages/firestore/src/core/sync_engine.ts

@@ -60,7 +60,7 @@ import {
 } from '../remote/remote_store';
 import { debugAssert, debugCast, fail, hardAssert } from '../util/assert';
 import { Code, FirestoreError } from '../util/error';
-import { logDebug } from '../util/log';
+import { logDebug, logWarn } from '../util/log';
 import { primitiveComparator } from '../util/misc';
 import { ObjectMap } from '../util/obj_map';
 import { Deferred } from '../util/promise';
@@ -1602,6 +1602,7 @@ async function loadBundleImpl(
     await saveBundle(syncEngine.localStore, metadata);
     task._completeWith(result.progress);
   } catch (e) {
+    logWarn(LOG_TAG, `Loading bundle failed with ${e}`);
     task._failWith(e);
   }
 }

packages/firestore/src/remote/serializer.ts

@@ -339,21 +339,26 @@ export function fromName(
   name: string
 ): DocumentKey {
   const resource = fromResourceName(name);
-  hardAssert(
-    resource.get(1) === serializer.databaseId.projectId,
-    'Tried to deserialize key from different project: ' +
-      resource.get(1) +
-      ' vs ' +
-      serializer.databaseId.projectId
-  );
-  hardAssert(
-    (!resource.get(3) && !serializer.databaseId.database) ||
-      resource.get(3) === serializer.databaseId.database,
-    'Tried to deserialize key from different database: ' +
-      resource.get(3) +
-      ' vs ' +
-      serializer.databaseId.database
-  );
+
+  if (resource.get(1) !== serializer.databaseId.projectId) {
+    throw new FirestoreError(
+      Code.INVALID_ARGUMENT,
+      'Tried to deserialize key from different project: ' +
+        resource.get(1) +
+        ' vs ' +
+        serializer.databaseId.projectId
+    );
+  }
+
+  if (resource.get(3) !== serializer.databaseId.database) {
+    throw new FirestoreError(
+      Code.INVALID_ARGUMENT,
+      'Tried to deserialize key from different database: ' +
+        resource.get(3) +
+        ' vs ' +
+        serializer.databaseId.database
+    );
+  }
   return new DocumentKey(extractLocalPathFromResourceName(resource));
 }