Use approximate FieldValue size for MemoryLRU

Author: schmidt-sebastian

packages/firestore/src/local/memory_persistence.ts

@@ -16,15 +16,13 @@
  */
 
 import { User } from '../auth/user';
-import { MaybeDocument } from '../model/document';
+import { Document, MaybeDocument } from '../model/document';
 import { DocumentKey } from '../model/document_key';
-import { JsonProtoSerializer } from '../remote/serializer';
 import { fail } from '../util/assert';
 import { debug } from '../util/log';
 import * as obj from '../util/obj';
 import { ObjectMap } from '../util/obj_map';
 import { encode } from './encoded_resource_path';
-import { LocalSerializer } from './local_serializer';
 import {
   ActiveTargets,
   LruDelegate,
@@ -77,11 +75,10 @@ export class MemoryPersistence implements Persistence {
 
   static createLruPersistence(
     clientId: ClientId,
-    serializer: JsonProtoSerializer,
     params: LruParams
   ): MemoryPersistence {
     const factory = (p: MemoryPersistence): MemoryLruDelegate =>
-      new MemoryLruDelegate(p, new LocalSerializer(serializer), params);
+      new MemoryLruDelegate(p, params);
     return new MemoryPersistence(clientId, factory);
   }
 
@@ -334,7 +331,6 @@ export class MemoryLruDelegate implements ReferenceDelegate, LruDelegate {
 
   constructor(
     private readonly persistence: MemoryPersistence,
-    private readonly serializer: LocalSerializer,
     lruParams: LruParams
   ) {
     this.garbageCollector = new LruGarbageCollector(this, lruParams);
@@ -471,21 +467,11 @@ export class MemoryLruDelegate implements ReferenceDelegate, LruDelegate {
   }
 
   documentSize(maybeDoc: MaybeDocument): number {
-    const remoteDocument = this.serializer.toDbRemoteDocument(
-      maybeDoc,
-      maybeDoc.version
-    );
-    let value: unknown;
-    if (remoteDocument.document) {
-      value = remoteDocument.document;
-    } else if (remoteDocument.unknownDocument) {
-      value = remoteDocument.unknownDocument;
-    } else if (remoteDocument.noDocument) {
-      value = remoteDocument.noDocument;
-    } else {
-      throw fail('Unknown remote document type');
+    let documentSize = maybeDoc.key.toString().length;
+    if (maybeDoc instanceof Document) {
+      documentSize += maybeDoc.data().byteSize();
     }
-    return JSON.stringify(value).length;
+    return documentSize;
   }
 
   private isPinned(

packages/firestore/src/model/field_value.ts

@@ -126,6 +126,12 @@ export abstract class FieldValue {
   abstract isEqual(other: FieldValue): boolean;
   abstract compareTo(other: FieldValue): number;
 
+  /** 
+   * Returns the approximate (and wildly inaccurate) in-memory size of the field 
+   * value. 
+   */
+  abstract byteSize(): number;
+
   toString(): string {
     const val = this.value();
     return val === null ? 'null' : val.toString();
@@ -167,6 +173,10 @@ export class NullValue extends FieldValue {
     return this.defaultCompareTo(other);
   }
 
+  byteSize(): number {
+    return 1;
+  }
+
   static INSTANCE = new NullValue();
 }
 
@@ -195,6 +205,10 @@ export class BooleanValue extends FieldValue {
     return this.defaultCompareTo(other);
   }
 
+  byteSize(): number {
+    return 1;
+  }
+
   static of(value: boolean): BooleanValue {
     return value ? BooleanValue.TRUE : BooleanValue.FALSE;
   }
@@ -221,6 +235,10 @@ export abstract class NumberValue extends FieldValue {
     }
     return this.defaultCompareTo(other);
   }
+
+  byteSize(): number {
+    return 4;
+  }
 }
 
 /** Utility function to compare doubles (using Firestore semantics for NaN). */
@@ -313,6 +331,13 @@ export class StringValue extends FieldValue {
     }
     return this.defaultCompareTo(other);
   }
+
+  byteSize(): number {
+    // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures:
+    // "JavaScript's String type is [...] a set of elements of 16-bit unsigned
+    // integer values"
+    return this.internalValue.length * 2;
+  }
 }
 
 export class TimestampValue extends FieldValue {
@@ -347,6 +372,11 @@ export class TimestampValue extends FieldValue {
       return this.defaultCompareTo(other);
     }
   }
+
+  byteSize(): number {
+    // Timestamps are made up of two distinct numbers (seconds/nanoseconds)
+    return 8;
+  }
 }
 
 /**
@@ -410,6 +440,13 @@ export class ServerTimestampValue extends FieldValue {
   toString(): string {
     return '<ServerTimestamp localTime=' + this.localWriteTime.toString() + '>';
   }
+
+  byteSize(): number {
+    return (
+      /* localWriteTime */ 8 +
+      (this.previousValue ? this.previousValue.byteSize() : 0)
+    );
+  }
 }
 
 export class BlobValue extends FieldValue {
@@ -436,6 +473,10 @@ export class BlobValue extends FieldValue {
     }
     return this.defaultCompareTo(other);
   }
+
+  byteSize(): number {
+    return this.internalValue.toUint8Array().byteLength;
+  }
 }
 
 export class RefValue extends FieldValue {
@@ -466,6 +507,14 @@ export class RefValue extends FieldValue {
     }
     return this.defaultCompareTo(other);
   }
+
+  byteSize(): number {
+    return (
+      this.databaseId.projectId.length +
+      this.databaseId.database.length +
+      this.key.toString().length
+    );
+  }
 }
 
 export class GeoPointValue extends FieldValue {
@@ -492,6 +541,11 @@ export class GeoPointValue extends FieldValue {
     }
     return this.defaultCompareTo(other);
   }
+
+  byteSize(): number {
+    // GeoPoints are made up of two distinct numbers (latitude/longitude)
+    return 8;
+  }
 }
 
 export class ObjectValue extends FieldValue {
@@ -634,6 +688,14 @@ export class ObjectValue extends FieldValue {
     return FieldMask.fromSet(fields);
   }
 
+  byteSize(): number {
+    let size = 0;
+    this.internalValue.inorderTraversal((key, val) => {
+      size += key.length + val.byteSize();
+    });
+    return size;
+  }
+
   toString(): string {
     return this.internalValue.toString();
   }
@@ -720,6 +782,13 @@ export class ArrayValue extends FieldValue {
     }
   }
 
+  byteSize(): number {
+    return this.internalValue.reduce(
+      (previousSize, value) => previousSize + value.byteSize(),
+      0
+    );
+  }
+
   toString(): string {
     const descriptions = this.internalValue.map(v => v.toString());
     return `[${descriptions.join(',')}]`;

packages/firestore/test/integration/api/database.test.ts

@@ -814,7 +814,7 @@ apiDescribe('Database', (persistence: boolean) => {
   // eslint-disable-next-line no-restricted-properties
   describe.skip('Listens are rejected remotely:', () => {
     //eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const queryForRejection : firestore.Query = null as any;
+    const queryForRejection: firestore.Query = null as any;
 
     it('will reject listens', () => {
       const deferred = new Deferred();

packages/firestore/test/unit/local/persistence_test_helpers.ts

@@ -128,11 +128,7 @@ export async function testMemoryEagerPersistence(): Promise<MemoryPersistence> {
 export async function testMemoryLruPersistence(
   params: LruParams = LruParams.DEFAULT
 ): Promise<MemoryPersistence> {
-  return MemoryPersistence.createLruPersistence(
-    AutoId.newId(),
-    JSON_SERIALIZER,
-    params
-  );
+  return MemoryPersistence.createLruPersistence(AutoId.newId(), params);
 }
 
 /** Clears the persistence in tests */

packages/firestore/test/unit/model/field_value.test.ts

@@ -19,6 +19,7 @@ import { expect } from 'chai';
 import { GeoPoint } from '../../../src/api/geo_point';
 import { Timestamp } from '../../../src/api/timestamp';
 import * as fieldValue from '../../../src/model/field_value';
+import { primitiveComparator } from '../../../src/util/misc';
 import * as typeUtils from '../../../src/util/types';
 import {
   blob,
@@ -485,4 +486,57 @@ describe('FieldValue', () => {
       }
     );
   });
+
+  it('estimates size correctly for fixed sized values', () => {
+    // This test verifies that each member of a group takes up the same amount
+    // of space in memory (based on its estimated in-memory size).
+    const equalityGroups = [
+      [wrap(null), wrap(false), wrap(true)],
+      [wrap(blob(0, 1)), wrap(blob(128, 129))],
+      [wrap(NaN), wrap(Infinity), wrap(1), wrap(1.1)],
+      [wrap(new GeoPoint(0, 0)), wrap(new GeoPoint(0, 0))],
+      [wrap(Timestamp.fromMillis(100)), wrap(Timestamp.now())],
+      [
+        new fieldValue.ServerTimestampValue(Timestamp.fromMillis(100), null),
+        new fieldValue.ServerTimestampValue(Timestamp.now(), null)
+      ]
+    ];
+
+    for (const group of equalityGroups) {
+      const expectedItemSize = group[0].byteSize();
+      for (const element of group) {
+        expect(element.byteSize()).to.equal(expectedItemSize);
+      }
+    }
+  });
+
+  it('estimates size correctly for relatively sized values', () => {
+    // This test verifies for each group that the estimated size increases
+    // as the size of the underlying data grows.
+    const relativeGroups = [
+      [wrap(blob(0)), wrap(blob(0, 1))],
+      [
+        new fieldValue.ServerTimestampValue(Timestamp.fromMillis(100), null),
+        new fieldValue.ServerTimestampValue(Timestamp.now(), wrap(null))
+      ],
+      [
+        new fieldValue.RefValue(dbId('p1', 'd1'), key('c1/doc1')),
+        new fieldValue.RefValue(dbId('p1', 'd1'), key('c1/doc1/c2/doc2'))
+      ],
+      [wrap('foo'), wrap('foobar')],
+      [wrap(['a', 'b']), wrap(['a', 'bc'])],
+      [wrap(['a', 'b']), wrap(['a', 'b', 'c'])],
+      [wrap({ a: 'a', b: 'b' }), wrap({ a: 'a', b: 'bc' })],
+      [wrap({ a: 'a', b: 'b' }), wrap({ a: 'a', bc: 'b' })],
+      [wrap({ a: 'a', b: 'b' }), wrap({ a: 'a', b: 'b', c: 'c' })]
+    ];
+
+    for (const group of relativeGroups) {
+      const expectedOrder = group;
+      const actualOrder = group
+        .slice()
+        .sort((l, r) => primitiveComparator(l.byteSize(), r.byteSize()));
+      expect(expectedOrder).to.deep.equal(actualOrder);
+    }
+  });
 });

packages/firestore/test/unit/specs/spec_test_runner.ts

@@ -1238,7 +1238,6 @@ class MemoryTestRunner extends TestRunner {
         ? MemoryPersistence.createEagerPersistence(this.clientId)
         : MemoryPersistence.createLruPersistence(
             this.clientId,
-            serializer,
             LruParams.DEFAULT
           )
     );