Skip to content

Commit 8d2536e

Browse files
ehsannasehsannas
committed
wip.
1 parent bdd3eae commit 8d2536e

File tree

7 files changed

+73
-87
lines changed

7 files changed

+73
-87
lines changed

‎packages/firestore/src/local/indexeddb_document_overlay_cache.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ export class IndexedDbDocumentOverlayCache implements DocumentOverlayCache {
104104
largestBatchId: number,
105105
overlays: MutationMap
106106
): PersistencePromise<void> {
107+
console.log("in saveOverlays");
107108
const promises: Array<PersistencePromise<void>> = [];
108109
overlays.forEach((_, mutation) => {
109110
const overlay = new Overlay(largestBatchId, mutation);
@@ -212,6 +213,7 @@ export class IndexedDbDocumentOverlayCache implements DocumentOverlayCache {
212213
transaction: PersistenceTransaction,
213214
overlay: Overlay
214215
): PersistencePromise<void> {
216+
console.log("in saveOverlay");
215217
return documentOverlayStore(transaction).put(
216218
toDbDocumentOverlay(this.serializer, this.userId, overlay)
217219
);

‎packages/firestore/src/local/indexeddb_mutation_queue.ts

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,7 @@ export class IndexedDbMutationQueue implements MutationQueue {
356356
transaction: PersistenceTransaction,
357357
documentKeys: SortedMap<DocumentKey, unknown>
358358
): PersistencePromise<MutationBatch[]> {
359+
console.log("in getAllMutationBatchesAffectingDocumentKeys");
359360
let uniqueBatchIDs = new SortedSet<BatchId>(primitiveComparator);
360361

361362
const promises: Array<PersistencePromise<void>> = [];
@@ -371,6 +372,9 @@ export class IndexedDbMutationQueue implements MutationQueue {
371372
(indexKey, _, control) => {
372373
const [userID, encodedPath, batchID] = indexKey;
373374

375+
console.log("\t iterating documentMutationStore");
376+
console.log(`\t batchId=${batchID}`);
377+
374378
// Only consider rows matching exactly the specific key of
375379
// interest. Note that because we order by path first, and we
376380
// order terminators before path separators, we'll encounter all
@@ -391,8 +395,10 @@ export class IndexedDbMutationQueue implements MutationQueue {
391395
promises.push(promise);
392396
});
393397

394-
return PersistencePromise.waitFor(promises).next(() =>
395-
this.lookupMutationBatches(transaction, uniqueBatchIDs)
398+
return PersistencePromise.waitFor(promises).next(() => {
399+
console.log(`uniqueBatchIDs.size=${uniqueBatchIDs.size}`);
400+
return this.lookupMutationBatches(transaction, uniqueBatchIDs)
401+
}
396402
);
397403
}
398404

@@ -458,6 +464,8 @@ export class IndexedDbMutationQueue implements MutationQueue {
458464
transaction: PersistenceTransaction,
459465
batchIDs: SortedSet<BatchId>
460466
): PersistencePromise<MutationBatch[]> {
467+
console.log("in lookupMutationBatches");
468+
console.log(`\t batchIDs.size=${batchIDs.size}`);
461469
const results: MutationBatch[] = [];
462470
const promises: Array<PersistencePromise<void>> = [];
463471
// TODO(rockwood): Implement this using iterate.
@@ -477,6 +485,7 @@ export class IndexedDbMutationQueue implements MutationQueue {
477485
mutation.userId === this.userId,
478486
`Unexpected user '${mutation.userId}' for mutation batch ${batchId}`
479487
);
488+
console.log(`\t found mutation in mutationStore`);
480489
results.push(fromDbMutationBatch(this.serializer, mutation));
481490
})
482491
);

‎packages/firestore/src/local/indexeddb_persistence.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ export class IndexedDbPersistence implements Persistence {
217217
private readonly forceOwningTab: boolean,
218218
private readonly schemaVersion = SCHEMA_VERSION
219219
) {
220+
console.log(`SCHEMA_VERSION=${schemaVersion}`);
220221
if (!IndexedDbPersistence.isAvailable()) {
221222
throw new FirestoreError(
222223
Code.UNIMPLEMENTED,

‎packages/firestore/src/local/indexeddb_schema.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ export const INDEXING_SCHEMA_VERSION = 15;
5454
* 12. Add document overlays.
5555
* 13. Rewrite the keys of the remote document cache to allow for efficient
5656
* document lookup via `getAll()`.
57-
* 14. Add indexing support.
57+
* 14. Add overlays.
58+
* 15. Add indexing support.
5859
*/
5960

6061
export const SCHEMA_VERSION = INDEXING_ENABLED ? INDEXING_SCHEMA_VERSION : 14;

‎packages/firestore/src/local/indexeddb_schema_converter.ts

Lines changed: 42 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import { User } from '../auth/user';
1919
import { ListenSequence } from '../core/listen_sequence';
2020
import { SnapshotVersion } from '../core/snapshot_version';
21-
import { documentKeySet } from '../model/collections';
21+
import { DocumentKeySet, documentKeySet } from '../model/collections';
2222
import { DocumentKey } from '../model/document_key';
2323
import { ResourcePath } from '../model/path';
2424
import { debugAssert, fail, hardAssert } from '../util/assert';
@@ -143,6 +143,7 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
143143
fromVersion: number,
144144
toVersion: number
145145
): PersistencePromise<void> {
146+
console.log(`in createOrUpgrade from ${fromVersion} to ${toVersion}`);
146147
debugAssert(
147148
fromVersion < toVersion &&
148149
fromVersion >= 0 &&
@@ -249,6 +250,7 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
249250
}
250251

251252
if (fromVersion < 14 && toVersion >= 14) {
253+
console.log("about to runOverlayMigration");
252254
p = p.next(() => this.runOverlayMigration(db, simpleDbTransaction));
253255
}
254256

@@ -471,9 +473,7 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
471473
db: IDBDatabase,
472474
transaction: SimpleDbTransaction
473475
): PersistencePromise<void> {
474-
const queuesStore = transaction.store<DbMutationQueueKey, DbMutationQueue>(
475-
DbMutationQueueStore
476-
);
476+
console.log("in runOverlayMigration");
477477
const mutationsStore = transaction.store<
478478
DbMutationBatchKey,
479479
DbMutationBatch
@@ -482,78 +482,56 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
482482
const remoteDocumentCache = newIndexedDbRemoteDocumentCache(
483483
this.serializer
484484
);
485+
const memoryPersistence = new MemoryPersistence(
486+
MemoryEagerDelegate.factory,
487+
this.serializer.remoteSerializer
488+
);
485489

486490
const promises: Array<PersistencePromise<void>> = [];
487-
let userIds = new Set<string>();
491+
let userToDocumentSet = new Map<string, DocumentKeySet>();
488492

489-
return queuesStore
493+
return mutationsStore
490494
.loadAll()
491-
.next(queues => {
492-
for (const queue of queues) {
493-
const userId = queue.userId;
494-
if (userIds.has(userId)) {
495-
// We have already processed this user.
496-
continue;
497-
}
498-
userIds = userIds.add(userId);
495+
.next(dbBatches => {
496+
dbBatches.forEach(dbBatch => {
497+
let documentSet =
498+
userToDocumentSet.get(dbBatch.userId) ?? documentKeySet();
499+
const batch = fromDbMutationBatch(this.serializer, dbBatch);
500+
batch.keys().forEach(key => (documentSet = documentSet.add(key)));
501+
userToDocumentSet.set(dbBatch.userId, documentSet);
502+
});
503+
})
504+
.next(() => {
505+
userToDocumentSet.forEach((allDocumentKeysForUser, userId) => {
499506
const user = new User(userId);
500507
const documentOverlayCache = IndexedDbDocumentOverlayCache.forUser(
501508
this.serializer,
502509
user
503510
);
504-
let allDocumentKeysForUser = documentKeySet();
505-
const range = IDBKeyRange.bound(
506-
[userId, BATCHID_UNKNOWN],
507-
[userId, Number.POSITIVE_INFINITY]
511+
// NOTE: The index manager and the reference delegate are
512+
// irrelevant for the purpose of recalculating and saving
513+
// overlays. We can therefore simply use the memory
514+
// implementation.
515+
const indexManager = memoryPersistence.getIndexManager(user);
516+
const mutationQueue = IndexedDbMutationQueue.forUser(
517+
user,
518+
this.serializer,
519+
indexManager,
520+
memoryPersistence.referenceDelegate
521+
);
522+
const localDocumentsView = new LocalDocumentsView(
523+
remoteDocumentCache,
524+
mutationQueue,
525+
documentOverlayCache,
526+
indexManager
508527
);
509528
promises.push(
510-
mutationsStore
511-
.loadAll(DbMutationBatchUserMutationsIndex, range)
512-
.next(dbBatches => {
513-
dbBatches.forEach(dbBatch => {
514-
hardAssert(
515-
dbBatch.userId === userId,
516-
`Cannot process batch ${dbBatch.batchId} from unexpected user`
517-
);
518-
const batch = fromDbMutationBatch(this.serializer, dbBatch);
519-
batch
520-
.keys()
521-
.forEach(
522-
key =>
523-
(allDocumentKeysForUser =
524-
allDocumentKeysForUser.add(key))
525-
);
526-
});
527-
})
528-
.next(() => {
529-
// NOTE: The index manager and the reference delegate are
530-
// irrelevant for the purpose of recalculating and saving
531-
// overlays. We can therefore simply use the memory
532-
// implementation.
533-
const memoryPersistence = new MemoryPersistence(
534-
MemoryEagerDelegate.factory,
535-
this.serializer.remoteSerializer
536-
);
537-
const indexManager = memoryPersistence.getIndexManager(user);
538-
const mutationQueue = IndexedDbMutationQueue.forUser(
539-
user,
540-
this.serializer,
541-
indexManager,
542-
memoryPersistence.referenceDelegate
543-
);
544-
const localDocumentsView = new LocalDocumentsView(
545-
remoteDocumentCache,
546-
mutationQueue,
547-
documentOverlayCache,
548-
indexManager
549-
);
550-
return localDocumentsView.recalculateAndSaveOverlaysForDocumentKeys(
551-
new IndexedDbTransaction(transaction, ListenSequence.INVALID),
552-
allDocumentKeysForUser
553-
);
554-
})
529+
localDocumentsView.recalculateAndSaveOverlaysForDocumentKeys(
530+
new IndexedDbTransaction(transaction, ListenSequence.INVALID),
531+
allDocumentKeysForUser
532+
)
555533
);
556-
}
534+
});
557535
})
558536
.next(() => PersistencePromise.waitFor(promises));
559537
}

‎packages/firestore/src/local/local_documents_view.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ export class LocalDocumentsView {
215215
transaction: PersistenceTransaction,
216216
docs: MutableDocumentMap
217217
): PersistencePromise<void> {
218+
console.log("in recalculateAndSaveOverlays")
218219
const masks = newDocumentKeyMap<FieldMask | null>();
219220
// A reverse lookup map from batch id to the documents within that batch.
220221
let documentsByBatchId = new SortedMap<number, DocumentKeySet>(
@@ -224,9 +225,13 @@ export class LocalDocumentsView {
224225
return this.mutationQueue
225226
.getAllMutationBatchesAffectingDocumentKeys(transaction, docs)
226227
.next(batches => {
228+
console.log("back in recalculateAndSaveOverlays");
229+
console.log(`got ${batches.length} batches`);
227230
for (const batch of batches) {
228231
batch.keys().forEach(key => {
232+
console.log(`\t key=${key.toString()}`);
229233
const baseDoc = docs.get(key);
234+
console.log(`\t baseDoc=${baseDoc ? baseDoc.toString() : "null"}`);
230235
if (baseDoc === null) {
231236
return;
232237
}
@@ -244,21 +249,29 @@ export class LocalDocumentsView {
244249
}
245250
})
246251
.next(() => {
252+
console.log("documentsByBatchId=");
253+
console.log(documentsByBatchId);
254+
console.log("masks=");
255+
console.log(masks);
247256
const promises: Array<PersistencePromise<void>> = [];
248257
// Iterate in descending order of batch IDs, and skip documents that are
249258
// already saved.
250259
const iter = documentsByBatchId.getReverseIterator();
251260
while (iter.hasNext()) {
261+
console.log(`reverse iteration over documentsByBatchId`);
252262
const entry = iter.getNext();
253263
const batchId = entry.key;
254264
const keys = entry.value;
265+
console.log(`\t batchId=${batchId}`);
266+
console.log(`\t keys=${keys}`);
255267
const overlays = newMutationMap();
256268
keys.forEach(key => {
257269
if (!processed.has(key)) {
258270
const overlayMutation = calculateOverlayMutation(
259271
docs.get(key)!,
260272
masks.get(key)!
261273
);
274+
console.log(`calculateOverlayMigration resulted in ${overlayMutation ? overlayMutation?.type.toString() : "null"}`);
262275
if (overlayMutation !== null) {
263276
overlays.set(key, overlayMutation);
264277
}
@@ -285,6 +298,7 @@ export class LocalDocumentsView {
285298
transaction: PersistenceTransaction,
286299
documentKeys: DocumentKeySet
287300
): PersistencePromise<void> {
301+
console.log("in recalculateAndSaveOverlaysForDocumentKeys");
288302
return this.remoteDocumentCache
289303
.getEntries(transaction, documentKeys)
290304
.next(docs => this.recalculateAndSaveOverlays(transaction, docs));

‎packages/firestore/test/unit/local/indexeddb_persistence.test.ts

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,7 +1066,7 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => {
10661066
});
10671067
});
10681068

1069-
it('can upgrade from schema version 13 to 14 (overlay migration)', function (this: Context) {
1069+
it.only('can upgrade from schema version 13 to 14 (overlay migration)', function (this: Context) {
10701070
// This test creates a database with schema version 13 that has three users,
10711071
// two of whom have local mutations.
10721072
const testWriteFoo = {
@@ -1163,25 +1163,6 @@ describe('IndexedDbSchema: createOrUpgradeDb', () => {
11631163
);
11641164
});
11651165
}
1166-
).next(() =>
1167-
// Populate the mutation queues' metadata
1168-
PersistencePromise.waitFor([
1169-
mutationQueuesStore.put({
1170-
userId: 'user1',
1171-
lastAcknowledgedBatchId: -1,
1172-
lastStreamToken: ''
1173-
}),
1174-
mutationQueuesStore.put({
1175-
userId: 'user2',
1176-
lastAcknowledgedBatchId: -1,
1177-
lastStreamToken: ''
1178-
}),
1179-
mutationQueuesStore.put({
1180-
userId: 'user3',
1181-
lastAcknowledgedBatchId: -1,
1182-
lastStreamToken: ''
1183-
})
1184-
])
11851166
);
11861167
}
11871168
);

0 commit comments

Comments
 (0)