Skip to content

Commit b524007

Browse files
schmidt-sebastianschmidt-sebastian
authored
Merging "benchmark spec tests (#1048)" (#1057)
1 parent 35753cd commit b524007

File tree

2 files changed

+267
-12
lines changed

2 files changed

+267
-12
lines changed

‎packages/firestore/test/unit/specs/describe_spec.ts

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,20 @@ const MULTI_CLIENT_TAG = 'multi-client';
3030
const NO_WEB_TAG = 'no-web';
3131
const NO_ANDROID_TAG = 'no-android';
3232
const NO_IOS_TAG = 'no-ios';
33-
const NO_LRU = 'no-lru';
33+
const NO_LRU_TAG = 'no-lru';
34+
const BENCHMARK_TAG = 'benchmark';
3435
const KNOWN_TAGS = [
36+
BENCHMARK_TAG,
3537
EXCLUSIVE_TAG,
3638
MULTI_CLIENT_TAG,
3739
NO_WEB_TAG,
3840
NO_ANDROID_TAG,
3941
NO_IOS_TAG,
40-
NO_LRU
42+
NO_LRU_TAG
4143
];
4244

43-
const WEB_SPEC_TEST_FILTER = (tags: string[], persistenceEnabled: boolean) => {
44-
return (
45-
tags.indexOf(NO_WEB_TAG) === -1 &&
46-
(tags.indexOf(MULTI_CLIENT_TAG) === -1 || persistenceEnabled)
47-
);
48-
};
45+
// TOOD(mrschmidt): Make this configurable with mocha options.
46+
const RUN_BENCHMARK_TESTS = false;
4947

5048
// The format of one describeSpec written to a JSON file.
5149
interface SpecOutputFormat {
@@ -78,11 +76,16 @@ export function setSpecJSONHandler(writer: (json: string) => void): void {
7876

7977
/** Gets the test runner based on the specified tags. */
8078
function getTestRunner(tags, persistenceEnabled): Function {
81-
if (!WEB_SPEC_TEST_FILTER(tags, persistenceEnabled)) {
79+
if (tags.indexOf(NO_WEB_TAG) >= 0) {
8280
return it.skip;
83-
} else if (persistenceEnabled && tags.indexOf('no-lru') !== -1) {
81+
} else if (persistenceEnabled && tags.indexOf(NO_LRU_TAG) !== -1) {
8482
// spec should have a comment explaining why it is being skipped.
8583
return it.skip;
84+
} else if (!persistenceEnabled && tags.indexOf(MULTI_CLIENT_TAG) !== -1) {
85+
// spec should have a comment explaining why it is being skipped.
86+
return it.skip;
87+
} else if (tags.indexOf(BENCHMARK_TAG) >= 0 && !RUN_BENCHMARK_TESTS) {
88+
return it.skip;
8689
} else if (tags.indexOf(EXCLUSIVE_TAG) >= 0) {
8790
return it.only;
8891
} else {
@@ -143,8 +146,14 @@ export function specTest(
143146
const runner = getTestRunner(tags, usePersistence);
144147
const mode = usePersistence ? '(Persistence)' : '(Memory)';
145148
const fullName = `${mode} ${name}`;
146-
runner(fullName, () => {
147-
return spec.runAsTest(fullName, usePersistence);
149+
runner(fullName, async () => {
150+
const start = Date.now();
151+
await spec.runAsTest(fullName, usePersistence);
152+
const end = Date.now();
153+
if (tags.indexOf(BENCHMARK_TAG) >= 0) {
154+
// tslint:disable-next-line:no-console
155+
console.log(`Runtime: ${end - start} ms.`);
156+
}
148157
});
149158
}
150159
} else {

‎packages/firestore/test/unit/specs/perf_spec.test.ts

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/**
2+
* Copyright 2018 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import { Query } from '../../../src/core/query';
18+
import { doc, orderBy, path } from '../../util/helpers';
19+
20+
import { describeSpec, specTest } from './describe_spec';
21+
import { spec } from './spec_builder';
22+
23+
/** The number of iterations for the benchmark spec tests. */
24+
const STEP_COUNT = 10;
25+
26+
describeSpec(
27+
`Performance Tests [${STEP_COUNT} iterations]:`,
28+
['benchmark'],
29+
() => {
30+
specTest('Insert a new document', [], () => {
31+
let steps = spec().withGCEnabled(false);
32+
for (let i = 0; i < STEP_COUNT; ++i) {
33+
steps = steps
34+
.userSets(`collection/{i}`, { doc: i })
35+
.writeAcks(`collection/{i}`, i);
36+
}
37+
return steps;
38+
});
39+
40+
specTest(
41+
'Start a listen, write a document, ack the write, handle watch snapshot, unlisten',
42+
[],
43+
() => {
44+
let currentVersion = 1;
45+
let steps = spec().withGCEnabled(false);
46+
47+
for (let i = 0; i < STEP_COUNT; ++i) {
48+
const query = Query.atPath(path(`collection/${i}`));
49+
const docLocal = doc(
50+
`collection/${i}`,
51+
0,
52+
{ doc: i },
53+
{ hasLocalMutations: true }
54+
);
55+
const docRemote = doc(`collection/${i}`, ++currentVersion, {
56+
doc: i
57+
});
58+
59+
steps = steps
60+
.userListens(query)
61+
.userSets(`collection/${i}`, { doc: i })
62+
.expectEvents(query, {
63+
added: [docLocal],
64+
fromCache: true,
65+
hasPendingWrites: true
66+
})
67+
.writeAcks(`collection/${i}`, ++currentVersion)
68+
.watchAcksFull(query, ++currentVersion, docRemote)
69+
.expectEvents(query, { metadata: [docRemote] })
70+
.userUnlistens(query)
71+
.watchRemoves(query);
72+
}
73+
return steps;
74+
}
75+
);
76+
77+
specTest('Write 100 documents and raise a snapshot', [], () => {
78+
const cachedDocumentCount = 100;
79+
80+
const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
81+
82+
let steps = spec().withGCEnabled(false);
83+
84+
const docs = [];
85+
86+
for (let i = 0; i < cachedDocumentCount; ++i) {
87+
steps.userSets(`collection/${i}`, { v: i });
88+
docs.push(
89+
doc(`collection/${i}`, 0, { v: i }, { hasLocalMutations: true })
90+
);
91+
}
92+
93+
for (let i = 1; i <= STEP_COUNT; ++i) {
94+
steps = steps
95+
.userListens(query)
96+
.expectEvents(query, {
97+
added: docs,
98+
fromCache: true,
99+
hasPendingWrites: true
100+
})
101+
.userUnlistens(query);
102+
}
103+
104+
return steps;
105+
});
106+
107+
specTest('Update a single document', [], () => {
108+
let steps = spec().withGCEnabled(false);
109+
steps = steps.userSets(`collection/doc`, { v: 0 });
110+
for (let i = 1; i <= STEP_COUNT; ++i) {
111+
steps = steps
112+
.userPatches(`collection/doc`, { v: i })
113+
.writeAcks(`collection/doc`, i);
114+
}
115+
return steps;
116+
});
117+
118+
specTest(
119+
'Update a document and wait for snapshot with existing listen',
120+
[],
121+
() => {
122+
const query = Query.atPath(path(`collection/doc`));
123+
124+
let currentVersion = 1;
125+
let steps = spec().withGCEnabled(false);
126+
127+
let docLocal = doc(
128+
`collection/doc`,
129+
0,
130+
{ v: 0 },
131+
{ hasLocalMutations: true }
132+
);
133+
let docRemote = doc(`collection/doc`, ++currentVersion, { v: 0 });
134+
let lastRemoteVersion = currentVersion;
135+
136+
steps = steps
137+
.userListens(query)
138+
.userSets(`collection/doc`, { v: 0 })
139+
.expectEvents(query, {
140+
added: [docLocal],
141+
fromCache: true,
142+
hasPendingWrites: true
143+
})
144+
.writeAcks(`collection/doc`, ++currentVersion)
145+
.watchAcksFull(query, ++currentVersion, docRemote)
146+
.expectEvents(query, { metadata: [docRemote] });
147+
148+
for (let i = 1; i <= STEP_COUNT; ++i) {
149+
docLocal = doc(
150+
`collection/doc`,
151+
lastRemoteVersion,
152+
{ v: i },
153+
{ hasLocalMutations: true }
154+
);
155+
docRemote = doc(`collection/doc`, ++currentVersion, { v: i });
156+
lastRemoteVersion = currentVersion;
157+
158+
steps = steps
159+
.userPatches(`collection/doc`, { v: i })
160+
.expectEvents(query, {
161+
modified: [docLocal],
162+
hasPendingWrites: true
163+
})
164+
.writeAcks(`collection/doc`, ++currentVersion)
165+
.watchSends({ affects: [query] }, docRemote)
166+
.watchSnapshots(++currentVersion)
167+
.expectEvents(query, { metadata: [docRemote] });
168+
}
169+
return steps;
170+
}
171+
);
172+
173+
specTest(
174+
'Process 100 documents from Watch and wait for snapshot',
175+
[],
176+
() => {
177+
const documentsPerStep = 100;
178+
179+
const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
180+
181+
let currentVersion = 1;
182+
let steps = spec().withGCEnabled(false);
183+
184+
steps = steps
185+
.userListens(query)
186+
.watchAcksFull(query, currentVersion)
187+
.expectEvents(query, {});
188+
189+
for (let i = 1; i <= STEP_COUNT; ++i) {
190+
const docs = [];
191+
192+
for (let j = 0; j < documentsPerStep; ++j) {
193+
docs.push(
194+
doc(`collection/${j}`, ++currentVersion, { v: currentVersion })
195+
);
196+
}
197+
198+
const changeType = i === 1 ? 'added' : 'modified';
199+
200+
steps = steps
201+
.watchSends({ affects: [query] }, ...docs)
202+
.watchSnapshots(++currentVersion)
203+
.expectEvents(query, { [changeType]: docs });
204+
}
205+
206+
return steps;
207+
}
208+
);
209+
210+
specTest(
211+
'Process 100 documents from Watch and wait for snapshot, then unlisten and wait for a cached snapshot',
212+
[],
213+
() => {
214+
const documentsPerStep = 100;
215+
216+
let currentVersion = 1;
217+
let steps = spec().withGCEnabled(false);
218+
219+
for (let i = 1; i <= STEP_COUNT; ++i) {
220+
const collPath = `collection/${i}/coll`;
221+
const query = Query.atPath(path(collPath)).addOrderBy(orderBy('v'));
222+
223+
const docs = [];
224+
for (let j = 0; j < documentsPerStep; ++j) {
225+
docs.push(doc(`${collPath}/${j}`, ++currentVersion, { v: j }));
226+
}
227+
228+
steps = steps
229+
.userListens(query)
230+
.watchAcksFull(query, ++currentVersion, ...docs)
231+
.expectEvents(query, { added: docs })
232+
.userUnlistens(query)
233+
.watchRemoves(query)
234+
.userListens(query, 'resume-token-' + currentVersion)
235+
.expectEvents(query, { added: docs, fromCache: true })
236+
.watchAcksFull(query, ++currentVersion)
237+
.expectEvents(query, {})
238+
.userUnlistens(query)
239+
.watchRemoves(query);
240+
}
241+
242+
return steps;
243+
}
244+
);
245+
}
246+
);

0 commit comments

Comments
 (0)