Skip to content

Commit 718953e

Browse files
schmidt-sebastianschmidt-sebastian
authored
Add benchmark spec tests (#1048)
1 parent fc2e967 commit 718953e

File tree

2 files changed

+257
-5
lines changed

2 files changed

+257
-5
lines changed

‎packages/firestore/test/unit/specs/describe_spec.ts

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ const NO_WEB_TAG = 'no-web';
3131
const NO_ANDROID_TAG = 'no-android';
3232
const NO_IOS_TAG = 'no-ios';
3333
const NO_LRU = 'no-lru';
34+
const BENCHMARK_TAG = 'benchmark';
3435
const KNOWN_TAGS = [
36+
BENCHMARK_TAG,
3537
EXCLUSIVE_TAG,
3638
PERSISTENCE_TAG,
3739
NO_WEB_TAG,
@@ -40,8 +42,8 @@ const KNOWN_TAGS = [
4042
NO_LRU
4143
];
4244

43-
const WEB_SPEC_TEST_FILTER = (tags: string[]) =>
44-
tags.indexOf(NO_WEB_TAG) === -1;
45+
// TOOD(mrschmidt): Make this configurable with mocha options.
46+
const RUN_BENCHMARK_TESTS = false;
4547

4648
// The format of one describeSpec written to a JSON file.
4749
interface SpecOutputFormat {
@@ -125,7 +127,9 @@ export function specTest(
125127
let runner: Function;
126128
if (tags.indexOf(EXCLUSIVE_TAG) >= 0) {
127129
runner = it.only;
128-
} else if (!WEB_SPEC_TEST_FILTER(tags)) {
130+
} else if (tags.indexOf(NO_WEB_TAG) >= 0) {
131+
runner = it.skip;
132+
} else if (tags.indexOf(BENCHMARK_TAG) >= 0 && !RUN_BENCHMARK_TESTS) {
129133
runner = it.skip;
130134
} else if (usePersistence && tags.indexOf('no-lru') !== -1) {
131135
// spec should have a comment explaining why it is being skipped.
@@ -135,8 +139,14 @@ export function specTest(
135139
}
136140
const mode = usePersistence ? '(Persistence)' : '(Memory)';
137141
const fullName = `${mode} ${name}`;
138-
runner(fullName, () => {
139-
return spec.runAsTest(fullName, usePersistence);
142+
runner(fullName, async () => {
143+
const start = Date.now();
144+
await spec.runAsTest(fullName, usePersistence);
145+
const end = Date.now();
146+
if (tags.indexOf(BENCHMARK_TAG) >= 0) {
147+
// tslint:disable-next-line:no-console
148+
console.log(`Runtime: ${end - start} ms.`);
149+
}
140150
});
141151
}
142152
} else {

‎packages/firestore/test/unit/specs/perf_spec.test.ts

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/**
2+
* Copyright 2018 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import { Query } from '../../../src/core/query';
18+
import { doc, orderBy, path } from '../../util/helpers';
19+
20+
import { describeSpec, specTest } from './describe_spec';
21+
import { spec } from './spec_builder';
22+
23+
/** The number of iterations for the benchmark spec tests. */
24+
const STEP_COUNT = 10;
25+
26+
describeSpec(
27+
`Performance Tests [${STEP_COUNT} iterations]:`,
28+
['benchmark'],
29+
() => {
30+
specTest('Insert a new document', [], () => {
31+
let steps = spec().withGCEnabled(false);
32+
for (let i = 0; i < STEP_COUNT; ++i) {
33+
steps = steps.userSets(`collection/{i}`, { doc: i }).writeAcks(i);
34+
}
35+
return steps;
36+
});
37+
38+
specTest(
39+
'Start a listen, write a document, ack the write, handle watch snapshot, unlisten',
40+
[],
41+
() => {
42+
let currentVersion = 1;
43+
let steps = spec().withGCEnabled(false);
44+
45+
for (let i = 0; i < STEP_COUNT; ++i) {
46+
const query = Query.atPath(path(`collection/${i}`));
47+
const docLocal = doc(
48+
`collection/${i}`,
49+
0,
50+
{ doc: i },
51+
{ hasLocalMutations: true }
52+
);
53+
const docRemote = doc(`collection/${i}`, ++currentVersion, {
54+
doc: i
55+
});
56+
57+
steps = steps
58+
.userListens(query)
59+
.userSets(`collection/${i}`, { doc: i })
60+
.expectEvents(query, {
61+
added: [docLocal],
62+
fromCache: true,
63+
hasPendingWrites: true
64+
})
65+
.writeAcks(++currentVersion)
66+
.watchAcksFull(query, ++currentVersion, docRemote)
67+
.expectEvents(query, { metadata: [docRemote] })
68+
.userUnlistens(query)
69+
.watchRemoves(query);
70+
}
71+
return steps;
72+
}
73+
);
74+
75+
specTest('Write 100 documents and raise a snapshot', [], () => {
76+
const cachedDocumentCount = 100;
77+
78+
const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
79+
80+
let steps = spec().withGCEnabled(false);
81+
82+
const docs = [];
83+
84+
for (let i = 0; i < cachedDocumentCount; ++i) {
85+
steps.userSets(`collection/${i}`, { v: i });
86+
docs.push(
87+
doc(`collection/${i}`, 0, { v: i }, { hasLocalMutations: true })
88+
);
89+
}
90+
91+
for (let i = 1; i <= STEP_COUNT; ++i) {
92+
steps = steps
93+
.userListens(query)
94+
.expectEvents(query, {
95+
added: docs,
96+
fromCache: true,
97+
hasPendingWrites: true
98+
})
99+
.userUnlistens(query);
100+
}
101+
102+
return steps;
103+
});
104+
105+
specTest('Update a single document', [], () => {
106+
let steps = spec().withGCEnabled(false);
107+
steps = steps.userSets(`collection/doc`, { v: 0 });
108+
for (let i = 1; i <= STEP_COUNT; ++i) {
109+
steps = steps.userPatches(`collection/doc`, { v: i }).writeAcks(i);
110+
}
111+
return steps;
112+
});
113+
114+
specTest(
115+
'Update a document and wait for snapshot with existing listen',
116+
[],
117+
() => {
118+
const query = Query.atPath(path(`collection/doc`));
119+
120+
let currentVersion = 1;
121+
let steps = spec().withGCEnabled(false);
122+
123+
let docLocal = doc(
124+
`collection/doc`,
125+
0,
126+
{ v: 0 },
127+
{ hasLocalMutations: true }
128+
);
129+
let docRemote = doc(`collection/doc`, ++currentVersion, { v: 0 });
130+
let lastRemoteVersion = currentVersion;
131+
132+
steps = steps
133+
.userListens(query)
134+
.userSets(`collection/doc`, { v: 0 })
135+
.expectEvents(query, {
136+
added: [docLocal],
137+
fromCache: true,
138+
hasPendingWrites: true
139+
})
140+
.writeAcks(++currentVersion)
141+
.watchAcksFull(query, ++currentVersion, docRemote)
142+
.expectEvents(query, { metadata: [docRemote] });
143+
144+
for (let i = 1; i <= STEP_COUNT; ++i) {
145+
docLocal = doc(
146+
`collection/doc`,
147+
lastRemoteVersion,
148+
{ v: i },
149+
{ hasLocalMutations: true }
150+
);
151+
docRemote = doc(`collection/doc`, ++currentVersion, { v: i });
152+
lastRemoteVersion = currentVersion;
153+
154+
steps = steps
155+
.userPatches(`collection/doc`, { v: i })
156+
.expectEvents(query, {
157+
modified: [docLocal],
158+
hasPendingWrites: true
159+
})
160+
.writeAcks(++currentVersion)
161+
.watchSends({ affects: [query] }, docRemote)
162+
.watchSnapshots(++currentVersion)
163+
.expectEvents(query, { metadata: [docRemote] });
164+
}
165+
return steps;
166+
}
167+
);
168+
169+
specTest(
170+
'Process 100 documents from Watch and wait for snapshot',
171+
[],
172+
() => {
173+
const documentsPerStep = 100;
174+
175+
const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
176+
177+
let currentVersion = 1;
178+
let steps = spec().withGCEnabled(false);
179+
180+
steps = steps
181+
.userListens(query)
182+
.watchAcksFull(query, currentVersion)
183+
.expectEvents(query, {});
184+
185+
for (let i = 1; i <= STEP_COUNT; ++i) {
186+
const docs = [];
187+
188+
for (let j = 0; j < documentsPerStep; ++j) {
189+
docs.push(
190+
doc(`collection/${j}`, ++currentVersion, { v: currentVersion })
191+
);
192+
}
193+
194+
const changeType = i === 1 ? 'added' : 'modified';
195+
196+
steps = steps
197+
.watchSends({ affects: [query] }, ...docs)
198+
.watchSnapshots(++currentVersion)
199+
.expectEvents(query, { [changeType]: docs });
200+
}
201+
202+
return steps;
203+
}
204+
);
205+
206+
specTest(
207+
'Process 100 documents from Watch and wait for snapshot, then unlisten and wait for a cached snapshot',
208+
[],
209+
() => {
210+
const documentsPerStep = 100;
211+
212+
let currentVersion = 1;
213+
let steps = spec().withGCEnabled(false);
214+
215+
for (let i = 1; i <= STEP_COUNT; ++i) {
216+
const collPath = `collection/${i}/coll`;
217+
const query = Query.atPath(path(collPath)).addOrderBy(orderBy('v'));
218+
219+
const docs = [];
220+
for (let j = 0; j < documentsPerStep; ++j) {
221+
docs.push(doc(`${collPath}/${j}`, ++currentVersion, { v: j }));
222+
}
223+
224+
steps = steps
225+
.userListens(query)
226+
.watchAcksFull(query, ++currentVersion, ...docs)
227+
.expectEvents(query, { added: docs })
228+
.userUnlistens(query)
229+
.watchRemoves(query)
230+
.userListens(query, 'resume-token-' + currentVersion)
231+
.expectEvents(query, { added: docs, fromCache: true })
232+
.watchAcksFull(query, ++currentVersion)
233+
.expectEvents(query, {})
234+
.userUnlistens(query)
235+
.watchRemoves(query);
236+
}
237+
238+
return steps;
239+
}
240+
);
241+
}
242+
);

0 commit comments

Comments
 (0)