Skip to content

Commit d191c78

Browse files
tinayuangaotinayuangao
committed
Download from database and compare diff
1 parent 6413e39 commit d191c78

File tree

5 files changed

+128
-151
lines changed

5 files changed

+128
-151
lines changed

‎functions/index.js

Lines changed: 83 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -5,125 +5,115 @@ const gcs = require('@google-cloud/storage')();
55
const admin = require('firebase-admin');
66
const jwt = require('jsonwebtoken');
77
const fs = require('fs');
8+
89
admin.initializeApp(functions.config().firebase);
910

1011
const dataTypes = ['filenames', 'commit', 'result', 'results', 'sha', 'travis'];
1112
const repoSlug = functions.config().repo.slug;
1213
const secret = functions.config().secret.key;
1314
const bucket = gcs.bucket(functions.config().firebase.storageBucket);
1415

16+
/** Copy valid data from /temp/screenshot/reports/$prNumber/$secureToken/ to /screenshot/reports/$prNumber */
1517
exports.copyData = functions.database.ref('/temp/screenshot/reports/{prNumber}/{token1}/{token2}/{token3}/{dataType}')
1618
.onWrite(event => {
17-
// Only edit data when it is first created.
18-
if (event.data.previous.exists()) {
19-
return;
20-
}
21-
// Exit when the data is deleted.
22-
if (!event.data.exists()) {
23-
return;
24-
}
25-
26-
const dataType = event.params.dataType;
27-
const prNumber = event.params.prNumber;
28-
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
29-
const original = event.data.val();
30-
31-
if (!dataTypes.includes(dataType)) {
32-
console.log(`Invalid data ${dataType}`);
33-
return;
34-
}
35-
36-
return validateSecureToken(secureToken, prNumber).then((payload) => {
37-
console.log(`Log ${original} to screenshot/reports/${prNumber}/filenames`);
38-
return admin.database().ref().child('screenshot/reports').child(prNumber).child(dataType).set(original).then(() => {
19+
// Only edit data when it is first created. Exit when the data is deleted.
20+
if (event.data.previous.exists() || !event.data.exists()) {
21+
return;
22+
}
23+
24+
const dataType = event.params.dataType;
25+
const prNumber = event.params.prNumber;
26+
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
27+
const original = event.data.val();
28+
29+
if (!dataTypes.includes(dataType)) {
30+
return;
31+
}
32+
33+
return validateSecureToken(secureToken, prNumber).then((payload) => {
34+
return admin.database().ref().child('screenshot/reports').child(prNumber).child(dataType).set(original).then(() => {
3935
return event.data.ref.parent.set(null);
40-
});
41-
}).catch((error) => {
42-
console.log(`Invalid secure token ${secureToken} ${error}`);
43-
return event.data.ref.parent.set(null);
44-
})
45-
36+
});
37+
}).catch((error) => {
38+
console.log(`Invalid secure token ${secureToken} ${error}`);
39+
return event.data.ref.parent.set(null);
4640
});
41+
});
4742

43+
/** Copy valid data from database /temp/screenshot/images/$prNumber/$secureToken/ to storage /screenshots/$prNumber */
4844
exports.copyImage = functions.database.ref('/temp/screenshot/images/{prNumber}/{token1}/{token2}/{token3}/{dataType}/{filename}')
4945
.onWrite(event => {
50-
// Only edit data when it is first created.
51-
if (event.data.previous.exists()) {
52-
return;
53-
}
54-
// Exit when the data is deleted.
55-
if (!event.data.exists()) {
56-
return;
57-
}
58-
59-
const dataType = event.params.dataType;
60-
const prNumber = event.params.prNumber;
61-
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
62-
const saveFilename = `${event.params.filename}.screenshot.png`;
63-
64-
if (dataType != 'diff' && dataType != 'test') {
65-
return;
66-
}
67-
68-
return validateSecureToken(secureToken, prNumber).then((payload) => {
46+
// Only edit data when it is first created. Exit when the data is deleted.
47+
if (event.data.previous.exists() || !event.data.exists()) {
48+
return;
49+
}
50+
51+
const dataType = event.params.dataType;
52+
const prNumber = event.params.prNumber;
53+
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
54+
const saveFilename = `${event.params.filename}.screenshot.png`;
55+
56+
if (dataType != 'diff' && dataType != 'test') {
57+
return;
58+
}
59+
60+
return validateSecureToken(secureToken, prNumber).then((payload) => {
6961
const tempPath = `/tmp/${dataType}-${saveFilename}`
7062
const filePath = `screenshots/${prNumber}/${dataType}/${saveFilename}`;
7163
const binaryData = new Buffer(event.data.val(), 'base64').toString('binary');
7264
fs.writeFile(tempPath, binaryData, 'binary');
7365
return bucket.upload(tempPath, {
74-
destination: filePath
75-
}).then(() => {
76-
return event.data.ref.parent.set(null);
66+
destination: filePath
67+
}).then(() => {
68+
return event.data.ref.parent.set(null);
7769
});
78-
}).catch((error) => {
79-
console.log(`Invalid secure token ${secureToken} ${error}`);
80-
return event.data.ref.parent.set(null);
81-
});
70+
}).catch((error) => {
71+
console.log(`Invalid secure token ${secureToken} ${error}`);
72+
return event.data.ref.parent.set(null);
8273
});
74+
});
8375

76+
/**
77+
* Copy valid goldens from storage /goldens/ to database /screenshot/goldens/
78+
* so we can read the goldens without credentials
79+
*/
8480
exports.copyGoldens = functions.storage.bucket(functions.config().firebase.storageBucket).object().onChange(event => {
85-
const object = event.data;
86-
const fileBucket = object.bucket;
87-
const filePath = object.name;
88-
const contentType = object.contentType; // File content type.
89-
const resourceState = object.resourceState;
90-
91-
// Get the file name.
92-
const fileNames = filePath.split('/');
93-
if (fileNames.length != 2 && fileNames[0] != 'golds') {
94-
return;
95-
}
96-
97-
const filenameKey = fileNames[1].replace('.screenshot.png', '');
98-
// Exit if this is a move or deletion event.
99-
if (resourceState === 'not_exists') {
100-
console.log('This is a deletion event.');
101-
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(null);
102-
}
103-
104-
// Download file from bucket.
105-
const bucket = gcs.bucket(fileBucket);
106-
const tempFilePath = `/tmp/${fileNames[1]}`;
107-
return bucket.file(filePath).download({
108-
destination: tempFilePath
109-
}).then(() => {
110-
const data = fs.readFileSync(tempFilePath);
111-
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(data);
112-
});
81+
const filePath = event.data.name;
82+
83+
// Get the file name.
84+
const fileNames = filePath.split('/');
85+
if (fileNames.length != 2 && fileNames[0] != 'goldens') {
86+
return;
87+
}
88+
const filenameKey = fileNames[1].replace('.screenshot.png', '');
89+
90+
if (event.data.resourceState === 'not_exists') {
91+
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(null);
92+
}
93+
94+
// Download file from bucket.
95+
const bucket = gcs.bucket(event.data.bucket);
96+
const tempFilePath = `/tmp/${fileNames[1]}`;
97+
return bucket.file(filePath).download({
98+
destination: tempFilePath
99+
}).then(() => {
100+
const data = fs.readFileSync(tempFilePath);
101+
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(data);
102+
});
113103
});
114104

115105
function validateSecureToken(token, prNumber) {
116106
return new Promise((resolve, reject) => {
117-
jwt.verify(token, secret, {issuer: 'Travis CI, GmbH'}, (err, payload) => {
118-
if (err) {
119-
reject(err.message || err);
120-
} else if (payload.slug !== repoSlug) {
121-
reject(`jwt slug invalid. expected: ${repoSlug}`);
122-
} else if (payload['pull-request'].toString() !== prNumber) {
123-
reject(`jwt pull-request invalid. expected: ${prNumber} actual: ${payload['pull-request']}`);
124-
} else {
125-
resolve(payload);
126-
}
127-
});
107+
jwt.verify(token, secret, {issuer: 'Travis CI, GmbH'}, (err, payload) => {
108+
if (err) {
109+
reject(err.message || err);
110+
} else if (payload.slug !== repoSlug) {
111+
reject(`jwt slug invalid. expected: ${repoSlug}`);
112+
} else if (payload['pull-request'].toString() !== prNumber) {
113+
reject(`jwt pull-request invalid. expected: ${prNumber} actual: ${payload['pull-request']}`);
114+
} else {
115+
resolve(payload);
116+
}
117+
});
128118
});
129119
}

‎functions/package.json

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,10 @@
11
{
2-
"name": "uppercase-quickstart-functions",
3-
"description": "Uppercaser Firebase Functions Quickstart sample",
2+
"name": "angular-material2-functions",
3+
"description": "Angular Material2 screenshot test functions",
44
"dependencies": {
55
"@google-cloud/storage": "^0.8.0",
66
"firebase-admin": "^4.1.3",
77
"firebase-functions": "^0.5.2",
8-
"jsonwebtoken": "^7.3.0",
9-
"typescript": "^2.2.1"
10-
},
11-
"devDependencies": {
12-
"chai": "^3.5.0",
13-
"chai-as-promised": "^6.0.0",
14-
"mocha": "^3.2.0",
15-
"sinon": "^1.17.7"
16-
},
17-
"scripts": {
18-
"test": "./node_modules/.bin/mocha --reporter spec"
8+
"jsonwebtoken": "^7.3.0"
199
}
2010
}

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
"dgeni-packages": "^0.16.5",
5959
"firebase-admin": "^4.1.2",
6060
"firebase-tools": "^2.2.1",
61+
"firebase": "^3.7.2",
6162
"fs-extra": "^2.0.0",
6263
"glob": "^7.1.1",
6364
"google-cloud": "^0.48.0",

‎tools/gulp/tasks/screenshots.ts

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import {task} from 'gulp';
2-
import {readdirSync, statSync, existsSync, mkdirp, readFileSync} from 'fs-extra';
2+
import {readdirSync, statSync, existsSync, mkdirp, readFileSync, writeFileSync} from 'fs-extra';
33
import * as path from 'path';
44
import * as admin from 'firebase-admin';
55
import * as firebase from 'firebase';
@@ -20,8 +20,8 @@ task('screenshots', () => {
2020
let firebaseApp = connectFirebaseScreenshots();
2121
let database = firebaseApp.database();
2222

23-
return getScreenshotFiles(database, filenames)
24-
.then((files: any[]) => downloadAllGoldsAndCompare(database, prNumber))
23+
return getScreenshotFiles(database)
24+
.then(() => downloadAllGoldsAndCompare(database, prNumber))
2525
.then((results: boolean) => updateResult(database, prNumber, results))
2626
.then((result: boolean) => updateGithubStatus(prNumber, result))
2727
.then(() => uploadScreenshotsData(database, 'diff', prNumber))
@@ -47,7 +47,7 @@ function updateResult(database: firebase.database.Database, prNumber: string, re
4747
return getPullRequestRef(database, prNumber).child('result').set(result).then(() => result);
4848
}
4949

50-
function getPullRequestRef(database: firebase.database.Database, prNumber: string) {
50+
function getPullRequestRef(database: firebase.database.Database | admin.database.Database, prNumber: string) {
5151
return database.ref(FIREBASE_REPORT).child(prNumber).child(getSecureToken());
5252
}
5353

@@ -61,17 +61,19 @@ function updateTravis(database: firebase.database.Database,
6161
}
6262

6363
/** Get a list of filenames from firebase database. */
64-
function getScreenshotFiles(database: firebase.database.Database, filenames: string[]) {
64+
function getScreenshotFiles(database: firebase.database.Database) {
6565
mkdirp(path.join(SCREENSHOT_DIR, `golds`));
6666
mkdirp(path.join(SCREENSHOT_DIR, `diff`));
6767

6868
return database.ref('screenshot/goldens').once('value').then((snapshot: firebase.database.DataSnapshot) => {
69-
let map = snapshot.val();
70-
map.forEach((key, value) => {
71-
var binaryData = new Buffer(value, 'base64').toString('binary');
72-
fs.writeFileSync(`${SCREENSHOT_DIR}/golds/${key}.screenshot.png`, binaryData, 'binary');
69+
let counter = 0;
70+
snapshot.forEach((childSnapshot: firebase.database.DataSnapshot) => {
71+
let key = childSnapshot.key;
72+
let binaryData = new Buffer(childSnapshot.val(), 'base64').toString('binary');
73+
writeFileSync(`${SCREENSHOT_DIR}/golds/${key}.screenshot.png`, binaryData, 'binary');
74+
if (++counter == snapshot.numChildren()) return true;
7375
});
74-
});
76+
}).catch((error: any) => console.log(error));
7577
}
7678

7779
function extractScreenshotName(fileName: string) {
@@ -85,21 +87,9 @@ function getLocalScreenshotFiles(dir: string): string[] {
8587
}
8688

8789
/**
88-
* Upload screenshots to google cloud storage.
90+
* Get processed secure token. The jwt token is connected by '.', while '.' is not a valid
91+
* path in firebase database. Replace all '.' to '/' to make the path valid
8992
*/
90-
function uploadScreenshots() {
91-
let bucket = openScreenshotsBucket();
92-
93-
let promises: any[] = [];
94-
getLocalScreenshotFiles(SCREENSHOT_DIR).forEach((file: string) => {
95-
let fileName = path.join(SCREENSHOT_DIR, file);
96-
let destination = `golds/${file}`;
97-
promises.push(bucket.upload(fileName, { destination: destination }));
98-
});
99-
return Promise.all(promises);
100-
}
101-
102-
/** Get processed secure token */
10393
function getSecureToken() {
10494
return process.env['FIREBASE_ACCESS_TOKEN'].replace(/[.]/g, '/');
10595
}
@@ -118,11 +108,9 @@ function uploadScreenshotsData(database: firebase.database.Database, mode: 'test
118108
let fileName = path.join(localDir, file);
119109
let filenameKey = extractScreenshotName(fileName);
120110
let secureToken = getSecureToken();
121-
readFileSync(fileName, (err, data) => {
122-
if (err) throw err;
123-
promises.push(database.ref(FIREBASE_IMAGE).child(prNumber)
124-
.child(secureToken).child(mode).child(filenameKey).set(data));
125-
});
111+
let data = readFileSync(fileName);
112+
promises.push(database.ref(FIREBASE_IMAGE).child(prNumber)
113+
.child(secureToken).child(mode).child(filenameKey).set(data));
126114
});
127115
return Promise.all(promises);
128116
}
@@ -190,3 +178,18 @@ function updateGithubStatus(prNumber: number, result: boolean) {
190178
url: `http://material2-screenshots.firebaseapp.com/${prNumber}`
191179
});
192180
}
181+
182+
/**
183+
* Upload screenshots to google cloud storage.
184+
*/
185+
function uploadScreenshots() {
186+
let bucket = openScreenshotsBucket();
187+
188+
let promises: any[] = [];
189+
getLocalScreenshotFiles(SCREENSHOT_DIR).forEach((file: string) => {
190+
let fileName = path.join(SCREENSHOT_DIR, file);
191+
let destination = `golds/${file}`;
192+
promises.push(bucket.upload(fileName, { destination: destination }));
193+
});
194+
return Promise.all(promises);
195+
}

0 commit comments

Comments
 (0)