Skip to content

Commit 525988d

Browse files
tinayuangaotinayuangao
committed
Address comments
1 parent 328f4c4 commit 525988d

File tree

4 files changed

+100
-115
lines changed

4 files changed

+100
-115
lines changed

‎tools/gulp/task_helpers.ts

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -208,14 +208,8 @@ export function isTravisPushBuild() {
208208
return process.env['TRAVIS_PULL_REQUEST'] === 'false';
209209
}
210210

211-
/** Decode the token for Travis to use. */
212-
function decode(str: string): string {
213-
return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n');
214-
}
215-
216211
/** Open Google Cloud Storage for screenshots */
217-
export function openScreenshotsCloudStorage() {
218-
// Enable Storage
212+
export function openScreenshotsBucket() {
219213
let gcs = gcloud.storage({
220214
projectId: 'material2-screenshots',
221215
credentials: {
@@ -236,12 +230,17 @@ export function openFirebaseScreenshotsDatabase() {
236230
credential: firebaseAdmin.credential.cert({
237231
project_id: 'material2-screenshots',
238232
client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
239-
// In Travis CI the private key will be incorrect because the line-breaks are escaped.
240-
// The line-breaks need to persist in the service account private key.
241233
private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
242234
}),
243235
databaseURL: 'https://material2-screenshots.firebaseio.com'
244236
}, 'material2-screenshots');
245237

246238
return screenshotApp.database();
247239
}
240+
241+
/** Decode the token for Travis to use. */
242+
function decode(str: string): string {
243+
// In Travis CI the private key will be incorrect because the line-breaks are escaped.
244+
// The line-breaks need to persist in the service account private key.
245+
return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n');
246+
}

‎tools/gulp/tasks/screenshots.ts

Lines changed: 53 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
11
import {task} from 'gulp';
2-
import {readdirSync, statSync, existsSync, mkdirSync} from 'fs';
3-
import {openScreenshotsCloudStorage, openFirebaseScreenshotsDatabase} from '../task_helpers';
2+
import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra';
43
import * as path from 'path';
54
import * as admin from 'firebase-admin';
5+
import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers';
6+
import {updateGithubStatus} from '../util-functions';
7+
68
const request = require('request');
79
const imageDiff = require('image-diff');
810

911
const SCREENSHOT_DIR = './screenshots';
10-
const FIREBASE_FILELIST = 'screenshot/filenames';
1112
const FIREBASE_REPORT = 'screenshot/reports';
1213

1314
/** Task which upload screenshots generated from e2e test. */
1415
task('screenshots', () => {
1516
let prNumber = process.env['TRAVIS_PULL_REQUEST'];
1617
if (prNumber) {
1718
let database = openFirebaseScreenshotsDatabase();
18-
return getScreenFilenames(database)
19-
.then((filenames: string[]) => downloadAllGolds(filenames, database, prNumber))
19+
return getScreenshotFiles(database)
20+
.then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber))
2021
.then((results: boolean) => updateResult(database, prNumber, results))
2122
.then((result: boolean) => updateGithubStatus(result, prNumber))
22-
.then(() => setScreenFilenames(database, prNumber))
2323
.then(() => uploadScreenshots(prNumber, 'diff'))
2424
.then(() => uploadScreenshots(prNumber, 'test'))
2525
.then(() => updateTravis(database, prNumber))
@@ -29,108 +29,92 @@ task('screenshots', () => {
2929

3030
function updateFileResult(database: admin.database.Database, prNumber: string,
3131
filenameKey: string, result: boolean): admin.Promise<void>{
32-
return database.ref(FIREBASE_REPORT).child(`${prNumber}/results/${filenameKey}`).set(result);
32+
return database.ref(FIREBASE_REPORT).child(prNumber).child('results').child(filenameKey).set(result);
3333
}
3434

3535
function updateResult(database: admin.database.Database, prNumber: string,
36-
result: boolean): admin.Promise<void> {
37-
return database.ref(FIREBASE_REPORT).child(`${prNumber}/result`).set(result).then(() => result);
36+
result: boolean) {
37+
return database.ref(FIREBASE_REPORT).child(prNumber).child('result').set(result).then(() => result);
3838
}
3939

4040
function updateTravis(database: admin.database.Database,
4141
prNumber: string): admin.Promise<void> {
4242
return database.ref(FIREBASE_REPORT).child(prNumber).update({
43-
'commit': process.env['TRAVIS_COMMIT'],
44-
'sha': process.env['TRAVIS_PULL_REQUEST_SHA'],
45-
'travis': process.env['TRAVIS_JOB_ID'],
43+
commit: process.env['TRAVIS_COMMIT'],
44+
sha: process.env['TRAVIS_PULL_REQUEST_SHA'],
45+
travis: process.env['TRAVIS_JOB_ID'],
4646
});
4747
}
4848

4949
/** Get a list of filenames from firebase database. */
50-
function getScreenFilenames(database: admin.database.Database): admin.Promise<string[]> {
51-
return database.ref(FIREBASE_FILELIST).once('value')
52-
.then((snapshots: admin.database.DataSnapshot) => {
53-
return snapshots.val();
50+
function getScreenshotFiles(database: admin.database.Database): admin.Promise<any[]> {
51+
let bucket = openScreenshotsBucket();
52+
return bucket.getFiles({ prefix: 'golds/' }).then(function(data: any) {
53+
return data[0].filter((file:any) => file.name.endsWith('.screenshot.png'));
5454
});
5555
}
5656

57-
/** Upload a list of filenames to firebase database as gold. */
58-
function setScreenFilenames(database: admin.database.Database,
59-
reportKey?: string): admin.Promise<void> {
60-
let filenames: string[] = [];
61-
readdirSync(SCREENSHOT_DIR).map((file: string) => {
62-
let fullName = path.join(SCREENSHOT_DIR, file);
63-
let key = file.replace('.screenshot.png', '');
64-
if (!statSync(fullName).isDirectory() && key) {
65-
filenames.push(file);
66-
}
67-
});
68-
let filelistDatabase = reportKey ?
69-
database.ref(FIREBASE_REPORT).child(reportKey).child('filenames') :
70-
database.ref(FIREBASE_FILELIST);
71-
return filelistDatabase.set(filenames);
57+
function extractScreenshotName(fileName: string) {
58+
return path.basename(fileName, '.screenshot.png');
59+
}
60+
61+
function getLocalScreenshotFiles(dir: string): string[] {
62+
return readdirSync(dir)
63+
.filter((fileName: string) => !statSync(path.join(SCREENSHOT_DIR, fileName)).isDirectory())
64+
.filter((fileName: string) => fileName.endsWith('.screenshot.png'));
7265
}
7366

7467
/**
7568
* Upload screenshots to google cloud storage.
76-
* @param {string} reportKey - The key used in firebase. Here it is the PR number.
77-
* If there's no reportKey, we will upload images to 'golds/' folder
78-
* @param {string} mode - Can be 'test' or 'diff' or null.
69+
* @param prNumber - The key used in firebase. Here it is the PR number.
70+
* If there's no prNumber, we will upload images to 'golds/' folder
71+
* @param mode - Can be 'test' or 'diff' or null.
7972
* If the images are the test results, mode should be 'test'.
8073
* If the images are the diff images generated, mode should be 'diff'.
8174
* For golds mode should be null.
8275
*/
83-
function uploadScreenshots(reportKey?: string, mode?: 'test' | 'diff') {
84-
let bucket = openScreenshotsCloudStorage();
76+
function uploadScreenshots(prNumber?: string, mode?: 'test' | 'diff') {
77+
let bucket = openScreenshotsBucket();
8578

8679
let promises: admin.Promise<void>[] = [];
87-
let localDir = mode == 'diff' ? `${SCREENSHOT_DIR}/diff` : SCREENSHOT_DIR;
88-
readdirSync(localDir).map((file: string) => {
80+
let localDir = mode == 'diff' ? path.join(SCREENSHOT_DIR, 'diff') : SCREENSHOT_DIR;
81+
getLocalScreenshotFiles(localDir).forEach((file: string) => {
8982
let fileName = path.join(localDir, file);
90-
let key = file.replace('.screenshot.png', '');
91-
let destination = (mode == null || !reportKey) ?
92-
`golds/${file}` : `screenshots/${reportKey}/${mode}/${file}`;
93-
94-
if (!statSync(fileName).isDirectory() && key) {
95-
promises.push(bucket.upload(fileName, { destination: destination }));
96-
}
83+
let destination = (mode == null || !prNumber) ?
84+
`golds/${file}` : `screenshots/${prNumber}/${mode}/${file}`;
85+
promises.push(bucket.upload(fileName, { destination: destination }));
9786
});
9887
return admin.Promise.all(promises);
9988
}
10089

101-
/** Check whether the directory exists. If not then create one. */
102-
function _makeDir(dirName: string) {
103-
if (!existsSync(dirName)) {
104-
mkdirSync(dirName, '744');
105-
}
106-
}
107-
10890
/** Download golds screenshots. */
109-
function downloadAllGolds(
110-
filenames: string[], database: admin.database.Database,
111-
reportKey: string): admin.Promise<boolean> {
112-
_makeDir(`${SCREENSHOT_DIR}/golds`);
91+
function downloadAllGoldsAndCompare(
92+
files: any[], database: admin.database.Database,
93+
prNumber: string): admin.Promise<boolean> {
94+
95+
mkdirp(path.join(SCREENSHOT_DIR, `golds`));
96+
mkdirp(path.join(SCREENSHOT_DIR, `diff`));
11397

114-
return admin.Promise.all(filenames.map((filename: string) => {
115-
return downloadGold(filename).then(() => diffScreenshot(filename, database, reportKey));
98+
return admin.Promise.all(files.map((file: any) => {
99+
return downloadGold(file).then(() => diffScreenshot(file.name, database, prNumber));
116100
})).then((results: boolean[]) => results.every((value: boolean) => value == true));
117101
}
118102

119103
/** Download one gold screenshot */
120-
function downloadGold(filename: string): Promise<void> {
121-
let bucket = openScreenshotsCloudStorage();
122-
return bucket.file(`golds/${filename}`).download({
123-
destination: `${SCREENSHOT_DIR}/golds/${filename}`
104+
function downloadGold(file: any): Promise<void> {
105+
return file.download({
106+
destination: path.join(SCREENSHOT_DIR, file.name)
124107
});
125108
}
126109

127110
function diffScreenshot(filename: string, database: admin.database.Database,
128-
reportKey: string): admin.Promise<boolean> {
111+
prNumber: string): admin.Promise<boolean> {
129112
// TODO(tinayuangao): Run the downloads and diffs in parallel.
130-
let goldUrl = `${SCREENSHOT_DIR}/golds/${filename}`;
131-
let pullRequestUrl = `${SCREENSHOT_DIR}/${filename}`;
132-
let diffUrl = `${SCREENSHOT_DIR}/diff/${filename}`;
133-
let filenameKey = filename.replace('.screenshot.png', '');
113+
filename = path.basename(filename);
114+
let goldUrl = path.join(SCREENSHOT_DIR, `golds`, filename);
115+
let pullRequestUrl = path.join(SCREENSHOT_DIR, filename);
116+
let diffUrl = path.join(SCREENSHOT_DIR, `diff`, filename);
117+
let filenameKey = extractScreenshotName(filename);
134118

135119
if (existsSync(goldUrl) && existsSync(pullRequestUrl)) {
136120
return new admin.Promise((resolve: any, reject: any) => {
@@ -145,46 +129,10 @@ function diffScreenshot(filename: string, database: admin.database.Database,
145129
reject(err);
146130
}
147131
resolve(imagesAreSame);
148-
return updateFileResult(database, reportKey, filenameKey, imagesAreSame);
132+
return updateFileResult(database, prNumber, filenameKey, imagesAreSame);
149133
});
150134
});
151135
} else {
152-
return updateFileResult(database, reportKey, filenameKey, false).then(() => false);
136+
return updateFileResult(database, prNumber, filenameKey, false).then(() => false);
153137
}
154138
}
155-
156-
function decode(value: string): string {
157-
return value.split('').reverse().join('');
158-
}
159-
160-
function updateGithubStatus(result: boolean, prNumber: string) {
161-
let state = result ? 'success' : 'failure';
162-
let sha = process.env['TRAVIS_PULL_REQUEST_SHA'];
163-
let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']);
164-
165-
let data = JSON.stringify({
166-
"state": state,
167-
"target_url": `http://material2-screenshots.firebaseapp.com/${prNumber}`,
168-
"context": "screenshot-diff",
169-
"description": `Screenshot test ${state}`
170-
});
171-
172-
let headers = {
173-
'Authorization': `token ${token}`,
174-
'User-Agent': 'ScreenshotDiff/1.0.0',
175-
'Content-Type': 'application/json',
176-
'Content-Length': Buffer.byteLength(data)
177-
};
178-
179-
return new admin.Promise((resolve, reject) => {
180-
request({
181-
url: `https://api.github.com/repos/angular/material2/statuses/${sha}`,
182-
method: 'POST',
183-
form: data,
184-
headers: headers
185-
}, function (error: any, response: any, body: any){
186-
resolve(response.statusCode);
187-
console.log(response.statusCode);
188-
});
189-
});
190-
}

‎tools/gulp/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"declaration": true,
44
"emitDecoratorMetadata": true,
55
"experimentalDecorators": true,
6-
"lib": ["es6", "es2015", "dom"],
6+
"lib": ["es6", "es2015"],
77
"module": "commonjs",
88
"moduleResolution": "node",
99
"noEmitOnError": true,

‎tools/gulp/util-functions.ts

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
const request = require('request');
2+
import * as admin from 'firebase-admin';
3+
4+
/** Update github pr status to success/failure */
5+
export function updateGithubStatus(result: boolean, prNumber: string) {
6+
let state = result ? 'success' : 'failure';
7+
let sha = process.env['TRAVIS_PULL_REQUEST_SHA'];
8+
let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']);
9+
10+
let data = JSON.stringify({
11+
state: state,
12+
target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`,
13+
context: "screenshot-diff",
14+
description: `Screenshot test ${state}`
15+
});
16+
17+
let headers = {
18+
'Authorization': `token ${token}`,
19+
'User-Agent': 'ScreenshotDiff/1.0.0',
20+
'Content-Type': 'application/json',
21+
'Content-Length': Buffer.byteLength(data)
22+
};
23+
24+
return new admin.Promise((resolve, reject) => {
25+
request({
26+
url: `https://api.github.com/repos/angular/material2/statuses/${sha}`,
27+
method: 'POST',
28+
form: data,
29+
headers: headers
30+
}, function (error: any, response: any, body: any){
31+
resolve(response.statusCode);
32+
});
33+
});
34+
}
35+
36+
function decode(value: string): string {
37+
return value.split('').reverse().join('');
38+
}

0 commit comments

Comments
 (0)