Skip to content

Commit b2c542d

Browse files
mcaulifnmcaulifn
committed
fix(logging): Add context to scale logs
Signed-off-by: Nathaniel McAuliffe <[email protected]>
1 parent f37df23 commit b2c542d

File tree

5 files changed

+82
-34
lines changed

5 files changed

+82
-34
lines changed

‎modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ import {
1111
} from '@octokit/auth-app/dist-types/types';
1212
import { OctokitOptions } from '@octokit/core/dist-types/types';
1313
import { getParameterValue } from './ssm';
14-
import { logger as rootLogger } from './logger';
14+
import { logger as rootLogger, LogFields } from './logger';
1515

16-
const logger = rootLogger.getChildLogger();
16+
const logger = rootLogger.getChildLogger({ name: 'gh-auth' });
1717

1818
export async function createOctoClient(token: string, ghesApiUrl = ''): Promise<Octokit> {
1919
const ocktokitOptions: OctokitOptions = {
@@ -55,7 +55,7 @@ async function createAuth(installationId: number | undefined, ghesApiUrl: string
5555
};
5656
if (installationId) authOptions = { ...authOptions, installationId };
5757

58-
logger.debug(`GHES API URL: ${ghesApiUrl}`);
58+
logger.debug(LogFields.fields, `GHES API URL: ${ghesApiUrl}`);
5959
if (ghesApiUrl) {
6060
authOptions.request = request.defaults({
6161
baseUrl: ghesApiUrl,

‎modules/runners/lambdas/runners/src/scale-runners/logger.ts

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,39 @@
1-
import { Logger } from 'tslog';
1+
import { Logger, ISettingsParam } from 'tslog';
22

33
export const logger = new Logger({
44
colorizePrettyLogs: false,
55
displayInstanceName: false,
66
maskAnyRegEx: ['--token [A-Z0-9]*'],
77
minLevel: process.env.LOG_LEVEL || 'info',
8-
name: 'scale-up',
8+
name: 'scale-runners',
99
overwriteConsole: true,
1010
type: process.env.LOG_TYPE || 'pretty',
1111
});
12+
13+
//interface LogFields {
14+
// [key: string]: string;
15+
//}
16+
17+
export class LogFields {
18+
static fields: { [key: string]: string } = {};
19+
}
20+
21+
// Should I overload each method...? This would allow for always adding fields
22+
23+
/*export class AwesomeLogger {
24+
//logger: Logger;
25+
params: ISettingsParam = {
26+
colorizePrettyLogs: false,
27+
displayInstanceName: false,
28+
maskAnyRegEx: ['--token [A-Z0-9]*'],
29+
minLevel: process.env.LOG_LEVEL || 'info',
30+
overwriteConsole: true,
31+
type: process.env.LOG_TYPE || 'pretty',
32+
};
33+
34+
static fields: LogFields = {};
35+
36+
constructor(name: string) {
37+
new Logger({ ...this.params, name: name });
38+
}
39+
}*/

‎modules/runners/lambdas/runners/src/scale-runners/runners.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { EC2, SSM } from 'aws-sdk';
2-
import { logger as rootLogger } from './logger';
2+
import { logger as rootLogger, LogFields } from './logger';
33

4-
const logger = rootLogger.getChildLogger();
4+
const logger = rootLogger.getChildLogger({ name: 'runners' });
55

66
export interface RunnerList {
77
instanceId: string;
@@ -75,16 +75,20 @@ export async function terminateRunner(instanceId: string): Promise<void> {
7575
InstanceIds: [instanceId],
7676
})
7777
.promise();
78-
logger.info(`Runner ${instanceId} has been terminated.`);
78+
logger.info(LogFields.fields, `Runner ${instanceId} has been terminated.`);
7979
}
8080

8181
export async function createRunner(runnerParameters: RunnerInputParameters, launchTemplateName: string): Promise<void> {
82-
logger.debug('Runner configuration: ' + JSON.stringify(runnerParameters));
82+
logger.debug(LogFields.fields, 'Runner configuration: ' + JSON.stringify(runnerParameters));
8383
const ec2 = new EC2();
8484
const runInstancesResponse = await ec2
8585
.runInstances(getInstanceParams(launchTemplateName, runnerParameters))
8686
.promise();
87-
logger.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(','));
87+
logger.info(
88+
LogFields.fields,
89+
'Created instance(s): ',
90+
runInstancesResponse.Instances?.map((i) => i.InstanceId).join(','),
91+
);
8892
const ssm = new SSM();
8993
runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => {
9094
await ssm

‎modules/runners/lambdas/runners/src/scale-runners/scale-down.ts

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,20 @@ import { listEC2Runners, RunnerInfo, RunnerList, terminateRunner } from './runne
44
import { getIdleRunnerCount, ScalingDownConfig } from './scale-down-config';
55
import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from './gh-auth';
66
import { githubCache, GhRunners } from './cache';
7-
import { logger as rootLogger } from './logger';
7+
import { logger as rootLogger, LogFields } from './logger';
88

9-
const logger = rootLogger.getChildLogger();
9+
const logger = rootLogger.getChildLogger({ name: 'scale-down' });
1010

1111
async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
1212
const key = runner.owner;
1313
const cachedOctokit = githubCache.clients.get(key);
1414

1515
if (cachedOctokit) {
16-
logger.debug(`[createGitHubClientForRunner] Cache hit for ${key}`);
16+
logger.debug(LogFields.fields, `[createGitHubClientForRunner] Cache hit for ${key}`);
1717
return cachedOctokit;
1818
}
1919

20-
logger.debug(`[createGitHubClientForRunner] Cache miss for ${key}`);
20+
logger.debug(LogFields.fields, `[createGitHubClientForRunner] Cache miss for ${key}`);
2121
const ghesBaseUrl = process.env.GHES_URL;
2222
let ghesApiUrl = '';
2323
if (ghesBaseUrl) {
@@ -50,11 +50,11 @@ async function listGitHubRunners(runner: RunnerInfo): Promise<GhRunners> {
5050
const key = runner.owner as string;
5151
const cachedRunners = githubCache.runners.get(key);
5252
if (cachedRunners) {
53-
logger.debug(`[listGithubRunners] Cache hit for ${key}`);
53+
logger.debug(LogFields.fields, `[listGithubRunners] Cache hit for ${key}`);
5454
return cachedRunners;
5555
}
5656

57-
logger.debug(`[listGithubRunners] Cache miss for ${key}`);
57+
logger.debug(LogFields.fields, `[listGithubRunners] Cache miss for ${key}`);
5858
const client = await getOrCreateOctokit(runner);
5959
const runners =
6060
runner.type === 'Org'
@@ -102,12 +102,18 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerId: number): Promise<
102102

103103
if (result.status == 204) {
104104
await terminateRunner(ec2runner.instanceId);
105-
logger.info(`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`);
105+
logger.info(
106+
LogFields.fields,
107+
`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`,
108+
);
106109
} else {
107-
logger.error(`Failed to de-register GitHub runner: ${result.status}`);
110+
logger.error(LogFields.fields, `Failed to de-register GitHub runner: ${result.status}`);
108111
}
109112
} catch (e) {
110-
logger.info(`Runner '${ec2runner.instanceId}' cannot be de-registered, most likely the runner is active.`);
113+
logger.info(
114+
LogFields.fields,
115+
`Runner '${ec2runner.instanceId}' cannot be de-registered, most likely the runner is active.`,
116+
);
111117
}
112118
}
113119

@@ -120,26 +126,29 @@ async function evaluateAndRemoveRunners(
120126

121127
for (const ownerTag of ownerTags) {
122128
const ec2RunnersFiltered = ec2Runners.filter((runner) => runner.owner === ownerTag);
123-
logger.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`);
129+
logger.debug(
130+
LogFields.fields,
131+
`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`,
132+
);
124133
for (const ec2Runner of ec2RunnersFiltered) {
125134
const ghRunners = await listGitHubRunners(ec2Runner);
126135
const ghRunner = ghRunners.find((runner) => runner.name === ec2Runner.instanceId);
127136
if (ghRunner) {
128137
if (runnerMinimumTimeExceeded(ec2Runner)) {
129138
if (idleCounter > 0) {
130139
idleCounter--;
131-
logger.info(`Runner '${ec2Runner.instanceId}' will kept idle.`);
140+
logger.info(LogFields.fields, `Runner '${ec2Runner.instanceId}' will kept idle.`);
132141
} else {
133-
logger.info(`Runner '${ec2Runner.instanceId}' will be terminated.`);
142+
logger.info(LogFields.fields, `Runner '${ec2Runner.instanceId}' will be terminated.`);
134143
await removeRunner(ec2Runner, ghRunner.id);
135144
}
136145
}
137146
} else {
138147
if (bootTimeExceeded(ec2Runner)) {
139-
logger.info(`Runner '${ec2Runner.instanceId}' is orphaned and will be removed.`);
148+
logger.info(LogFields.fields, `Runner '${ec2Runner.instanceId}' is orphaned and will be removed.`);
140149
terminateOrphan(ec2Runner.instanceId);
141150
} else {
142-
logger.debug(`Runner ${ec2Runner.instanceId} has not yet booted.`);
151+
logger.debug(LogFields.fields, `Runner ${ec2Runner.instanceId} has not yet booted.`);
143152
}
144153
}
145154
}

‎modules/runners/lambdas/runners/src/scale-runners/scale-up.ts

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import { listEC2Runners, createRunner, RunnerInputParameters } from './runners';
22
import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from './gh-auth';
33
import yn from 'yn';
44
import { Octokit } from '@octokit/rest';
5-
import { logger as rootLogger } from './logger';
5+
import { logger as rootLogger, LogFields } from './logger';
66

7-
const logger = rootLogger.getChildLogger();
7+
const logger = rootLogger.getChildLogger({ name: 'scale-up' });
88

99
export interface ActionRequestMessage {
1010
id: number;
@@ -23,7 +23,16 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
2323
const environment = process.env.ENVIRONMENT;
2424
const ghesBaseUrl = process.env.GHES_URL;
2525

26-
console.info(`Received ${payload.eventType} from ${payload.repositoryOwner}/${payload.repositoryName}`);
26+
const runnerType = enableOrgLevel ? 'Org' : 'Repo';
27+
const runnerOwner = enableOrgLevel ? payload.repositoryOwner : `${payload.repositoryOwner}/${payload.repositoryName}`;
28+
29+
LogFields.fields = {};
30+
LogFields.fields.runnerType = runnerType;
31+
LogFields.fields.runnerOwner = runnerOwner;
32+
LogFields.fields.event = payload.eventType;
33+
LogFields.fields.id = payload.id.toString();
34+
35+
console.info(LogFields.fields, `Received event`);
2736

2837
let ghesApiUrl = '';
2938
if (ghesBaseUrl) {
@@ -50,8 +59,6 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
5059

5160
const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl);
5261
const githubInstallationClient = await createOctoClient(ghAuth.token, ghesApiUrl);
53-
const runnerType = enableOrgLevel ? 'Org' : 'Repo';
54-
const runnerOwner = enableOrgLevel ? payload.repositoryOwner : `${payload.repositoryOwner}/${payload.repositoryName}`;
5562

5663
const isQueued = await getJobStatus(githubInstallationClient, payload);
5764
if (isQueued) {
@@ -60,7 +67,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
6067
runnerType,
6168
runnerOwner,
6269
});
63-
logger.info(`${runnerType} ${runnerOwner} has ${currentRunners.length}/${maximumRunners} runners`);
70+
logger.info(LogFields.fields, `Current runners: ${currentRunners.length} of ${maximumRunners}`);
6471

6572
if (currentRunners.length < maximumRunners) {
6673
console.info(`Attempting to launch a new runner`);
@@ -87,7 +94,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
8794
runnerType,
8895
});
8996
} else {
90-
logger.info('No runner will be created, maximum number of runners reached.');
97+
logger.info(LogFields.fields, 'No runner will be created, maximum number of runners reached.');
9198
}
9299
}
93100
}
@@ -112,7 +119,7 @@ async function getJobStatus(githubInstallationClient: Octokit, payload: ActionRe
112119
throw Error(`Event ${payload.eventType} is not supported`);
113120
}
114121
if (!isQueued) {
115-
logger.info(`Job ${payload.id} is not queued`);
122+
logger.info(LogFields.fields, `Job not queued`);
116123
}
117124
return isQueued;
118125
}
@@ -121,13 +128,13 @@ export async function createRunnerLoop(runnerParameters: RunnerInputParameters):
121128
const launchTemplateNames = process.env.LAUNCH_TEMPLATE_NAME?.split(',') as string[];
122129
let launched = false;
123130
for (let i = 0; i < launchTemplateNames.length; i++) {
124-
logger.info(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]}.`);
131+
logger.info(LogFields.fields, `Attempt '${i}' to launch instance using ${launchTemplateNames[i]}.`);
125132
try {
126133
await createRunner(runnerParameters, launchTemplateNames[i]);
127134
launched = true;
128135
break;
129136
} catch (error) {
130-
logger.debug(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]} FAILED.`);
137+
logger.debug(LogFields.fields, `Attempt '${i}' to launch instance using ${launchTemplateNames[i]} FAILED.`);
131138
logger.error(error);
132139
}
133140
}

0 commit comments

Comments
 (0)