add webhook (#2)

* Add tf code for gateway / lambda

* Remove gitkeep

* Add sqs

* Add queue and polcies

* Add readme

* Fix some errors

* Send events to sqs

* Add test lambda

* Rename sqs url parameter

* Cleanup

* Add build dist command

* Rework lambda a bit and add tests

* Add ci for webhook

* Update descriptions

* Update path expression

* Try working dir

* Add terraform checks to ci

* Fix validate

* Build only on PR

Co-authored-by: Gertjan Maas <[email protected]>

Author: npalm

.github/workflows/lambda-agent-webhook.yml

@@ -0,0 +1,26 @@
+name: Lambda Agent Webhook
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - .github/workflows/lambda-agent-webhook.yml
+      - "modules/agent/lambdas/webhook/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: node:12
+    defaults:
+      run:
+        working-directory: modules/agent/lambdas/webhook
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build distribution
+        run: yarn build

.github/workflows/terraform.yml

@@ -0,0 +1,47 @@
+name: "Terraform root module checks"
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths-ignore:
+      - "modules/*/lambdas/**"
+
+env:
+  tf_version: "0.12.24"
+  tf_working_dir: "."
+  AWS_REGION: eu-west-1
+jobs:
+  terraform:
+    name: "Terraform"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+      - name: "Terraform Format"
+        uses: hashicorp/terraform-github-actions@master
+        with:
+          tf_actions_version: ${{ env.tf_version }}
+          tf_actions_subcommand: "fmt"
+          tf_actions_working_dir: ${{ env.tf_working_dir }}
+          tf_actions_comment: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Terraform Init"
+        uses: hashicorp/terraform-github-actions@master
+        with:
+          tf_actions_version: ${{ env.tf_version }}
+          tf_actions_subcommand: "init"
+          tf_actions_working_dir: ${{ env.tf_working_dir }}
+          tf_actions_comment: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Terraform Validate"
+        uses: hashicorp/terraform-github-actions@master
+        with:
+          tf_actions_version: ${{ env.tf_version }}
+          tf_actions_subcommand: "validate"
+          tf_actions_working_dir: ${{ env.tf_working_dir }}
+          tf_actions_comment: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

modules/agent/README.md

@@ -0,0 +1,6 @@
+# Agent for orchestration of action runners
+
+Agent to orchestrate the the action runners are composed of:
+- API Gatewway and lambda to receive GitHub events
+- SQS queue for decouple web hook to orchestrator
+- Lambda to create EC2 action runner instances based queue events and limits.

modules/agent/lambdas/webhook/.gitignore

@@ -0,0 +1,15 @@
+# dependencies
+node_modules/
+
+# production
+dist/
+build/
+
+# misc
+.DS_Store
+.env*
+*.zip
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

modules/agent/lambdas/webhook/.nvmrc

@@ -0,0 +1 @@
+v12.16.1

modules/agent/lambdas/webhook/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "singleQuote": true,
+  "trailingComma": "all"
+}

modules/agent/lambdas/webhook/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

modules/agent/lambdas/webhook/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "github-runner-lambda-agent-webhook",
+  "version": "1.0.0",
+  "main": "lambda.ts",
+  "license": "MIT",
+  "scripts": {
+    "start": "ts-node-dev src/local.ts",
+    "test": "NODE_ENV=test jest",
+    "watch": "ts-node-dev --respawn --exit-child src/local.ts",
+    "build": "ncc build src/lambda.ts -o dist",
+    "dist": "yarn build && cd dist && zip ../webhook.zip index.js"
+  },
+  "devDependencies": {
+    "@octokit/webhooks": "^7.4.0",
+    "@types/express": "^4.17.3",
+    "@types/jest": "^25.2.1",
+    "@types/node": "^13.13.4",
+    "@zeit/ncc": "^0.22.1",
+    "aws-sdk": "^2.645.0",
+    "body-parser": "^1.19.0",
+    "express": "^4.17.1",
+    "jest": "^25.4.0",
+    "ts-jest": "^25.4.0",
+    "ts-node-dev": "^1.0.0-pre.44",
+    "typescript": "^3.8.3"
+  },
+  "dependencies": {
+    "crypto": "^1.0.1"
+  }
+}

modules/agent/lambdas/webhook/src/lambda.ts

@@ -0,0 +1,8 @@
+import { handle as githubWebhook } from './webhook/handler';
+
+module.exports.githubWebhook = async (event: any, context: any, callback: any) => {
+  const statusCode = await githubWebhook(event.headers, event.body);
+  return callback(null, {
+    statusCode: statusCode,
+  });
+};

modules/agent/lambdas/webhook/src/local.ts

@@ -0,0 +1,20 @@
+import express from 'express';
+import bodyParser from 'body-parser';
+import { handle } from './webhook/handler';
+
+const app = express();
+
+app.use(bodyParser.json());
+
+app.post('/event_handler', (req, res) => {
+  handle(req.headers, JSON.stringify(req.body))
+    .then((c) => res.status(c).end())
+    .catch((e) => {
+      console.log(e);
+      res.status(404);
+    });
+});
+
+app.listen(3000, (): void => {
+  console.log('webhook app listening on port 3000!');
+});

modules/agent/lambdas/webhook/src/sqs/index.ts

@@ -0,0 +1,26 @@
+import { SQS } from 'aws-sdk';
+import AWS from 'aws-sdk';
+
+AWS.config.update({
+  region: process.env.AWS_REGION,
+});
+
+const sqs = new SQS();
+
+export interface ActionRequestMessage {
+  id: number;
+  eventType: string;
+  repositoryName: string;
+  repositoryOwner: string;
+  installationId: number;
+}
+
+export const sendActionRequest = async (message: ActionRequestMessage) => {
+  await sqs
+    .sendMessage({
+      QueueUrl: String(process.env.SQS_URL_WEBHOOK),
+      MessageBody: JSON.stringify(message),
+      MessageGroupId: String(message.id),
+    })
+    .promise();
+};

modules/agent/lambdas/webhook/src/webhook/handler.test.ts

@@ -0,0 +1,69 @@
+import { handle } from './handler';
+import check_run_event from '../../test/resources/github_check_run_event.json';
+
+import { sendActionRequest } from '../sqs';
+
+jest.mock('../sqs');
+
+describe('handler', () => {
+  let originalError: Console['error'];
+
+  beforeEach(() => {
+    process.env.GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET';
+    originalError = console.error;
+    console.error = jest.fn();
+    jest.clearAllMocks();
+  });
+
+  afterEach(() => {
+    console.error = originalError;
+  });
+
+  it('returns 500 if no signature available', async () => {
+    const resp = await handle({}, '');
+    expect(resp).toBe(500);
+  });
+
+  it('returns 401 if signature is invalid', async () => {
+    const resp = await handle({ 'X-Hub-Signature': 'bbb' }, 'aaaa');
+    expect(resp).toBe(401);
+  });
+
+  it('handles check_run events', async () => {
+    const resp = await handle(
+      { 'X-Hub-Signature': 'sha1=4a82d2f60346e16dab3546eb3b56d8dde4d5b659', 'X-GitHub-Event': 'check_run' },
+      JSON.stringify(check_run_event),
+    );
+    expect(resp).toBe(200);
+    expect(sendActionRequest).toBeCalled();
+  });
+
+  it('does not handle other events', async () => {
+    const resp = await handle(
+      { 'X-Hub-Signature': 'sha1=4a82d2f60346e16dab3546eb3b56d8dde4d5b659', 'X-GitHub-Event': 'push' },
+      JSON.stringify(check_run_event),
+    );
+    expect(resp).toBe(200);
+    expect(sendActionRequest).not.toBeCalled();
+  });
+
+  it('does not handle check_run events with actions other than created', async () => {
+    const event = { ...check_run_event, action: 'completed' };
+    const resp = await handle(
+      { 'X-Hub-Signature': 'sha1=891749859807857017f7ee56a429e8fcead6f3e1', 'X-GitHub-Event': 'push' },
+      JSON.stringify(event),
+    );
+    expect(resp).toBe(200);
+    expect(sendActionRequest).not.toBeCalled();
+  });
+
+  it('does not handle check_run events with status other than queued', async () => {
+    const event = { ...check_run_event, check_run: { id: 1234, status: 'completed' } };
+    const resp = await handle(
+      { 'X-Hub-Signature': 'sha1=73dfae4aa56de5b038af8921b40d7a412ce7ca19', 'X-GitHub-Event': 'push' },
+      JSON.stringify(event),
+    );
+    expect(resp).toBe(200);
+    expect(sendActionRequest).not.toBeCalled();
+  });
+});

modules/agent/lambdas/webhook/src/webhook/handler.ts

@@ -0,0 +1,49 @@
+import { IncomingHttpHeaders } from 'http';
+import crypto from 'crypto';
+import { sendActionRequest } from '../sqs';
+import { WebhookPayloadCheckRun } from '@octokit/webhooks';
+
+function signRequestBody(key: string, body: any) {
+  return `sha1=${crypto.createHmac('sha1', key).update(body, 'utf8').digest('hex')}`;
+}
+
+export const handle = async (headers: IncomingHttpHeaders, payload: any): Promise<number> => {
+  // ensure header keys lower case since github headers can contain capitals.
+  for (const key in headers) {
+    headers[key.toLowerCase()] = headers[key];
+  }
+
+  const secret = process.env.GITHUB_APP_WEBHOOK_SECRET as string;
+  const signature = headers['x-hub-signature'];
+  if (!signature) {
+    console.error("Github event doesn't have signature. This webhook requires a secret to be configured.");
+    return 500;
+  }
+
+  const calculatedSig = signRequestBody(secret, payload);
+  if (signature !== calculatedSig) {
+    console.error('Unable to verify signature!');
+    return 401;
+  }
+
+  const githubEvent = headers['x-github-event'];
+
+  console.debug(`Received Github event: "${githubEvent}"`);
+
+  if (githubEvent === 'check_run') {
+    const body = JSON.parse(payload) as WebhookPayloadCheckRun;
+    if (body.action === 'created' && body.check_run.status === 'queued') {
+      await sendActionRequest({
+        id: body.check_run.id,
+        repositoryName: body.repository.name,
+        repositoryOwner: body.repository.owner.login,
+        eventType: githubEvent,
+        installationId: body.installation!.id,
+      });
+    }
+  } else {
+    console.debug('Ignore event ' + githubEvent);
+  }
+
+  return 200;
+};