sync GitHub action distribution to S3 (#5)

* WIP lambda to sync s3 dist

* Add update check

* Add syncer

* Refactor

* Refactor to functions

* Fix terraform validate

* Fix error

* Add linter, clenup

* Add modules to root modules

* Update modules/action-runner-binary-cache/variables.tf

Co-authored-by: Gertjan Maas <[email protected]>

* Fix review comments

* Update jabascript libs

* Add aws tags

* Update default example

Co-authored-by: Gertjan Maas <[email protected]>

Author: npalm

.github/workflows/lambda-syncer.yml

@@ -0,0 +1,26 @@
+name: Lambda Runner Distribution Syncer
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - .github/workflows/lambda-agent-webhook.yml
+      - "modules/action-runner-binary-cache/lambdas/syncer/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: node:12
+    defaults:
+      run:
+        working-directory: modules/action-runner-binary-cache/lambdas/syncer
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: yarn install
+      - name: Run linter
+        run: yarn lint
+      - name: Build distribution
+        run: yarn build

.github/workflows/terraform.yml

@@ -18,6 +18,10 @@ jobs:
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
+      - name: "Fake zip files" # Validate will fail if it cannot find the zip files
+        run: |
+          touch modules/action-runner-binary-cache/lambdas/syncer/syncer.zip
+          touch modules/agent/lambdas/webhook/webhook.zip
       - name: "Terraform Format"
         uses: hashicorp/terraform-github-actions@master
         with:

.gitignore

@@ -13,3 +13,6 @@
 *.out
 example/*.secrets*.tfvars
 .envrc
+*.zip
+*.gz
+*.tgz

examples/default/main.tf

@@ -1,8 +1,14 @@
 locals {
-  environment = "default-action-runners"
+  environment = "default"
   aws_region  = "eu-west-1"
 }
 
+
+resource "random_password" "random" {
+  length = 32
+}
+
+
 module "runners" {
   source = "../../"
 
@@ -14,5 +20,8 @@ module "runners" {
     Project = "ProjectX"
   }
 
+  github_app_webhook_secret = random_password.random.result
+
 }
 
+

examples/default/outputs.tf

@@ -3,3 +3,13 @@ output "action_runners" {
     runners = module.runners.runners
   }
 }
+
+
+output "lambda_syncer_function_name" {
+  value = module.runners.lambda_s3_action_runner_dist_syncer.id
+}
+
+
+output "github_app_webhook_secret" {
+  value = random_password.random.result
+}

main.tf

@@ -1,3 +1,9 @@
+locals {
+  tags = merge(var.tags, {
+    Environment = var.environment
+  })
+
+}
 resource "random_string" "random" {
   length  = 24
   special = false
@@ -9,7 +15,7 @@ module "dsitrubtion_cache" {
 
   aws_region  = var.aws_region
   environment = var.environment
-  tags        = var.tags
+  tags        = local.tags
 
   distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
 }
@@ -20,12 +26,23 @@ module "runners" {
   aws_region  = var.aws_region
   vpc_id      = var.vpc_id
   environment = var.environment
-  tags        = var.tags
+  tags        = local.tags
 
   s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
 }
 
 
+module "agent" {
+  source = "./modules/agent"
+
+  aws_region  = var.aws_region
+  environment = var.environment
+  tags        = local.tags
+
+  github_app_webhook_secret = var.github_app_webhook_secret
+}
+
+
 resource "aws_iam_policy" "dist_bucket" {
   name        = "${var.environment}-gh-distribution-bucket"
   path        = "/"

modules/action-runner-binary-cache/lambdas/syncer/.eslintrc.js

@@ -0,0 +1,10 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/eslint-recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+};

modules/action-runner-binary-cache/lambdas/syncer/.gitignore

@@ -0,0 +1,15 @@
+# dependencies
+node_modules/
+
+# production
+dist/
+build/
+
+# misc
+.DS_Store
+.env*
+*.zip
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

modules/action-runner-binary-cache/lambdas/syncer/.nvmrc

@@ -0,0 +1 @@
+v12.16.1

modules/action-runner-binary-cache/lambdas/syncer/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "singleQuote": true,
+  "trailingComma": "all"
+}

modules/action-runner-binary-cache/lambdas/syncer/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

modules/action-runner-binary-cache/lambdas/syncer/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "github-runner-lambda-syncer",
+  "version": "1.0.0",
+  "main": "lambda.ts",
+  "license": "MIT",
+  "scripts": {
+    "start": "ts-node-dev src/local.ts",
+    "test": "NODE_ENV=test jest",
+    "test:watch": "NODE_ENV=test jest --watch",
+    "lint": "yarn eslint --ext ts,tsx src",
+    "watch": "ts-node-dev --respawn --exit-child src/local.ts",
+    "build": "ncc build src/lambda.ts -o dist",
+    "dist": "yarn build && cd dist && zip ../syncer.zip index.js"
+  },
+  "devDependencies": {
+    "@octokit/rest": "^17.6.0",
+    "@types/jest": "^25.2.1",
+    "@types/node": "^13.13.4",
+    "@types/request": "^2.48.4",
+    "@typescript-eslint/eslint-plugin": "^2.30.0",
+    "@typescript-eslint/parser": "^2.30.0",
+    "@zeit/ncc": "^0.22.1",
+    "aws-sdk": "^2.645.0",
+    "eslint": "^6.8.0",
+    "jest": "^25.4.0",
+    "ts-jest": "^25.4.0",
+    "ts-node-dev": "^1.0.0-pre.44",
+    "typescript": "^3.8.3"
+  }
+}

modules/action-runner-binary-cache/lambdas/syncer/src/lambda.ts

@@ -0,0 +1,6 @@
+import { handle } from './syncer/handler';
+
+module.exports.handler = async (event: any, context: any, callback: any): Promise<any> => {
+  await handle();
+  return callback();
+};

modules/action-runner-binary-cache/lambdas/syncer/src/local.ts

@@ -0,0 +1,7 @@
+import { handle } from './syncer/handler';
+
+handle()
+  .then()
+  .catch((e) => {
+    console.log(e);
+  });

modules/action-runner-binary-cache/lambdas/syncer/src/syncer/handler.ts

@@ -0,0 +1,92 @@
+import { Octokit } from '@octokit/rest';
+import { PassThrough } from 'stream';
+import request from 'request';
+import { S3 } from 'aws-sdk';
+import AWS from 'aws-sdk';
+
+AWS.config.update({
+  region: process.env.AWS_REGION,
+});
+const s3 = new S3();
+
+const versionKey = 'name';
+const bucketName = process.env.S3_BUCKET_NAME as string;
+const bucketObjectKey = process.env.S3_OBJECT_KEY as string;
+if (!bucketName || !bucketObjectKey) {
+  throw new Error('Please check all mandatory variables are set.');
+}
+
+async function getCachedVersion(): Promise<string | undefined> {
+  try {
+    const objectTagging = await s3
+      .getObjectTagging({
+        Bucket: bucketName,
+        Key: bucketObjectKey,
+      })
+      .promise();
+    const versions = objectTagging.TagSet?.filter((t: S3.Tag) => t.Key === versionKey);
+    return versions.length === 1 ? versions[0].Value : undefined;
+  } catch (e) {
+    console.debug('No tags found');
+    return undefined;
+  }
+}
+
+interface ReleaseAsset {
+  name: string;
+  downloadUrl: string;
+}
+
+async function getLinuxReleaseAsset(): Promise<ReleaseAsset | undefined> {
+  const githubClient = new Octokit();
+  const linuxAssets = (
+    await githubClient.repos.getLatestRelease({
+      owner: 'actions',
+      repo: 'runner',
+    })
+  ).data.assets.filter((a) => a.name?.includes('actions-runner-linux-x64-'));
+  return linuxAssets?.length === 1
+    ? { name: linuxAssets[0].name, downloadUrl: linuxAssets[0].browser_download_url }
+    : undefined;
+}
+
+async function uploadToS3(actionRunnerReleaseAsset: ReleaseAsset): Promise<void> {
+  const writeStream = new PassThrough();
+  s3.upload({
+    Bucket: bucketName,
+    Key: bucketObjectKey,
+    Tagging: versionKey + '=' + actionRunnerReleaseAsset.name,
+    Body: writeStream,
+  }).promise();
+
+  await new Promise((resolve, reject) => {
+    console.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
+    request
+      .get(actionRunnerReleaseAsset.downloadUrl)
+      .pipe(writeStream)
+      .on('finish', () => {
+        console.info(`The new distribution is uploaded to S3.`);
+        resolve();
+      })
+      .on('error', (error) => {
+        reject(error);
+      });
+  }).catch((error) => {
+    console.error(`Exception: ${error}`);
+  });
+}
+
+export const handle = async (): Promise<void> => {
+  const actionRunnerReleaseAsset = await getLinuxReleaseAsset();
+  if (actionRunnerReleaseAsset === undefined) {
+    throw Error('Cannot find github release asset.');
+  }
+
+  const currentVersion = await getCachedVersion();
+  console.log('latest: ' + currentVersion);
+  if (currentVersion === undefined || currentVersion != actionRunnerReleaseAsset.name) {
+    uploadToS3(actionRunnerReleaseAsset);
+  } else {
+    console.debug('Distribution is up-to-date, no action.');
+  }
+};

modules/action-runner-binary-cache/lambdas/syncer/tsconfig.json

@@ -0,0 +1,60 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    "target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
+    "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
+    "outDir": "build",
+    "lib": ["es2020", "DOM"] /* Specify library files to be included in the compilation. */,
+    "allowJs": true /* Allow javascript files to be compiled. */,
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    // "outDir": "./",                        /* Redirect output structure to the directory. */
+    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "incremental": true,                   /* Enable incremental compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    "downlevelIteration": true /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */,
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+    /* Strict Type-Checking Options */
+    "strict": true /* Enable all strict type-checking options. */,
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+    /* Module Resolution Options */
+    "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */,
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [] /* List of folders to include type definitions from. */,
+    // "types": [] /* Type declaration files to be included in compilation. */,
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+    /* Experimental Options */
+    "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */,
+    "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"]
+}