Add the Go Modules builder

Add a builder for Go that assumes the given go project uses Go Modules, and so
can be built using standard go tooling.

Author: volkangurel

aws_lambda_builders/actions.py

@@ -92,10 +92,16 @@ class CopySourceAction(BaseAction):
 
     PURPOSE = Purpose.COPY_SOURCE
 
-    def __init__(self, source_dir, dest_dir, excludes=None):
+    def __init__(self, source_dir, dest_dir, only=None, excludes=None):
         self.source_dir = source_dir
         self.dest_dir = dest_dir
+        self.only = only
         self.excludes = excludes or []
 
     def execute(self):
-        copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))
+        if self.only:
+            def ignore(source, names):
+                return [name for name in names if name not in self.only]
+        else:
+            ignore = shutil.ignore_patterns(*self.excludes)
+        copytree(self.source_dir, self.dest_dir, ignore=ignore)

aws_lambda_builders/workflows/__init__.py

@@ -6,3 +6,4 @@
 import aws_lambda_builders.workflows.nodejs_npm
 import aws_lambda_builders.workflows.ruby_bundler
 import aws_lambda_builders.workflows.go_dep
+import aws_lambda_builders.workflows.go_modules

aws_lambda_builders/workflows/go_modules/DESIGN.md

@@ -0,0 +1,53 @@
+## Go - Go Modules Lambda Builder
+
+### Scope
+
+This package leverages standard Go tooling available as of Go1.11 to build Go
+applications to be deployed in an AWS Lambda environment. The scope of this
+builder is to take an existing directory containing customer code, and a
+top-level `go.mod` file specifying third party dependencies. The builder will
+run `go build` on the project and put the resulting binary in the given
+artifacts folder.
+
+### Interface
+
+The top level interface is presented by the `GoModulesBuilder` class. There
+will be one public method `build`, which takes the provided arguments and
+builds a static binary using standard go tools.
+
+```python
+def build(self, source_dir_path, artifacts_dir_path, ui=None, config=None):
+    """Builds a go project into an artifact directory.
+
+    :type source_dir_path: str
+    :param source_dir_path: Directory with the source files.
+
+    :type artifacts_dir_path: str
+    :param artifacts_dir_path: Directory to write dependencies into.
+
+    :type executable_name: str
+    :param executable_name: Name of the executable to create from the build.
+
+    :type ui: :class:`lambda_builders.utils.UI` or None
+    :param ui: A class that traps all progress information such as status
+        and errors. If injected by the caller, it can be used to monitor
+        the status of the build process or forward this information
+        elsewhere.
+
+    :type config: :class:`lambda_builders.utils.Config` or None
+    :param config: To be determined. This is an optional config object
+        we can extend at a later date to add more options to how modules is
+        called.
+    """
+```
+
+### Implementation
+
+The general algorithm for preparing a Go package for use on AWS Lambda
+is very simple. It's as follows:
+
+Pass in GOOS=linux and GOARCH=amd64 to the `go build` command to target the
+OS and architecture used on AWS Lambda. Let go tooling handle the
+cross-compilation, regardless of the build environment. Move the resulting
+static binary to the artifacts folder to be shipped as a single-file zip
+archive.

aws_lambda_builders/workflows/go_modules/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds Go Lambda functions using standard Go tooling
+"""
+
+from .workflow import GoModulesWorkflow

aws_lambda_builders/workflows/go_modules/actions.py

@@ -0,0 +1,29 @@
+"""
+Action to build a Go project using standard Go tooling
+"""
+
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from .builder import BuilderError
+
+
+class GoModulesBuildAction(BaseAction):
+
+    NAME = "Build"
+    DESCRIPTION = "Building Go package with Go Modules"
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+
+    def __init__(self, source_dir, artifacts_dir, executable_name, builder):
+        self.source_dir = source_dir
+        self.artifacts_dir = artifacts_dir
+        self.executable_name = executable_name
+        self.builder = builder
+
+    def execute(self):
+        try:
+            self.builder.build(
+                self.source_dir,
+                self.artifacts_dir,
+                self.executable_name
+            )
+        except BuilderError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/go_modules/builder.py

@@ -0,0 +1,67 @@
+"""
+Build a Go project using standard Go tooling
+"""
+import logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class BuilderError(Exception):
+    MESSAGE = "Builder Failed: {message}"
+
+    def __init__(self, **kwargs):
+        Exception.__init__(self, self.MESSAGE.format(**kwargs))
+
+
+class GoModulesBuilder(object):
+    def __init__(self, osutils):
+        """Initialize a GoModulesBuilder.
+
+        :type osutils: :class:`lambda_builders.utils.OSUtils`
+        :param osutils: A class used for all interactions with the
+            outside OS.
+        """
+        self.osutils = osutils
+
+    def build(self, source_dir_path, artifacts_dir_path, executable_name):
+        """Builds a go project into an artifact directory.
+
+        :type source_dir_path: str
+        :param source_dir_path: Directory with the source files.
+
+        :type artifacts_dir_path: str
+        :param artifacts_dir_path: Directory to write dependencies into.
+
+        :type executable_name: str
+        :param executable_name: Name of the executable to create from the build.
+
+        :type ui: :class:`lambda_builders.utils.UI` or None
+        :param ui: A class that traps all progress information such as status
+            and errors. If injected by the caller, it can be used to monitor
+            the status of the build process or forward this information
+            elsewhere.
+
+        :type config: :class:`lambda_builders.utils.Config` or None
+        :param config: To be determined. This is an optional config object
+            we can extend at a later date to add more options to how modules is
+            called.
+        """
+        env = {}
+        env.update(self.osutils.environ)
+        env.update({"GOOS": "linux", "GOARCH": "amd64"})
+        cmd = ["go", "build", "-o", self.osutils.joinpath(artifacts_dir_path, executable_name), source_dir_path]
+
+        p = self.osutils.popen(
+            cmd,
+            cwd=source_dir_path,
+            env=env,
+            stdout=self.osutils.pipe,
+            stderr=self.osutils.pipe,
+        )
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise BuilderError(message=err.decode("utf8").strip())
+
+        return out.decode("utf8").strip()

aws_lambda_builders/workflows/go_modules/utils.py

@@ -0,0 +1,30 @@
+"""
+Commonly used utilities
+"""
+
+import os
+import subprocess
+
+
+class OSUtils(object):
+    """
+    Wrapper around file system functions, to make it easy to
+    unit test actions in memory
+    """
+    @property
+    def environ(self):
+        return os.environ
+
+    def joinpath(self, *args):
+        return os.path.join(*args)
+
+    def basename(self, *args):
+        return os.path.basename(*args)
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+        return p
+
+    @property
+    def pipe(self):
+        return subprocess.PIPE

aws_lambda_builders/workflows/go_modules/workflow.py

@@ -0,0 +1,45 @@
+"""
+Go Modules Workflow
+"""
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+from aws_lambda_builders.actions import CopySourceAction
+
+from .actions import GoModulesBuildAction
+from .builder import GoModulesBuilder
+from .utils import OSUtils
+
+
+class GoModulesWorkflow(BaseWorkflow):
+
+    NAME = "GoModulesBuilder"
+
+    CAPABILITY = Capability(language="go",
+                            dependency_manager="modules",
+                            application_framework=None)
+
+    def __init__(self,
+                 source_dir,
+                 artifacts_dir,
+                 scratch_dir,
+                 manifest_path,
+                 runtime=None,
+                 osutils=None,
+                 **kwargs):
+
+        super(GoModulesWorkflow, self).__init__(
+            source_dir,
+            artifacts_dir,
+            scratch_dir,
+            manifest_path,
+            runtime=runtime,
+            **kwargs)
+
+        if osutils is None:
+            osutils = OSUtils()
+
+        executable_name = osutils.basename(source_dir)
+        builder = GoModulesBuilder(osutils)
+        self.actions = [
+            GoModulesBuildAction(source_dir, artifacts_dir, executable_name, builder),
+            CopySourceAction(source_dir, artifacts_dir, only=[executable_name]),
+        ]

tests/functional/workflows/go_modules/test_go_utils.py

@@ -0,0 +1,43 @@
+import os
+import sys
+
+from unittest import TestCase
+
+from aws_lambda_builders.workflows.go_modules import utils
+
+
+class TestOSUtils(TestCase):
+
+    def setUp(self):
+        self.osutils = utils.OSUtils()
+
+    def test_environ_returns_environment(self):
+        result = self.osutils.environ
+        self.assertEqual(result, os.environ)
+
+    def test_joinpath_joins_path_components(self):
+        result = self.osutils.joinpath('a', 'b', 'c')
+        self.assertEqual(result, os.path.join('a', 'b', 'c'))
+
+    def test_basename_returns_path_basename(self):
+        result = self.osutils.basename(os.path.dirname(__file__))
+        self.assertEqual(result, 'go_modules')
+
+    def test_popen_runs_a_process_and_returns_outcome(self):
+        cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py')
+        p = self.osutils.popen([sys.executable, cwd_py],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe)
+        out, err = p.communicate()
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(out.decode('utf8').strip(), os.getcwd())
+
+    def test_popen_can_accept_cwd(self):
+        testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata')
+        p = self.osutils.popen([sys.executable, 'cwd.py'],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=testdata_dir)
+        out, err = p.communicate()
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir))

tests/integration/workflows/go_modules/test_go.py

@@ -0,0 +1,52 @@
+import os
+import shutil
+import tempfile
+
+from unittest import TestCase
+
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowFailedError
+
+
+class TestGoWorkflow(TestCase):
+    """
+    Verifies that `go` workflow works by building a Lambda using Go Modules
+    """
+
+    TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+
+    def setUp(self):
+        self.artifacts_dir = tempfile.mkdtemp()
+        self.scratch_dir = tempfile.mkdtemp()
+        self.builder = LambdaBuilder(language="go",
+                                     dependency_manager="modules",
+                                     application_framework=None)
+
+    def tearDown(self):
+        shutil.rmtree(self.artifacts_dir)
+        shutil.rmtree(self.scratch_dir)
+
+    def test_builds_project_without_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "go.mod"))
+        expected_files = {"no-deps"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        print(output_files)
+        self.assertEquals(expected_files, output_files)
+
+    def test_builds_project_with_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps")
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "go.mod"))
+        expected_files = {"with-deps"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+    def test_fails_if_modules_cannot_resolve_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "go.mod"))
+        self.assertIn("GoModulesBuilder:Build - Builder Failed: ",
+                      str(ctx.exception))

tests/integration/workflows/go_modules/testdata/broken-deps/go.mod

@@ -0,0 +1,12 @@
+module github.com/awslabs/aws-lambda-builders
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/aws/aws-lambda-go v0.9999.0 // doesn't exist, broken dependency
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	github.com/stretchr/testify v1.2.2 // indirect
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+	gopkg.in/yaml.v2 v2.2.2 // indirect
+)

tests/integration/workflows/go_modules/testdata/broken-deps/main.go

@@ -0,0 +1,4 @@
+package main
+
+func main() {
+}

tests/integration/workflows/go_modules/testdata/no-deps/go.mod

@@ -0,0 +1 @@
+module github.com/awslabs/aws-lambda-builders

tests/integration/workflows/go_modules/testdata/no-deps/main.go

@@ -0,0 +1,4 @@
+package main
+
+func main() {
+}

tests/integration/workflows/go_modules/testdata/with-deps/go.mod

@@ -0,0 +1,12 @@
+module github.com/awslabs/aws-lambda-builders
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/aws/aws-lambda-go v1.8.0
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	github.com/stretchr/testify v1.2.2 // indirect
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+	gopkg.in/yaml.v2 v2.2.2 // indirect
+)