feat(go): Golang Mod Builder (#65)

Author: volkangurel

aws_lambda_builders/workflows/__init__.py

@@ -6,3 +6,4 @@
 import aws_lambda_builders.workflows.nodejs_npm
 import aws_lambda_builders.workflows.ruby_bundler
 import aws_lambda_builders.workflows.go_dep
+import aws_lambda_builders.workflows.go_modules

aws_lambda_builders/workflows/go_modules/DESIGN.md

@@ -0,0 +1,38 @@
+## Go - Go Modules Lambda Builder
+
+### Scope
+
+This package leverages standard Go tooling available as of Go1.11 to build Go
+applications to be deployed in an AWS Lambda environment. The scope of this
+builder is to take an existing directory containing customer code, and a
+top-level `go.mod` file specifying third party dependencies. The builder will
+run `go build` on the project and put the resulting binary in the given
+artifacts folder.
+
+### Interface
+
+The top level interface is presented by the `GoModulesBuilder` class. There
+will be one public method `build`, which takes the provided arguments and
+builds a static binary using standard go tools.
+
+```python
+def build(self, source_dir_path, artifacts_dir_path, executable_name):
+    """Builds a go project onto an output path.
+
+    :type source_dir_path: str
+    :param source_dir_path: Directory with the source files.
+
+    :type output_path: str
+    :param output_path: Filename to write the executable output to.
+```
+
+### Implementation
+
+The general algorithm for preparing a Go package for use on AWS Lambda
+is very simple. It's as follows:
+
+Pass in GOOS=linux and GOARCH=amd64 to the `go build` command to target the
+OS and architecture used on AWS Lambda. Let go tooling handle the
+cross-compilation, regardless of the build environment. Move the resulting
+static binary to the artifacts folder to be shipped as a single-file zip
+archive.

aws_lambda_builders/workflows/go_modules/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds Go Lambda functions using standard Go tooling
+"""
+
+from .workflow import GoModulesWorkflow

aws_lambda_builders/workflows/go_modules/actions.py

@@ -0,0 +1,27 @@
+"""
+Action to build a Go project using standard Go tooling
+"""
+
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from .builder import BuilderError
+
+
+class GoModulesBuildAction(BaseAction):
+
+    NAME = "Build"
+    DESCRIPTION = "Building Go package with Go Modules"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    def __init__(self, source_dir, output_path, builder):
+        self.source_dir = source_dir
+        self.output_path = output_path
+        self.builder = builder
+
+    def execute(self):
+        try:
+            self.builder.build(
+                self.source_dir,
+                self.output_path,
+            )
+        except BuilderError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/go_modules/builder.py

@@ -0,0 +1,61 @@
+"""
+Build a Go project using standard Go tooling
+"""
+import logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class BuilderError(Exception):
+    MESSAGE = "Builder Failed: {message}"
+
+    def __init__(self, **kwargs):
+        Exception.__init__(self, self.MESSAGE.format(**kwargs))
+
+
+class GoModulesBuilder(object):
+
+    LANGUAGE = "go"
+
+    def __init__(self, osutils, binaries):
+        """Initialize a GoModulesBuilder.
+
+        :type osutils: :class:`lambda_builders.utils.OSUtils`
+        :param osutils: A class used for all interactions with the
+            outside OS.
+
+        :type binaries: dict
+        :param binaries: A dict of language binaries
+        """
+        self.osutils = osutils
+        self.binaries = binaries
+
+    def build(self, source_dir_path, output_path):
+        """Builds a go project onto an output path.
+
+        :type source_dir_path: str
+        :param source_dir_path: Directory with the source files.
+
+        :type output_path: str
+        :param output_path: Filename to write the executable output to.
+        """
+        env = {}
+        env.update(self.osutils.environ)
+        env.update({"GOOS": "linux", "GOARCH": "amd64"})
+        runtime_path = self.binaries[self.LANGUAGE].binary_path
+        cmd = [runtime_path, "build", "-o", output_path, source_dir_path]
+
+        p = self.osutils.popen(
+            cmd,
+            cwd=source_dir_path,
+            env=env,
+            stdout=self.osutils.pipe,
+            stderr=self.osutils.pipe,
+        )
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise BuilderError(message=err.decode("utf8").strip())
+
+        return out.decode("utf8").strip()

aws_lambda_builders/workflows/go_modules/utils.py

@@ -0,0 +1,27 @@
+"""
+Commonly used utilities
+"""
+
+import os
+import subprocess
+
+
+class OSUtils(object):
+    """
+    Wrapper around file system functions, to make it easy to
+    unit test actions in memory
+    """
+    @property
+    def environ(self):
+        return os.environ.copy()
+
+    def joinpath(self, *args):
+        return os.path.join(*args)
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+        return p
+
+    @property
+    def pipe(self):
+        return subprocess.PIPE

aws_lambda_builders/workflows/go_modules/validator.py

@@ -0,0 +1,68 @@
+"""
+Go Runtime Validation
+"""
+
+import logging
+import os
+import subprocess
+
+from aws_lambda_builders.exceptions import MisMatchRuntimeError
+
+LOG = logging.getLogger(__name__)
+
+
+class GoRuntimeValidator(object):
+
+    LANGUAGE = "go"
+    SUPPORTED_RUNTIMES = {
+        "go1.x"
+    }
+
+    def __init__(self, runtime):
+        self.runtime = runtime
+        self._valid_runtime_path = None
+
+    def has_runtime(self):
+        """
+        Checks if the runtime is supported.
+        :param string runtime: Runtime to check
+        :return bool: True, if the runtime is supported.
+        """
+        return self.runtime in self.SUPPORTED_RUNTIMES
+
+    def validate(self, runtime_path):
+        """
+        Checks if the language supplied matches the required lambda runtime
+        :param string runtime_path: runtime to check eg: /usr/bin/go
+        :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime
+        """
+        if not self.has_runtime():
+            LOG.warning("'%s' runtime is not "
+                        "a supported runtime", self.runtime)
+            return None
+
+        expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split('.')[0])
+        min_expected_minor_version = 11 if expected_major_version == 1 else 0
+
+        p = subprocess.Popen([runtime_path, "version"],
+                             cwd=os.getcwd(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, _ = p.communicate()
+
+        if p.returncode == 0:
+            out_parts = out.decode().split()
+            if len(out_parts) >= 3:
+                version_parts = [int(x) for x in out_parts[2].replace(self.LANGUAGE, "").split('.')]
+                if len(version_parts) == 3:
+                    if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version:
+                        self._valid_runtime_path = runtime_path
+                        return self._valid_runtime_path
+
+        # otherwise, raise mismatch exception
+        raise MisMatchRuntimeError(language=self.LANGUAGE,
+                                   required_runtime=self.runtime,
+                                   runtime_path=runtime_path)
+
+    @property
+    def validated_runtime_path(self):
+        return self._valid_runtime_path

aws_lambda_builders/workflows/go_modules/workflow.py

@@ -0,0 +1,51 @@
+"""
+Go Modules Workflow
+"""
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+
+from .actions import GoModulesBuildAction
+from .builder import GoModulesBuilder
+from .validator import GoRuntimeValidator
+from .utils import OSUtils
+
+
+class GoModulesWorkflow(BaseWorkflow):
+
+    NAME = "GoModulesBuilder"
+
+    CAPABILITY = Capability(language="go",
+                            dependency_manager="modules",
+                            application_framework=None)
+
+    def __init__(self,
+                 source_dir,
+                 artifacts_dir,
+                 scratch_dir,
+                 manifest_path,
+                 runtime=None,
+                 osutils=None,
+                 **kwargs):
+
+        super(GoModulesWorkflow, self).__init__(
+            source_dir,
+            artifacts_dir,
+            scratch_dir,
+            manifest_path,
+            runtime=runtime,
+            **kwargs)
+
+        if osutils is None:
+            osutils = OSUtils()
+
+        options = kwargs.get("options") or {}
+        handler = options.get("handler", None)
+
+        output_path = osutils.joinpath(artifacts_dir, handler)
+
+        builder = GoModulesBuilder(osutils, binaries=self.binaries)
+        self.actions = [
+            GoModulesBuildAction(source_dir, output_path, builder),
+        ]
+
+    def get_validators(self):
+        return [GoRuntimeValidator(runtime=self.runtime)]

aws_lambda_builders/workflows/python_pip/validator.py

@@ -70,4 +70,4 @@ def _validate_python_cmd(self, runtime_path):
 
     @property
     def validated_runtime_path(self):
-        return self._valid_runtime_path if self._valid_runtime_path is not None else None
+        return self._valid_runtime_path

tests/functional/workflows/go_modules/test_go_utils.py

@@ -0,0 +1,39 @@
+import os
+import sys
+
+from unittest import TestCase
+
+from aws_lambda_builders.workflows.go_modules import utils
+
+
+class TestOSUtils(TestCase):
+
+    def setUp(self):
+        self.osutils = utils.OSUtils()
+
+    def test_environ_returns_environment(self):
+        result = self.osutils.environ
+        self.assertEqual(result, os.environ)
+
+    def test_joinpath_joins_path_components(self):
+        result = self.osutils.joinpath('a', 'b', 'c')
+        self.assertEqual(result, os.path.join('a', 'b', 'c'))
+
+    def test_popen_runs_a_process_and_returns_outcome(self):
+        cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py')
+        p = self.osutils.popen([sys.executable, cwd_py],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe)
+        out, err = p.communicate()
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(out.decode('utf8').strip(), os.getcwd())
+
+    def test_popen_can_accept_cwd(self):
+        testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata')
+        p = self.osutils.popen([sys.executable, 'cwd.py'],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=testdata_dir)
+        out, err = p.communicate()
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir))

tests/integration/workflows/go_modules/test_go.py

@@ -0,0 +1,59 @@
+import os
+import shutil
+import tempfile
+
+from unittest import TestCase
+
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowFailedError
+
+
+class TestGoWorkflow(TestCase):
+    """
+    Verifies that `go` workflow works by building a Lambda using Go Modules
+    """
+
+    TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+
+    def setUp(self):
+        self.artifacts_dir = tempfile.mkdtemp()
+        self.scratch_dir = tempfile.mkdtemp()
+        self.builder = LambdaBuilder(language="go",
+                                     dependency_manager="modules",
+                                     application_framework=None)
+        self.runtime = "go1.x"
+
+    def tearDown(self):
+        shutil.rmtree(self.artifacts_dir)
+        shutil.rmtree(self.scratch_dir)
+
+    def test_builds_project_without_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "go.mod"),
+                           runtime=self.runtime,
+                           options={"handler": "no-deps-main"})
+        expected_files = {"no-deps-main"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        print(output_files)
+        self.assertEquals(expected_files, output_files)
+
+    def test_builds_project_with_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps")
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "go.mod"),
+                           runtime=self.runtime,
+                           options={"handler": "with-deps-main"})
+        expected_files = {"with-deps-main"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+    def test_fails_if_modules_cannot_resolve_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "go.mod"),
+                               runtime=self.runtime,
+                               options={"handler": "failed"})
+        self.assertIn("GoModulesBuilder:Build - Builder Failed: ",
+                      str(ctx.exception))