feat(go): Golang Dep Builder (#54)

Author: sthulb

.appveyor.yml

@@ -2,6 +2,8 @@ version: 1.0.{build}
 image: Visual Studio 2017
 
 environment:
+  GOPATH: c:\gopath
+  GOVERSION: 1.11
 
   matrix:
 
@@ -24,6 +26,15 @@ install:
 - "gem install bundler -v 1.17.3 --no-ri --no-rdoc"
 - "bundler --version"
 
+# setup go
+- rmdir c:\go /s /q
+- "choco install golang"
+- "choco install bzr"
+- "choco install dep"
+- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
+- "go version"
+- "go env"
+
 test_script:
 - "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
 - "%PYTHON%\\python.exe -m pytest tests/integration"

.gitignore

@@ -383,4 +383,6 @@ $RECYCLE.BIN/
 
 /Dockerfile
 
-# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode
+tests/integration/workflows/go_dep/data/src/*/vendor/*
+
+# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode

.travis.yml

@@ -16,6 +16,12 @@ install:
   - nvm install 8.10.0
   - nvm use 8.10.0
 
+  # Go workflow integ tests require Go 1.11+
+  - eval "$(gimme 1.11.2)"
+  - go version
+
+  - go get -u github.com/golang/dep/cmd/dep
+
   # Install the code requirements
   - make init
 script:

aws_lambda_builders/workflows/__init__.py

@@ -5,3 +5,4 @@
 import aws_lambda_builders.workflows.python_pip
 import aws_lambda_builders.workflows.nodejs_npm
 import aws_lambda_builders.workflows.ruby_bundler
+import aws_lambda_builders.workflows.go_dep

aws_lambda_builders/workflows/go_dep/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds Go Lambda functions using the `dep` dependency manager
+"""
+
+from .workflow import GoDepWorkflow

aws_lambda_builders/workflows/go_dep/actions.py

@@ -0,0 +1,66 @@
+"""
+Actions for Go dependency resolution with dep
+"""
+
+import logging
+import os
+
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+
+from .subproc_exec import ExecutionError
+
+
+LOG = logging.getLogger(__name__)
+
+class DepEnsureAction(BaseAction):
+
+    """
+    A Lambda Builder Action which runs dep to install dependencies from Gopkg.toml
+    """
+
+    NAME = "DepEnsure"
+    DESCRIPTION = "Ensures all dependencies are installed for a project"
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+
+    def __init__(self, base_dir, subprocess_dep):
+        super(DepEnsureAction, self).__init__()
+
+        self.base_dir = base_dir
+        self.subprocess_dep = subprocess_dep
+
+    def execute(self):
+        try:
+            self.subprocess_dep.run(["ensure"],
+                                    cwd=self.base_dir)
+        except ExecutionError as ex:
+            raise ActionFailedError(str(ex))
+
+class GoBuildAction(BaseAction):
+
+    """
+    A Lambda Builder Action which runs `go build` to create a binary
+    """
+
+    NAME = "GoBuild"
+    DESCRIPTION = "Builds final binary"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    def __init__(self, base_dir, source_path, output_path, subprocess_go, env=None):
+        super(GoBuildAction, self).__init__()
+
+        self.base_dir = base_dir
+        self.source_path = source_path
+        self.output_path = output_path
+
+        self.subprocess_go = subprocess_go
+        self.env = env if not env is None else {}
+
+    def execute(self):
+        env = self.env
+        env.update({"GOOS": "linux", "GOARCH": "amd64"})
+
+        try:
+            self.subprocess_go.run(["build", "-o", self.output_path, self.source_path],
+                                    cwd=self.source_path, env=env)
+        except ExecutionError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/go_dep/subproc_exec.py

@@ -0,0 +1,93 @@
+"""
+Wrapper around calling dep through a subprocess.
+"""
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ExecutionError(Exception):
+    """
+    Exception raised in case binary execution fails.
+    It will pass on the standard error output from the binary console.
+    """
+
+    MESSAGE = "Exec Failed: {}"
+
+    def __init__(self, message):
+        raw_message = message
+        if isinstance(message, bytes):
+            message = message.decode('utf-8')
+
+        try:
+            Exception.__init__(self, self.MESSAGE.format(message.strip()))
+        except UnicodeError:
+            Exception.__init__(self, self.MESSAGE.format(raw_message.strip()))
+
+class SubprocessExec(object):
+
+    """
+    Wrapper around the Dep command line utility, making it
+    easy to consume execution results.
+    """
+
+    def __init__(self, osutils, binary=None):
+        """
+        :type osutils: aws_lambda_builders.workflows.go_dep.utils.OSUtils
+        :param osutils: An instance of OS Utilities for file manipulation
+
+        :type binary: str
+        :param binary: Path to the binary. If not set,
+            the default executable path will be used
+        """
+        self.osutils = osutils
+
+        self.binary = binary
+
+
+    def run(self, args, cwd=None, env=None):
+
+        """
+        Runs the action.
+
+        :type args: list
+        :param args: Command line arguments to pass to the binary
+
+        :type cwd: str
+        :param cwd: Directory where to execute the command (defaults to current dir)
+
+        :rtype: str
+        :return: text of the standard output from the command
+
+        :raises aws_lambda_builders.workflows.go_dep.dep.ExecutionError:
+            when the command executes with a non-zero return code. The exception will
+            contain the text of the standard error output from the command.
+
+        :raises ValueError: if arguments are not provided, or not a list
+        """
+
+        if not isinstance(args, list):
+            raise ValueError("args must be a list")
+
+        if not args:
+            raise ValueError("requires at least one arg")
+
+        invoke_bin = [self.binary] + args
+
+        LOG.debug("executing binary: %s", invoke_bin)
+
+        p = self.osutils.popen(invoke_bin,
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=cwd,
+                               env=env)
+
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise ExecutionError(message=err)
+
+        out = out.decode('utf-8') if isinstance(out, bytes) else out
+
+        return out.strip()

aws_lambda_builders/workflows/go_dep/utils.py

@@ -0,0 +1,42 @@
+"""
+Commonly used utilities
+"""
+
+import os
+import platform
+import tarfile
+import subprocess
+
+
+class OSUtils(object):
+
+    """
+    Wrapper around file system functions, to make it easy to
+    unit test actions in memory
+
+    TODO: move to somewhere generic
+    """
+
+    def joinpath(self, *args):
+        return os.path.join(*args)
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+        return p
+
+    @property
+    def pipe(self):
+        return subprocess.PIPE
+
+    @property
+    def environ(self):
+        return os.environ.copy()
+
+    def dirname(self, path):
+        return os.path.dirname(path)
+
+    def abspath(self, path):
+        return os.path.abspath(path)
+
+    def is_windows(self):
+        return platform.system().lower() == 'windows'

aws_lambda_builders/workflows/go_dep/workflow.py

@@ -0,0 +1,63 @@
+"""
+Go Dep Workflow
+"""
+
+import logging
+import os
+
+from aws_lambda_builders.actions import CopySourceAction
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+
+from .actions import DepEnsureAction, GoBuildAction
+from .utils import OSUtils
+from .subproc_exec import SubprocessExec
+
+LOG = logging.getLogger(__name__)
+
+class GoDepWorkflow(BaseWorkflow):
+    """
+    A Lambda builder workflow that knows how to build
+    Go projects using `dep`
+    """
+
+    NAME = "GoDepBuilder"
+
+    CAPABILITY = Capability(language="go",
+                            dependency_manager="dep",
+                            application_framework=None)
+
+    EXCLUDED_FILES = (".aws-sam")
+
+    def __init__(self,
+                source_dir,
+                artifacts_dir,
+                scratch_dir,
+                manifest_path,
+                runtime=None,
+                osutils=None,
+                **kwargs):
+
+        super(GoDepWorkflow, self).__init__(source_dir,
+                                            artifacts_dir,
+                                            scratch_dir,
+                                            manifest_path,
+                                            runtime=runtime,
+                                            **kwargs)
+
+        options = kwargs["options"] if "options" in kwargs else {}
+        handler = options.get("handler", None)
+
+        if osutils is None:
+            osutils = OSUtils()
+
+        # project base name, where the Gopkg.toml and vendor dir are.
+        base_dir = osutils.abspath(osutils.dirname(manifest_path))
+        output_path = osutils.joinpath(osutils.abspath(artifacts_dir), handler)
+
+        subprocess_dep = SubprocessExec(osutils, "dep")
+        subprocess_go = SubprocessExec(osutils, "go")
+
+        self.actions = [
+            DepEnsureAction(base_dir, subprocess_dep),
+            GoBuildAction(base_dir, osutils.abspath(source_dir), output_path, subprocess_go, env=osutils.environ)
+        ]

tests/functional/workflows/go_dep/test_godep_utils.py

@@ -0,0 +1,63 @@
+import os
+import shutil
+import sys
+import tempfile
+
+from unittest import TestCase
+
+from aws_lambda_builders.workflows.go_dep import utils
+
+
+class TestGoDepOSUtils(TestCase):
+
+    def setUp(self):
+
+        self.osutils = utils.OSUtils()
+
+    def test_dirname_returns_directory_for_path(self):
+        dirname = self.osutils.dirname(sys.executable)
+
+        self.assertEqual(dirname, os.path.dirname(sys.executable))
+
+    def test_abspath_returns_absolute_path(self):
+
+        result = self.osutils.abspath('.')
+
+        self.assertTrue(os.path.isabs(result))
+
+        self.assertEqual(result, os.path.abspath('.'))
+
+    def test_joinpath_joins_path_components(self):
+
+        result = self.osutils.joinpath('a', 'b', 'c')
+
+        self.assertEqual(result, os.path.join('a', 'b', 'c'))
+
+    def test_popen_runs_a_process_and_returns_outcome(self):
+
+        cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py')
+
+        p = self.osutils.popen([sys.executable, cwd_py],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe)
+
+        out, err = p.communicate()
+
+        self.assertEqual(p.returncode, 0)
+
+        self.assertEqual(out.decode('utf8').strip(), os.getcwd())
+
+    def test_popen_can_accept_cwd(self):
+
+        testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata')
+
+        p = self.osutils.popen([sys.executable, 'cwd.py'],
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=testdata_dir)
+
+        out, err = p.communicate()
+
+        self.assertEqual(p.returncode, 0)
+
+        self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir))