[WIP] support for building nodejs_npm functions

aws_lambda_builders/workflows/__init__.py

@@ -3,3 +3,4 @@
 """
 
 import aws_lambda_builders.workflows.python_pip
+import aws_lambda_builders.workflows.nodejs_npm

aws_lambda_builders/workflows/nodejs_npm/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds NodeJS Lambda functions using NPM dependency manager
+"""
+
+from .workflow import NodejsNpmWorkflow

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -0,0 +1,85 @@
+"""
+Action to resolve NodeJS dependencies using NPM
+"""
+
+import logging
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from .utils import OSUtils
+from .npm import SubprocessNpm, NpmExecutionError
+
+LOG = logging.getLogger(__name__)
+
+
+class NodejsNpmPackAction(BaseAction):
+
+    NAME = 'NpmPack'
+    DESCRIPTION = "Packaging source using NPM"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, osutils=None, subprocess_npm=None):
+        self.artifacts_dir = artifacts_dir
+        self.manifest_path = manifest_path
+        self.scratch_dir = scratch_dir
+        self.runtime = runtime
+
+        self.osutils = osutils
+        if osutils is None:
+            self.osutils = OSUtils()
+
+        self.subprocess_npm = subprocess_npm
+
+        if self.subprocess_npm is None:
+            self.subprocess_npm = SubprocessNpm(self.osutils)
+
+    def execute(self):
+        try:
+            package_path = "file:{}".format(self.osutils.abspath(self.osutils.dirname(self.manifest_path)))
+
+            LOG.debug("NODEJS packaging %s to %s", package_path, self.scratch_dir)
+
+            tarfile_name = self.subprocess_npm.run(['pack', '-q', package_path], cwd=self.scratch_dir)
+
+            LOG.debug("NODEJS packed to %s", tarfile_name)
+
+            tarfile_path = self.osutils.joinpath(self.scratch_dir, tarfile_name)
+
+            LOG.debug("NODEJS extracting to %s", self.artifacts_dir)
+
+            self.osutils.extract_tarfile(tarfile_path, self.artifacts_dir)
+
+        except NpmExecutionError as ex:
+            raise ActionFailedError(str(ex))
+
+
+class NodejsNpmInstallAction(BaseAction):
+
+    NAME = 'NpmInstall'
+    DESCRIPTION = "Installing dependencies from NPM"
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+
+    def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, osutils=None, subprocess_npm=None):
+        self.artifacts_dir = artifacts_dir
+        self.manifest_path = manifest_path
+        self.scratch_dir = scratch_dir
+        self.runtime = runtime
+
+        self.osutils = osutils
+        if osutils is None:
+            self.osutils = OSUtils()
+
+        self.subprocess_npm = subprocess_npm
+
+        if self.subprocess_npm is None:
+            self.subprocess_npm = SubprocessNpm(self.osutils)
+
+    def execute(self):
+        try:
+            LOG.debug("NODEJS installing in: %s from: %s", self.artifacts_dir, self.manifest_path)
+
+            self.subprocess_npm.run(
+                    ['install', '-q', '--no-audit', '--no-save', '--production'],
+                    cwd=self.artifacts_dir
+            )
+
+        except NpmExecutionError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/nodejs_npm/npm.py

@@ -0,0 +1,49 @@
+"""
+Wrapper around calling npm through a subprocess.
+"""
+
+import logging
+
+from aws_lambda_builders.exceptions import LambdaBuilderError
+
+from .utils import OSUtils
+
+LOG = logging.getLogger(__name__)
+
+
+class NpmExecutionError(LambdaBuilderError):
+    MESSAGE = "NPM Failed: {message}"
+
+
+class SubprocessNpm(object):
+
+    def __init__(self, osutils=None, npm_exe=None):
+        if osutils is None:
+            osutils = OSUtils()
+        self.osutils = osutils
+
+        if npm_exe is None:
+            npm_exe = 'npm'
+
+        self.npm_exe = npm_exe
+
+    def run(self, args, cwd=None):
+
+        if not isinstance(args, list):
+            raise ValueError('args must be a list')
+
+        invoke_npm = [self.npm_exe] + args
+
+        LOG.debug("executing NPM: %s", invoke_npm)
+
+        p = self.osutils.popen(invoke_npm,
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=cwd)
+
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise NpmExecutionError(message=err.decode('utf8').strip())
+
+        return out.decode('utf8').strip()

aws_lambda_builders/workflows/nodejs_npm/utils.py

@@ -0,0 +1,53 @@
+"""
+Commonly used utilities
+"""
+
+import os
+import shutil
+import tarfile
+import subprocess
+
+
+class OSUtils(object):
+
+    def environ(self):
+        return os.environ
+
+    def file_exists(self, filename):
+        return os.path.isfile(filename)
+
+    def extract_tarfile(self, tarfile_path, unpack_dir):
+        with tarfile.open(tarfile_path, 'r:*') as tar:
+            tar.extractall(unpack_dir)
+
+    def get_directory_contents(self, path):
+        return os.listdir(path)
+
+    def joinpath(self, *args):
+        return os.path.join(*args)
+
+    def copytree(self, source, destination):
+        if not os.path.exists(destination):
+            self.makedirs(destination)
+        names = self.get_directory_contents(source)
+        for name in names:
+            new_source = os.path.join(source, name)
+            new_destination = os.path.join(destination, name)
+            if os.path.isdir(new_source):
+                self.copytree(new_source, new_destination)
+            else:
+                shutil.copy2(new_source, new_destination)
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+        return p
+
+    @property
+    def pipe(self):
+        return subprocess.PIPE
+
+    def dirname(self, path):
+        return os.path.dirname(path)
+
+    def abspath(self, path):
+        return os.path.abspath(path)

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -0,0 +1,56 @@
+"""
+NodeJS NPM Workflow
+"""
+
+import hashlib
+
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+from aws_lambda_builders.actions import CopySourceAction
+
+from .actions import NodejsNpmPackAction, NodejsNpmInstallAction
+from .utils import OSUtils
+
+
+def unique_dir(manifest_path):
+    md5 = hashlib.md5()
+    md5.update(manifest_path.encode('utf8'))
+    return md5.hexdigest()
+
+
+class NodejsNpmWorkflow(BaseWorkflow):
+
+    NAME = "NodejsNpmBuilder"
+
+    CAPABILITY = Capability(language="nodejs",
+                            dependency_manager="npm",
+                            application_framework=None)
+
+    EXCLUDED_FILES = (".aws-sam")
+
+    def __init__(self,
+                 source_dir,
+                 artifacts_dir,
+                 scratch_dir,
+                 manifest_path,
+                 runtime=None,
+                 osutils=None,
+                 **kwargs):
+
+        super(NodejsNpmWorkflow, self).__init__(source_dir,
+                                                artifacts_dir,
+                                                scratch_dir,
+                                                manifest_path,
+                                                runtime=runtime,
+                                                **kwargs)
+
+        if osutils is None:
+            osutils = OSUtils()
+
+        tar_dest_dir = osutils.joinpath(scratch_dir, unique_dir(manifest_path))
+        tar_package_dir = osutils.joinpath(tar_dest_dir, 'package')
+
+        self.actions = [
+            NodejsNpmPackAction(tar_dest_dir, scratch_dir, manifest_path, runtime),
+            CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES),
+            NodejsNpmInstallAction(artifacts_dir, scratch_dir, manifest_path, runtime)
+        ]

tests/functional/testdata/cwd.py

@@ -0,0 +1,3 @@
+import os
+
+print(os.getcwd())

tests/functional/workflows/nodejs_npm/test_utils.py

@@ -0,0 +1,44 @@
+import io
+
+import pytest
+
+import sys
+
+import os
+
+from aws_lambda_builders.workflows.nodejs_npm import utils
+
+
+@pytest.fixture
+def osutils():
+    return utils.OSUtils()
+
+
+class TestOSUtils(object):
+    def test_dirname_returns_directory_for_path(self, tmpdir, osutils):
+        dirname = osutils.dirname(sys.executable)
+
+        assert dirname == os.path.dirname(sys.executable)
+
+    def test_abspath_returns_absolute_path(self, tmpdir, osutils):
+
+        result = osutils.abspath('.')
+
+        assert os.path.isabs(result)
+
+        assert result == os.path.abspath('.')
+
+    def test_popen_can_accept_cwd(self, tmpdir, osutils):
+
+        testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata')
+
+        p = osutils.popen([sys.executable, 'cwd.py'],
+                          stdout=osutils.pipe,
+                          stderr=osutils.pipe,
+                          cwd=testdata_dir)
+
+        out, err = p.communicate()
+
+        assert p.returncode == 0
+
+        assert out.decode('utf8').strip() == os.path.abspath(testdata_dir)

tests/integration/workflows/nodejs_npm/test_nodejs_npm.py

@@ -0,0 +1,79 @@
+
+import os
+import shutil
+import tempfile
+from unittest import TestCase
+
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowFailedError
+
+
+class TestNodejsNpmWorkflow(TestCase):
+    """
+    Verifies that `nodejs_npm` workflow works by building a Lambda using NPM
+    """
+
+    TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+
+    def setUp(self):
+        self.artifacts_dir = tempfile.mkdtemp()
+        self.scratch_dir = tempfile.mkdtemp()
+
+        self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+
+        self.builder = LambdaBuilder(language="nodejs",
+                                     dependency_manager="npm",
+                                     application_framework=None)
+        self.runtime = "nodejs8.10"
+
+    def tearDown(self):
+        shutil.rmtree(self.artifacts_dir)
+        shutil.rmtree(self.scratch_dir)
+
+    def test_builds_project_without_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "package.json"),
+                           runtime=self.runtime)
+
+        expected_files = {"package.json", "included.js"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+    def test_builds_project_with_remote_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
+
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "package.json"),
+                           runtime=self.runtime)
+
+        expected_files = {"package.json", "included.js", "node_modules"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+        expected_modules = {"minimal-request-promise"}
+        output_modules = set(os.listdir(os.path.join(self.artifacts_dir, "node_modules")))
+        self.assertEquals(expected_modules, output_modules)
+
+    def test_fails_if_npm_cannot_resolve_dependencies(self):
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "package.json"),
+                               runtime=self.runtime)
+
+        self.assertIn("No matching version found for [email protected]_EXISTENT", str(ctx.exception))
+
+    def test_fails_if_package_json_is_broken(self):
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-package")
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "package.json"),
+                               runtime=self.runtime)
+
+        self.assertIn("Unexpected end of JSON input", str(ctx.exception))

tests/integration/workflows/nodejs_npm/testdata/broken-deps/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/broken-deps/included.js

@@ -0,0 +1,2 @@
+//included
+const x = 1;