Merge branch 'develop' into rust-cargo-lambda-workflow

Author: mndeveci

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create an issue to report a bug for AWS Lambda Builders
+title: "Bug: TITLE"
+labels: ['type/bug', 'stage/needs-triage']
+assignees: ''
+
+---
+
+<!-- Make sure we don't have an existing Issue that reports the bug you are seeing (both open and closed). 
+If you do find an existing Issue, re-open or add a comment to that Issue instead of creating a new one. -->
+
+### Description:
+<!-- Briefly describe the bug you are facing.-->
+
+
+
+### Steps to reproduce:
+<!-- Provide detailed steps to replicate the bug, including steps from third party tools (CDK, etc.) -->
+
+
+
+### Observed result:
+<!-- Please provide command output with `--debug` flag set.-->
+
+
+
+### Expected result:
+<!-- Describe what you expected.-->
+
+
+
+### Additional environment details (Ex: Windows, Mac, Amazon Linux etc)
+
+1. OS:
+2. If using SAM CLI, `sam --version`:
+3. AWS region:
+
+`Add --debug flag to any SAM CLI commands you are running`

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea/feature/enhancement for AWS Lambda Builders
+title: "Feature request: TITLE"
+labels: ['type/feature', 'stage/needs-triage']
+assignees: ''
+
+---
+
+<!-- Make sure we don't have an existing Issue for that feature request (open or closed). -->
+
+### Describe your idea/feature/enhancement
+
+Provide a clear description.
+
+Ex: I wish the AWS Lambda Builders would [...]
+
+### Proposal
+
+Add details on how to add this to the product.
+
+### Additional Details

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,8 @@
+---
+name: Other
+about: Choose if your issue doesn't apply to the other templates
+title: ''
+labels: ['stage/needs-triage']
+assignees: ''
+
+---

.github/ISSUE_TEMPLATE/other.md

@@ -34,6 +34,6 @@ jobs:
                 issue_number: context.issue.number,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                labels: ['pr/external']
+                labels: ['pr/external', 'stage/needs-triage']
               })
             }

.github/workflows/pr-labeler.yml

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.16.0"
+__version__ = "1.18.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/__init__.py

@@ -5,7 +5,7 @@
 import logging
 import os
 import shutil
-import six
+from typing import Set, Iterator, Tuple
 
 from aws_lambda_builders.utils import copytree
 
@@ -58,7 +58,7 @@ def __new__(mcs, name, bases, class_dict):
 
         # Validate class variables
         # All classes must provide a name
-        if not isinstance(cls.NAME, six.string_types):
+        if not isinstance(cls.NAME, str):
             raise ValueError("Action must provide a valid name")
 
         if not Purpose.has_value(cls.PURPOSE):
@@ -67,7 +67,7 @@ def __new__(mcs, name, bases, class_dict):
         return cls
 
 
-class BaseAction(six.with_metaclass(_ActionMetaClass, object)):
+class BaseAction(object, metaclass=_ActionMetaClass):
     """
     Base class for all actions. It does not provide any implementation.
     """
@@ -125,14 +125,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir):
         self.dest_dir = destination_dir
 
     def execute(self):
-        source = set(os.listdir(self.source_dir))
-        artifact = set(os.listdir(self.artifact_dir))
-        dependencies = artifact - source
-
-        for name in dependencies:
-            dependencies_source = os.path.join(self.artifact_dir, name)
-            new_destination = os.path.join(self.dest_dir, name)
+        deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir)
 
+        for dependencies_source, new_destination in deps_manager.yield_source_dest():
             if os.path.isdir(dependencies_source):
                 copytree(dependencies_source, new_destination)
             else:
@@ -154,14 +149,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir):
         self.dest_dir = destination_dir
 
     def execute(self):
-        source = set(os.listdir(self.source_dir))
-        artifact = set(os.listdir(self.artifact_dir))
-        dependencies = artifact - source
-
-        for name in dependencies:
-            dependencies_source = os.path.join(self.artifact_dir, name)
-            new_destination = os.path.join(self.dest_dir, name)
+        deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir)
 
+        for dependencies_source, new_destination in deps_manager.yield_source_dest():
             # shutil.move can't create subfolders if this is the first file in that folder
             if os.path.isfile(dependencies_source):
                 os.makedirs(os.path.dirname(new_destination), exist_ok=True)
@@ -198,3 +188,36 @@ def execute(self):
                 shutil.rmtree(target_path)
             else:
                 os.remove(target_path)
+
+
+class DependencyManager:
+    """
+    Class for handling the management of dependencies between directories
+    """
+
+    # Ignore these files when comparing against which dependencies to move
+    # This allows for the installation of dependencies in the source directory
+    IGNORE_LIST = ["node_modules"]
+
+    def __init__(self, source_dir, artifact_dir, destination_dir) -> None:
+        self._source_dir: str = source_dir
+        self._artifact_dir: str = artifact_dir
+        self._dest_dir: str = destination_dir
+        self._dependencies: Set[str] = set()
+
+    def yield_source_dest(self) -> Iterator[Tuple[str, str]]:
+        self._set_dependencies()
+        for dep in self._dependencies:
+            yield os.path.join(self._artifact_dir, dep), os.path.join(self._dest_dir, dep)
+
+    def _set_dependencies(self) -> None:
+        source = self._get_source_files_exclude_deps()
+        artifact = set(os.listdir(self._artifact_dir))
+        self._dependencies = artifact - source
+
+    def _get_source_files_exclude_deps(self) -> Set[str]:
+        source_files = set(os.listdir(self._source_dir))
+        for item in self.IGNORE_LIST:
+            if item in source_files:
+                source_files.remove(item)
+        return source_files

aws_lambda_builders/actions.py

@@ -6,7 +6,6 @@
 import logging
 
 from collections import namedtuple
-import six
 
 from aws_lambda_builders.binary_path import BinaryPath
 from aws_lambda_builders.path_resolver import PathResolver
@@ -118,7 +117,7 @@ def __new__(mcs, name, bases, class_dict):
         # Validate class variables
 
         # All classes must provide a name
-        if not isinstance(cls.NAME, six.string_types):
+        if not isinstance(cls.NAME, str):
             raise ValueError("Workflow must provide a valid name")
 
         # All workflows must express their capabilities
@@ -131,7 +130,7 @@ def __new__(mcs, name, bases, class_dict):
         return cls
 
 
-class BaseWorkflow(six.with_metaclass(_WorkflowMetaClass, object)):
+class BaseWorkflow(object, metaclass=_WorkflowMetaClass):
     """
     Default implementation of the builder workflow. It provides several useful capabilities out-of-box that help
     minimize the scope of build actions.

aws_lambda_builders/workflow.py

@@ -1 +1 @@
-six~=1.11
+

requirements/base.txt

@@ -137,7 +137,15 @@ def test_mismatch_runtime_python_project(self):
                 runtime=self.runtime_mismatch[self.runtime],
             )
         except WorkflowFailedError as ex:
-            self.assertIn("Binary validation failed", str(ex))
+            # handle both e.g. missing /usr/bin/python2.7 and situation where
+            # python2.7 does not have pip installed (as is the case in some
+            # Mac environments)
+            ex_s = str(ex)
+            if (
+                "Binary validation failed" not in ex_s
+                and "pip executable not found in your python environment" not in ex_s
+            ):
+                raise AssertionError("Unexpected exception") from ex
 
     def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
         with self.assertRaises(WorkflowFailedError):

tests/integration/workflows/python_pip/test_python_pip.py

@@ -1,5 +1,8 @@
+from pathlib import Path
+from typing import List, Tuple
 from unittest import TestCase
-from mock import patch, ANY
+from mock import patch, ANY, Mock
+from parameterized import parameterized
 
 from aws_lambda_builders.actions import (
     BaseAction,
@@ -8,6 +11,7 @@
     CopyDependenciesAction,
     MoveDependenciesAction,
     CleanUpAction,
+    DependencyManager,
 )
 
 
@@ -128,3 +132,47 @@ def test_must_copy(self, path_mock, listdir_mock, isdir_mock, rmtree_mock, rm_mo
         listdir_mock.assert_any_call(target_dir)
         rmtree_mock.assert_any_call("dir")
         rm_mock.assert_any_call("file")
+
+
+class TestDependencyManager(TestCase):
+    @parameterized.expand(
+        [
+            (
+                ["app.js", "package.js", "libs", "node_modules"],
+                ["app.js", "package.js", "libs", "node_modules"],
+                [("artifacts/node_modules", "dest/node_modules")],
+                None,
+            ),
+            (
+                ["file1, file2", "dep1", "dep2"],
+                ["file1, file2", "dep1", "dep2"],
+                [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")],
+                ["dep1", "dep2"],
+            ),
+            (
+                ["file1, file2"],
+                ["file1, file2", "dep1", "dep2"],
+                [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")],
+                ["dep1", "dep2"],
+            ),
+        ]
+    )
+    @patch("aws_lambda_builders.actions.os.listdir")
+    def test_excludes_dependencies_from_source(
+        self, source_files, artifact_files, expected, mock_dependencies, patched_list_dir
+    ):
+        dependency_manager = DependencyManager("source", "artifacts", "dest")
+        dependency_manager.IGNORE_LIST = (
+            dependency_manager.IGNORE_LIST if mock_dependencies is None else mock_dependencies
+        )
+        patched_list_dir.side_effect = [source_files, artifact_files]
+        source_destinations = list(
+            TestDependencyManager._convert_strings_to_paths(list(dependency_manager.yield_source_dest()))
+        )
+        expected_paths = TestDependencyManager._convert_strings_to_paths(expected)
+        for expected_source_dest in expected_paths:
+            self.assertIn(expected_source_dest, source_destinations)
+
+    @staticmethod
+    def _convert_strings_to_paths(source_dest_list):
+        return map(lambda item: (Path(item[0]), Path(item[1])), source_dest_list)