feat: add python resolver and validator

Author: sriram-mv

aws_lambda_builders/path_resolver.py

@@ -1,9 +1,7 @@
 """
-Basic Path Resolver that just looks for the language in the path.
+Basic Path Resolver that just returns the runtime.
 """
 
-from whichcraft import which
-
 
 class PathResolver(object):
 
@@ -12,4 +10,4 @@ def __init__(self, runtime):
 
     @property
     def path(self):
-        return which(self.runtime)
+        return self.runtime

aws_lambda_builders/validator.py

@@ -9,7 +9,8 @@
 
 class RuntimeValidator(object):
 
-    def __init__(self, runtime_path):
+    def __init__(self, runtime, runtime_path):
+        self.runtime = runtime
         self.runtime_path = runtime_path
 
     def validate_runtime(self):

aws_lambda_builders/workflow.py

@@ -166,7 +166,7 @@ def get_validator(self):
         """
         No-op validator that does not validate the runtime_path.
         """
-        return RuntimeValidator(runtime_path=self.get_executable())
+        return RuntimeValidator(runtime_path=self.get_executable(), runtime=self.runtime)
 
     @sanitize
     def run(self):

aws_lambda_builders/workflows/python_pip/actions.py

@@ -3,7 +3,8 @@
 """
 
 from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
-from .packager import PythonPipDependencyBuilder, PackagerError
+from aws_lambda_builders.workflows.python_pip.utils import OSUtils
+from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner
 
 
 class PythonPipBuildAction(BaseAction):
@@ -12,12 +13,20 @@ class PythonPipBuildAction(BaseAction):
     DESCRIPTION = "Installing dependencies from PIP"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime):
+    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, runtime_path):
         self.artifacts_dir = artifacts_dir
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.package_builder = PythonPipDependencyBuilder(runtime=runtime)
+        self.runtime_path = runtime_path
+        self.pip = SubprocessPip(osutils=OSUtils(), python_exe=runtime_path)
+        self.pip_runner = PipRunner(python_exe=runtime_path, pip=self.pip)
+        self.dependency_builder = DependencyBuilder(osutils=OSUtils(), pip_runner=self.pip_runner,
+                                                    runtime=runtime)
+
+        self.package_builder = PythonPipDependencyBuilder(osutils=OSUtils(),
+                                                          runtime=runtime,
+                                                          dependency_builder=self.dependency_builder)
 
     def execute(self):
         try:

aws_lambda_builders/workflows/python_pip/compat.py

@@ -1,13 +1,21 @@
 import os
 
+from aws_lambda_builders.workflows.python_pip.utils import OSUtils
 
-def pip_import_string():
-    import pip
-    pip_major_version = pip.__version__.split('.')[0]
+
+def pip_import_string(python_exe):
+    os_utils = OSUtils()
+    cmd = [
+        python_exe,
+        "-c",
+        "import pip; assert pip.__version__.split('.')[0] == 9"
+    ]
+    p = os_utils.popen(cmd,stdout=os_utils.pipe, stderr=os_utils.pipe)
+    p.communicate()
     # Pip moved its internals to an _internal module in version 10.
     # In order to be compatible with version 9 which has it at at the
     # top level we need to figure out the correct import path here.
-    if pip_major_version == '9':
+    if p.returncode == 0:
         return 'from pip import main'
     else:
         return 'from pip._internal import main'

aws_lambda_builders/workflows/python_pip/packager.py

@@ -177,7 +177,7 @@ def __init__(self, osutils, runtime, pip_runner=None):
         """
         self._osutils = osutils
         if pip_runner is None:
-            pip_runner = PipRunner(SubprocessPip(osutils))
+            pip_runner = PipRunner(python_exe=None, pip=SubprocessPip(osutils))
         self._pip = pip_runner
         self.runtime = runtime
 
@@ -546,25 +546,25 @@ def get_package_name_and_version(self, sdist_path):
 
 class SubprocessPip(object):
     """Wrapper around calling pip through a subprocess."""
-    def __init__(self, osutils=None, import_string=None):
+    def __init__(self, osutils=None, python_exe=None, import_string=None):
         if osutils is None:
             osutils = OSUtils()
         self._osutils = osutils
+        self.python_exe = python_exe
         if import_string is None:
-            import_string = pip_import_string()
+            import_string = pip_import_string(python_exe=self.python_exe)
         self._import_string = import_string
 
     def main(self, args, env_vars=None, shim=None):
         if env_vars is None:
             env_vars = self._osutils.environ()
         if shim is None:
             shim = ''
-        python_exe = sys.executable
         run_pip = (
             'import sys; %s; sys.exit(main(%s))'
         ) % (self._import_string, args)
         exec_string = '%s%s' % (shim, run_pip)
-        invoke_pip = [python_exe, '-c', exec_string]
+        invoke_pip = [self.python_exe, '-c', exec_string]
         p = self._osutils.popen(invoke_pip,
                                 stdout=self._osutils.pipe,
                                 stderr=self._osutils.pipe,
@@ -581,9 +581,10 @@ class PipRunner(object):
                             "  Link is a directory,"
                             " ignoring download_dir")
 
-    def __init__(self, pip, osutils=None):
+    def __init__(self, python_exe, pip, osutils=None):
         if osutils is None:
             osutils = OSUtils()
+        self.python_exe = python_exe
         self._wrapped_pip = pip
         self._osutils = osutils
 

aws_lambda_builders/workflows/python_pip/path_resolver.py

@@ -0,0 +1,25 @@
+"""
+Python Path Resolver that looks for the executable by runtime first, before proceeding to 'python' in PATH.
+"""
+
+from whichcraft import which
+
+
+class PythonPathResolver(object):
+
+    def __init__(self, runtime):
+        self.language = 'python'
+        self.runtime = runtime
+        self.executables = [self.runtime, self.language]
+
+    def _which(self):
+        for executable in self.executables:
+            path = which(executable)
+            if path:
+                return path
+        raise ValueError("Path resolution for runtime: {} of language: "
+                         "{} was not successful".format(self.runtime, self.language))
+
+    @property
+    def path(self):
+        return self._which()

aws_lambda_builders/workflows/python_pip/validator.py

@@ -0,0 +1,65 @@
+"""
+Supported Runtimes and their validations.
+"""
+
+import logging
+import os
+import subprocess
+
+from aws_lambda_builders.exceptions import MisMatchRuntimeError
+
+LOG = logging.getLogger(__name__)
+
+
+class PythonRuntimeValidator(object):
+    SUPPORTED_RUNTIMES = [
+        "python2.7",
+        "python3.6",
+        "python3.7"
+    ]
+
+    def __init__(self, runtime, runtime_path):
+        self.language = "python"
+        self.runtime = runtime
+        self.runtime_path = runtime_path
+
+    def has_runtime(self):
+        """
+        Checks if the runtime is supported.
+        :param string runtime: Runtime to check
+        :return bool: True, if the runtime is supported.
+        """
+        return self.runtime in self.SUPPORTED_RUNTIMES
+
+    def validate_runtime(self):
+        """
+        Checks if the language supplied matches the required lambda runtime
+        :param string runtime_path: runtime to check eg: /usr/bin/python3.6
+        :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime
+        """
+        if not self.has_runtime():
+            LOG.warning("'%s' runtime is not "
+                        "a supported runtime", self.runtime_path)
+            return
+        cmd = self._validate_python_cmd(self.runtime_path)
+
+        p = subprocess.Popen(cmd,
+                             cwd=os.getcwd(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p.communicate()
+        if p.returncode != 0:
+            raise MisMatchRuntimeError(language=self.language,
+                                       required_runtime=self.runtime,
+                                       runtime_path=self.runtime_path)
+
+    def _validate_python_cmd(self, runtime_path):
+        major, minor = self.runtime.replace(self.language, "").split('.')
+        cmd = [
+            runtime_path,
+            "-c",
+            "import sys; "
+            "assert sys.version_info.major == {major} "
+            "and sys.version_info.minor == {minor}".format(
+                major=major,
+                minor=minor)]
+        return cmd

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -4,6 +4,8 @@
 
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.actions import CopySourceAction
+from aws_lambda_builders.workflows.python_pip.path_resolver import PythonPathResolver
+from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
 
 from .actions import PythonPipBuildAction
 
@@ -65,6 +67,12 @@ def __init__(self,
 
         self.actions = [
             PythonPipBuildAction(artifacts_dir, scratch_dir,
-                                 manifest_path, runtime),
+                                 manifest_path, runtime, runtime_path=self.get_executable()),
             CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES),
         ]
+
+    def get_executable(self):
+        return PythonPathResolver(runtime=self.runtime).path
+
+    def get_validator(self):
+        return PythonRuntimeValidator(runtime=self.runtime, runtime_path=self.get_executable())

tests/integration/workflows/python_pip/test_python_pip.py

@@ -5,6 +5,8 @@
 import tempfile
 from unittest import TestCase
 
+from whichcraft import which
+
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowFailedError, MisMatchRuntimeError
 
@@ -53,13 +55,13 @@ def test_must_build_python_project(self):
         self.assertEquals(expected_files, output_files)
 
     def test_mismatch_runtime_python_project(self):
-        with self.assertRaises(MisMatchRuntimeError) as mismatch_error:
+        # NOTE : Build still works if other versions of python are accesible on the path. eg: /usr/bin python2.7
+        # is still accessible within a python 3 virtualenv.
+        try:
             self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
                                runtime=self.runtime_mismatch[self.runtime])
-            self.assertEquals(mismatch_error.msg,
-                              MisMatchRuntimeError(language="python",
-                                                   required_runtime=self.runtime_mismatch[self.runtime],
-                                                   found_runtime=self.runtime).MESSAGE)
+        except MisMatchRuntimeError as ex:
+            self.assertIsNone(which(self.runtime_mismatch[self.runtime]))
 
     def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
         with self.assertRaises(WorkflowFailedError):