[WIP] feat: path resolver for languages

aws_lambda_builders/__main__.py

@@ -94,6 +94,7 @@ def main():  # pylint: disable=too-many-statements
                       params["scratch_dir"],
                       params["manifest_path"],
                       runtime=params["runtime"],
+                      runtime_path=params["runtime_path"],
                       optimizations=params["optimizations"],
                       options=params["options"])
 

aws_lambda_builders/actions.py

@@ -23,6 +23,9 @@ class Purpose(object):
     Enum like object to describe the purpose of each action.
     """
 
+    # Action is identifying language version against provided runtime
+    RESOLVE_LANGUAGE_VERSION = "RESOLVE_LANGUAGE_VERSION"
+
     # Action is identifying dependencies, downloading, compiling and resolving them
     RESOLVE_DEPENDENCIES = "RESOLVE_DEPENDENCIES"
 

aws_lambda_builders/builder.py

@@ -6,7 +6,6 @@
 import logging
 
 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY
-from aws_lambda_builders.validate import RuntimeValidator
 from aws_lambda_builders.workflow import Capability
 
 LOG = logging.getLogger(__name__)
@@ -56,7 +55,7 @@ def __init__(self, language, dependency_manager, application_framework, supporte
         LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability)
 
     def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
-              runtime=None, optimizations=None, options=None):
+              runtime=None, runtime_path=None, optimizations=None, options=None):
         """
         Actually build the code by running workflows
 
@@ -90,30 +89,18 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
         :param options:
             Optional dictionary of options ot pass to build action. **Not supported**.
         """
-        if runtime:
-            self._validate_runtime(runtime)
-
-
+        # import ipdb
+        # ipdb.set_trace()
         workflow = self.selected_workflow_cls(source_dir,
                                               artifacts_dir,
                                               scratch_dir,
                                               manifest_path,
                                               runtime=runtime,
+                                              runtime_path=runtime_path,
                                               optimizations=optimizations,
                                               options=options)
 
         return workflow.run()
 
-    def _validate_runtime(self, runtime):
-        """
-        validate runtime and local runtime version to make sure they match
-
-        :type runtime: str
-        :param runtime:
-            String matching a lambda runtime eg: python3.6
-        """
-        RuntimeValidator.validate_runtime(required_language=self.capability.language,
-                                          required_runtime=runtime)
-
     def _clear_workflows(self):
         DEFAULT_REGISTRY.clear()

aws_lambda_builders/path_resolver.py

@@ -0,0 +1,21 @@
+from aws_lambda_builders.utils import which
+
+
+class PathResolver(object):
+
+    def __init__(self, language, runtime):
+        self.language = language
+        self.runtime = runtime
+        self.executables = [self.runtime, self.language]
+
+    def _which(self):
+        for executable in self.executables:
+            path = which(executable)
+            if path:
+                return path
+        raise ValueError("Path resolution for runtime: {} of language: "
+                         "{} was not successful".format(self.runtime, self.language))
+
+    @property
+    def path(self):
+        return self._which()

aws_lambda_builders/utils.py

@@ -3,13 +3,77 @@
 """
 
 import shutil
+import sys
 import os
 import logging
 
 
 LOG = logging.getLogger(__name__)
 
 
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
+
 def copytree(source, destination, ignore=None):
     """
     Similar to shutil.copytree except that it removes the limitation that the destination directory should

aws_lambda_builders/validator.py

@@ -0,0 +1,29 @@
+"""
+Base Class for Runtime Validators
+"""
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class RuntimeValidator(object):
+    SUPPORTED_RUNTIMES = []
+
+    @abc.abstractmethod
+    def has_runtime(self):
+        """
+        Checks if the runtime is supported.
+        :param string runtime: Runtime to check
+        :return bool: True, if the runtime is supported.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def validate_runtime(self, runtime_path):
+        """
+        Checks if the language supplied matches the required lambda runtime
+        :param string runtime_path: runtime language path to validate
+        :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime
+        """
+        raise NotImplementedError

aws_lambda_builders/workflow.py

@@ -2,15 +2,16 @@
 Implementation of a base workflow
 """
 
-import os
 import logging
-
+import os
 from collections import namedtuple
+
 import six
 
-from aws_lambda_builders.registry import DEFAULT_REGISTRY
-from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError
 from aws_lambda_builders.actions import ActionFailedError
+from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError
+from aws_lambda_builders.registry import DEFAULT_REGISTRY
+from aws_lambda_builders.validator import RuntimeValidator
 
 LOG = logging.getLogger(__name__)
 
@@ -78,6 +79,7 @@ def __init__(self,
                  artifacts_dir,
                  scratch_dir,
                  manifest_path,
+                 runtime_path,
                  runtime=None,
                  optimizations=None,
                  options=None):
@@ -123,10 +125,24 @@ def __init__(self,
         self.runtime = runtime
         self.optimizations = optimizations
         self.options = options
+        self.runtime_path = runtime_path
 
         # Actions are registered by the subclasses as they seem fit
         self.actions = []
 
+    def get_validator(self, runtime):
+
+        class EmptyRuntimeValidator(RuntimeValidator):
+            def __init__(self):
+                pass
+
+            def has_runtime(self):
+                pass
+
+            def validate_runtime(self, runtime_path):
+                pass
+        return EmptyRuntimeValidator()
+
     def is_supported(self):
         """
         Is the given manifest supported? If the workflow exposes no manifests names, then we it is assumed that
@@ -138,6 +154,10 @@ def is_supported(self):
 
         return True
 
+    def resolve_runtime(self):
+        if self.runtime:
+            self.get_validator(self.runtime).validate_runtime(self.runtime_path)
+
     def run(self):
         """
         Actually perform the build by executing registered actions.
@@ -150,6 +170,15 @@ def run(self):
 
         LOG.debug("Running workflow '%s'", self.NAME)
 
+        try:
+            self.resolve_runtime()
+        except MisMatchRuntimeError as ex:
+            raise WorkflowFailedError(workflow_name=self.NAME,
+                                      action_name=None,
+                                      reason=str(ex))
+
+        LOG.info("%s path : '%s'", self.CAPABILITY.language, self.runtime_path)
+
         if not self.actions:
             raise WorkflowFailedError(workflow_name=self.NAME,
                                       action_name=None,

aws_lambda_builders/workflows/python_pip/actions.py

@@ -3,7 +3,8 @@
 """
 
 from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
-from .packager import PythonPipDependencyBuilder, PackagerError
+from aws_lambda_builders.workflows.python_pip.utils import OSUtils
+from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner
 
 
 class PythonPipBuildAction(BaseAction):
@@ -12,15 +13,22 @@ class PythonPipBuildAction(BaseAction):
     DESCRIPTION = "Installing dependencies from PIP"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime):
+    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, runtime_path):
         self.artifacts_dir = artifacts_dir
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.package_builder = PythonPipDependencyBuilder()
+        self.language_exec_path = runtime_path
+        self.pip = SubprocessPip(osutils=OSUtils(), python_exe=runtime_path)
+        self.pip_runner = PipRunner(python_exe=runtime_path, pip=self.pip)
+        self.package_builder = PythonPipDependencyBuilder(
+            dependency_builder=DependencyBuilder(osutils=OSUtils(),
+                pip_runner=self.pip_runner))
 
     def execute(self):
         try:
+            # import ipdb
+            # ipdb.set_trace()
             self.package_builder.build_dependencies(
                 self.artifacts_dir,
                 self.manifest_path,