feat: re-write of path resolver

- Every workflow has a list of validators, resolvers and binaries
- Actions can choose to use it or not.

Author: sriram-mv

.appveyor.yml

@@ -15,16 +15,14 @@ build: off
 install:
 # To run Nodejs workflow integ tests
 - ps: Install-Product node 8.10
-- ps: $env:Path = "$env:PYTHON;$env:PYTHON\\Scripts;$env:PYTHON\\bin;$env:Path"
-- ps: python --version
-- ps: python -m pip install -r requirements/dev.txt
-- ps: python -m pip install -e .
-- ps: python --version
-- ps: $env:Path = "C:\\Ruby25-x64\\bin;$env:Path"
-- ps: ruby --version
-- ps: gem install bundler --no-ri --no-rdoc
-- ps: bundler --version
+
+- "set PATH=%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+- "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
+- "%PYTHON%\\python.exe -m pip install -e ."
+- "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
+- "gem install bundler --no-ri --no-rdoc"
+- "bundler --version"
 
 test_script:
-- ps: python -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional
-- ps: python -m pytest tests/integration
+- "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
+- "%PYTHON%\\python.exe -m pytest tests/integration"

.pylintrc

@@ -9,7 +9,7 @@
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=compat.py, path_resolver.py, validator.py
+ignore=compat.py, utils.py, path_resolver.py, validator.py
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -360,4 +360,4 @@ int-import-graph=
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=Exception

Makefile

@@ -3,8 +3,8 @@ init:
 
 test:
 	# Run unit tests
-	# Fail if coverage falls below 94%
-	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 94 tests/unit tests/functional
+	# Fail if coverage falls below 93%
+	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 93 tests/unit tests/functional
 
 func-test:
 	LAMBDA_BUILDERS_DEV=1 pytest tests/functional

aws_lambda_builders/binary_path.py

@@ -0,0 +1,21 @@
+"""
+Class containing resolved path of binary given a validator and a resolver and the name of the binary.
+"""
+
+
+class BinaryPath(object):
+
+    def __init__(self, resolver, validator, binary, binary_path=None):
+        self.resolver = resolver
+        self.validator = validator
+        setattr(self, binary, binary)
+        self._binary_path = binary_path
+        self.path_provided = True if self._binary_path else False
+
+    @property
+    def binary_path(self):
+        return self._binary_path
+
+    @binary_path.setter
+    def binary_path(self, binary_path):
+        self._binary_path = binary_path

aws_lambda_builders/path_resolver.py

@@ -1,13 +1,28 @@
 """
-Basic Path Resolver that just returns the runtime.
+Basic Path Resolver that looks for the executable by runtime first, before proceeding to 'language' in PATH.
 """
 
+from aws_lambda_builders.utils import which
+
 
 class PathResolver(object):
 
-    def __init__(self, runtime):
+    def __init__(self, binary, runtime):
+        self.binary = binary
         self.runtime = runtime
+        self.executables = [self.runtime, self.binary]
+
+    def _which(self):
+        exec_paths = []
+        for executable in [executable for executable in self.executables if executable is not None]:
+            paths = which(executable)
+            exec_paths.extend(paths)
+
+        if not exec_paths:
+            raise ValueError("Path resolution for runtime: {} of binary: "
+                             "{} was not successful".format(self.runtime, self.binary))
+        return exec_paths
 
     @property
-    def exec_path(self):
-        return self.runtime
+    def exec_paths(self):
+        return self._which()

aws_lambda_builders/utils.py

@@ -3,6 +3,7 @@
 """
 
 import shutil
+import sys
 import os
 import logging
 
@@ -57,3 +58,69 @@ def copytree(source, destination, ignore=None):
             copytree(new_source, new_destination, ignore=ignore)
         else:
             shutil.copy2(new_source, new_destination)
+
+
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the paths which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+    Note: This function was backported from the Python 3 source code.
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+
+    def _access_check(fn, mode):
+        return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)
+
+    # If we're given a path with a directory part, look it up directly
+    # rather than referring to PATH directories. This includes checking
+    # relative to the current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if os.curdir not in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path
+        # extensions. This will allow us to short circuit when given
+        # "python.exe". If it does match, only test that one, otherwise we
+        # have to try others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    paths = []
+
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if normdir not in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    paths.append(name)
+    return paths

aws_lambda_builders/validator.py

@@ -9,9 +9,12 @@
 
 class RuntimeValidator(object):
 
-    def __init__(self, runtime, runtime_path):
+    def __init__(self, runtime):
         self.runtime = runtime
-        self.runtime_path = runtime_path
+        self._runtime_path = None
+
+    def validate(self, runtime_path):
+        self._runtime_path = runtime_path
+        return runtime_path
+
 
-    def validate_runtime(self):
-        pass

aws_lambda_builders/workflow.py

@@ -8,6 +8,7 @@
 from collections import namedtuple
 import six
 
+from aws_lambda_builders.binary_path import BinaryPath
 from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.validator import RuntimeValidator
 from aws_lambda_builders.registry import DEFAULT_REGISTRY
@@ -33,9 +34,24 @@ def sanitize(func):
 
     @functools.wraps(func)
     def wrapper(self, *args, **kwargs):
+        valid_paths = []
         # NOTE: we need to access workflow object to get the validator.
-        validator = self.get_validator()
-        validator.validate_runtime()
+        self.get_binaries()
+        # import ipdb
+        # ipdb.set_trace()
+        for binary_path in self.binaries:
+            validator = binary_path.validator
+            exec_paths = binary_path.resolver.exec_paths if not binary_path.path_provided else binary_path.binary_path
+            for executable_path in exec_paths:
+                valid_path = validator.validate(executable_path)
+                if valid_path:
+                    binary_path.binary_path = valid_path
+                    valid_paths.append(valid_path)
+                    break
+        if not valid_paths:
+            raise WorkflowFailedError(workflow_name=self.NAME,
+                                      action_name=None,
+                                      reason='Binary validation failed!')
         func(self, *args, **kwargs)
     return wrapper
 
@@ -144,6 +160,7 @@ def __init__(self,
 
         # Actions are registered by the subclasses as they seem fit
         self.actions = []
+        self.binaries = None
 
     def is_supported(self):
         """
@@ -156,17 +173,25 @@ def is_supported(self):
 
         return True
 
-    def get_executable(self):
+    def get_resolvers(self):
         """
-        Non specialized path resolver that just returns the first executable for the runtime on the path.
+        Non specialized path resolver that just returns the list of executable for the runtime on the path.
         """
-        return PathResolver(runtime=self.runtime).exec_path
+        return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language)]
 
-    def get_validator(self):
+    def get_validators(self):
         """
         No-op validator that does not validate the runtime_path.
         """
-        return RuntimeValidator(runtime_path=self.get_executable(), runtime=self.runtime)
+        return [RuntimeValidator(runtime=self.runtime)]
+
+    def get_binaries(self):
+        if not self.binaries:
+            resolvers = self.get_resolvers()
+            validators = self.get_validators()
+            self.binaries = [BinaryPath(resolver=resolver, validator=validator, binary=resolver.binary)
+                             for resolver, validator in zip(resolvers, validators)]
+        return self.binaries
 
     @sanitize
     def run(self):

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -1,7 +1,7 @@
 """
 NodeJS NPM Workflow
 """
-
+from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.actions import CopySourceAction
 from .actions import NodejsNpmPackAction, NodejsNpmInstallAction
@@ -60,3 +60,9 @@ def __init__(self,
             CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES),
             npm_install,
         ]
+
+    def get_resolvers(self):
+        """
+        specialized path resolver that just returns the list of executable for the runtime on the path.
+        """
+        return [PathResolver(runtime=self.runtime, binary="node")]

aws_lambda_builders/workflows/python_pip/actions.py

@@ -12,26 +12,30 @@ class PythonPipBuildAction(BaseAction):
     NAME = 'ResolveDependencies'
     DESCRIPTION = "Installing dependencies from PIP"
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+    LANGUAGE = 'python'
 
-    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, runtime_path):
+    def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, binaries):
         self.artifacts_dir = artifacts_dir
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.runtime_path = runtime_path
-        self.os_utils = OSUtils()
-        self.pip = SubprocessPip(osutils=self.os_utils, python_exe=runtime_path)
-        self.pip_runner = PipRunner(python_exe=runtime_path, pip=self.pip)
-        self.dependency_builder = DependencyBuilder(osutils=self.os_utils, pip_runner=self.pip_runner,
-                                                    runtime=runtime)
-
-        self.package_builder = PythonPipDependencyBuilder(osutils=self.os_utils,
-                                                          runtime=runtime,
-                                                          dependency_builder=self.dependency_builder)
+        self.binaries = binaries
 
     def execute(self):
+        os_utils = OSUtils()
+        # import ipdb
+        # ipdb.set_trace()
+        python_path = [binary.binary_path for binary in self.binaries if getattr(binary, self.LANGUAGE)][0]
+        pip = SubprocessPip(osutils=os_utils, python_exe=python_path)
+        pip_runner = PipRunner(python_exe=python_path, pip=pip)
+        dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner,
+                                               runtime=self.runtime)
+
+        package_builder = PythonPipDependencyBuilder(osutils=os_utils,
+                                                     runtime=self.runtime,
+                                                     dependency_builder=dependency_builder)
         try:
-            self.package_builder.build_dependencies(
+            package_builder.build_dependencies(
                 self.artifacts_dir,
                 self.manifest_path,
                 self.scratch_dir