feat: allow path resolver to look at additional paths (#78)

* feat: allow path resolver to look at additional paths

- search for executables on additional paths that are not on `$PATH`
- additional paths take precedence before searching for executables
  on `$PATH`
- rev protocol version to 0.2 with introduction of additional search
  paths.

* fix: version checks for RPC calls

- address comments
- add functional tests

* fix: add Debug statements

- extra func test for back compat

* fix: explanation for version check compatibility

* fix: appveyor build

Author: sriram-mv

.appveyor.yml

@@ -23,7 +23,7 @@ install:
 - "%PYTHON%\\python.exe -m pip install -e ."
 - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
 - "gem --version"
-- "gem install bundler -v 1.17.3 --no-ri --no-rdoc"
+- "gem install bundler -v 1.17.3"
 - "bundler --version"
 - "echo %PATH%"
 

aws_lambda_builders/__init__.py

@@ -2,4 +2,4 @@
 AWS Lambda Builder Library
 """
 __version__ = '0.0.5'
-RPC_PROTOCOL_VERSION = "0.1"
+RPC_PROTOCOL_VERSION = "0.2"

aws_lambda_builders/__main__.py

@@ -10,10 +10,11 @@
 import json
 import os
 import logging
+import re
 
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError
-
+from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
 
 log_level = int(os.environ.get("LAMBDA_BUILDERS_LOG_LEVEL", logging.INFO))
 
@@ -24,6 +25,8 @@
 
 LOG = logging.getLogger(__name__)
 
+VERSION_REGEX = re.compile("^([0-9])+.([0-9]+)$")
+
 
 def _success_response(request_id, artifacts_dir):
     return json.dumps({
@@ -46,6 +49,31 @@ def _error_response(request_id, http_status_code, message):
     })
 
 
+def _parse_version(version_string):
+
+    if VERSION_REGEX.match(version_string):
+        return float(version_string)
+    else:
+        ex = "Protocol Version does not match : {}".format(VERSION_REGEX.pattern)
+        LOG.debug(ex)
+        raise ValueError(ex)
+
+
+def version_compatibility_check(version):
+    # The following check is between current protocol version vs version of the protocol
+    # with which aws-lambda-builders is called.
+    # Example:
+    # 0.2 < 0.2 comparison will fail, don't throw a value Error saying incompatible version.
+    # 0.2 < 0.3 comparison will pass, throwing a ValueError
+    # 0.2 < 0.1 comparison will fail, don't throw a value Error saying incompatible version
+
+    if _parse_version(lambda_builders_protocol_version) < version:
+        ex = "Incompatible Protocol Version : {}, " \
+             "Current Protocol Version: {}".format(version, lambda_builders_protocol_version)
+        LOG.error(ex)
+        raise ValueError(ex)
+
+
 def _write_response(response, exit_code):
     sys.stdout.write(response)
     sys.stdout.flush()  # Make sure it is written
@@ -77,11 +105,20 @@ def main():  # pylint: disable=too-many-statements
         response = _error_response(request_id, -32601, "Method unavailable")
         return _write_response(response, 1)
 
+    try:
+        protocol_version = _parse_version(params.get("__protocol_version"))
+        version_compatibility_check(protocol_version)
+
+    except ValueError:
+        response = _error_response(request_id, 505, "Unsupported Protocol Version")
+        return _write_response(response, 1)
+
     capabilities = params["capability"]
     supported_workflows = params.get("supported_workflows")
 
     exit_code = 0
     response = None
+
     try:
         builder = LambdaBuilder(language=capabilities["language"],
                                 dependency_manager=capabilities["dependency_manager"],
@@ -93,6 +130,7 @@ def main():  # pylint: disable=too-many-statements
                       params["artifacts_dir"],
                       params["scratch_dir"],
                       params["manifest_path"],
+                      executable_search_paths=params['executable_search_paths'],
                       runtime=params["runtime"],
                       optimizations=params["optimizations"],
                       options=params["options"])

aws_lambda_builders/builder.py

@@ -56,7 +56,7 @@ def __init__(self, language, dependency_manager, application_framework, supporte
         LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability)
 
     def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
-              runtime=None, optimizations=None, options=None):
+              runtime=None, optimizations=None, options=None, executable_search_paths=None):
         """
         Actually build the code by running workflows
 
@@ -89,6 +89,10 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
         :type options: dict
         :param options:
             Optional dictionary of options ot pass to build action. **Not supported**.
+
+        :type executable_search_paths: list
+        :param executable_search_paths:
+            Additional list of paths to search for executables required by the workflow.
         """
 
         if not os.path.exists(scratch_dir):
@@ -100,7 +104,8 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
                                               manifest_path,
                                               runtime=runtime,
                                               optimizations=optimizations,
-                                              options=options)
+                                              options=options,
+                                              executable_search_paths=executable_search_paths)
 
         return workflow.run()
 

aws_lambda_builders/path_resolver.py

@@ -7,15 +7,16 @@
 
 class PathResolver(object):
 
-    def __init__(self, binary, runtime):
+    def __init__(self, binary, runtime, executable_search_paths=None):
         self.binary = binary
         self.runtime = runtime
         self.executables = [self.runtime, self.binary]
+        self.executable_search_paths = executable_search_paths
 
     def _which(self):
         exec_paths = []
         for executable in [executable for executable in self.executables if executable is not None]:
-            paths = which(executable)
+            paths = which(executable, executable_search_paths=self.executable_search_paths)
             exec_paths.extend(paths)
 
         if not exec_paths:

aws_lambda_builders/utils.py

@@ -68,15 +68,27 @@ def copytree(source, destination, ignore=None):
 # Copyright 2019 by the Python Software Foundation
 
 
-def which(cmd, mode=os.F_OK | os.X_OK, path=None):  # pragma: no cover
-    """Given a command, mode, and a PATH string, return the paths which
-    conforms to the given mode on the PATH, or None if there is no such
-    file.
-    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
-    of os.environ.get("PATH"), or can be overridden with a custom search
-    path.
+def which(cmd, mode=os.F_OK | os.X_OK, executable_search_paths=None):  # pragma: no cover
+    """Given a command, mode, and executable search paths list, return the paths which
+    conforms to the given mode on the PATH with the prepended additional search paths,
+    or None if there is no such file.
+    `mode` defaults to os.F_OK | os.X_OK. the default search `path` defaults
+    to the result of os.environ.get("PATH")
     Note: This function was backported from the Python 3 source code.
+
+    :type cmd: str
+    :param cmd:
+        Executable to be looked up in PATH.
+
+    :type mode: str
+    :param mode:
+        Modes of access for the executable.
+
+    :type executable_search_paths: list
+    :param executable_search_paths:
+        List of paths to look for `cmd` in preference order.
     """
+
     # Check that a given file can be accessed with the correct mode.
     # Additionally check that `file` is not a directory, as on Windows
     # directories pass the os.access check.
@@ -93,13 +105,16 @@ def _access_check(fn, mode):
 
         return None
 
-    if path is None:
-        path = os.environ.get("PATH", os.defpath)
+    path = os.environ.get("PATH", os.defpath)
+
     if not path:
         return None
 
     path = path.split(os.pathsep)
 
+    if executable_search_paths:
+        path = executable_search_paths + path
+
     if sys.platform == "win32":
         # The current directory takes precedence on Windows.
         if os.curdir not in path:

aws_lambda_builders/workflow.py

@@ -117,6 +117,7 @@ def __init__(self,
                  scratch_dir,
                  manifest_path,
                  runtime=None,
+                 executable_search_paths=None,
                  optimizations=None,
                  options=None):
         """
@@ -152,6 +153,10 @@ def __init__(self,
         :type options: dict
         :param options:
             Optional dictionary of options ot pass to build action. **Not supported**.
+
+        :type executable_search_paths: list
+        :param executable_search_paths:
+            Optional, Additional list of paths to search for executables required by the workflow.
         """
 
         self.source_dir = source_dir
@@ -161,6 +166,7 @@ def __init__(self,
         self.runtime = runtime
         self.optimizations = optimizations
         self.options = options
+        self.executable_search_paths = executable_search_paths
 
         # Actions are registered by the subclasses as they seem fit
         self.actions = []
@@ -181,7 +187,8 @@ def get_resolvers(self):
         """
         Non specialized path resolver that just returns the list of executable for the runtime on the path.
         """
-        return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language)]
+        return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language,
+                             executable_search_paths=self.executable_search_paths)]
 
     def get_validators(self):
         """

tests/functional/test_builder.py

@@ -4,6 +4,11 @@
 import shutil
 import tempfile
 
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
+
 from unittest import TestCase
 from aws_lambda_builders.builder import LambdaBuilder
 
@@ -40,16 +45,17 @@ def tearDown(self):
         # Remove the workflows folder from PYTHONPATH
         sys.path.remove(self.TEST_WORKFLOWS_FOLDER)
 
-    def test_run_hello_workflow(self):
+    def test_run_hello_workflow_with_exec_paths(self):
 
         self.hello_builder.build(self.source_dir,
                                  self.artifacts_dir,
                                  self.scratch_dir,
-                                 "/ignored")
+                                 "/ignored",
+                                 executable_search_paths=[str(pathlib.Path(sys.executable).parent)])
 
         self.assertTrue(os.path.exists(self.expected_filename))
         contents = ''
         with open(self.expected_filename, 'r') as fp:
             contents = fp.read()
 
-        self.assertEquals(contents, self.expected_contents)
+        self.assertEquals(contents, self.expected_contents)

tests/functional/test_cli.py

@@ -5,10 +5,19 @@
 import tempfile
 import subprocess
 import copy
+import sys
 
 from unittest import TestCase
 from parameterized import parameterized
 
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
+
+
+from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
+
 
 class TestCliWithHelloWorkflow(TestCase):
 
@@ -39,19 +48,21 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.source_dir)
         shutil.rmtree(self.artifacts_dir)
-        shutil.rmtree(self.scratch_dir)
 
     @parameterized.expand([
-        ("request_through_stdin"),
-        ("request_through_argument")
+        ("request_through_stdin", lambda_builders_protocol_version),
+        ("request_through_argument", lambda_builders_protocol_version),
+        ("request_through_stdin", "0.1"),
+        ("request_through_argument", "0.1"),
     ])
-    def test_run_hello_workflow(self, flavor):
+    def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version):
 
         request_json = json.dumps({
             "jsonschema": "2.0",
             "id": 1234,
             "method": "LambdaBuilder.build",
             "params": {
+                "__protocol_version": protocol_version,
                 "capability": {
                     "language": self.language,
                     "dependency_manager": self.dependency_manager,
@@ -65,6 +76,7 @@ def test_run_hello_workflow(self, flavor):
                 "runtime": "ignored",
                 "optimizations": {},
                 "options": {},
+                "executable_search_paths": [str(pathlib.Path(sys.executable).parent)]
             }
         })
 
@@ -94,4 +106,52 @@ def test_run_hello_workflow(self, flavor):
             contents = fp.read()
 
         self.assertEquals(contents, self.expected_contents)
+        shutil.rmtree(self.scratch_dir)
+
+    @parameterized.expand([
+        ("request_through_stdin"),
+        ("request_through_argument")
+    ])
+    def test_run_hello_workflow_incompatible(self, flavor):
+
+        request_json = json.dumps({
+            "jsonschema": "2.0",
+            "id": 1234,
+            "method": "LambdaBuilder.build",
+            "params": {
+                "__protocol_version": "2.0",
+                "capability": {
+                    "language": self.language,
+                    "dependency_manager": self.dependency_manager,
+                    "application_framework": self.application_framework
+                },
+                "supported_workflows": [self.HELLO_WORKFLOW_MODULE],
+                "source_dir": self.source_dir,
+                "artifacts_dir": self.artifacts_dir,
+                "scratch_dir": self.scratch_dir,
+                "manifest_path": "/ignored",
+                "runtime": "ignored",
+                "optimizations": {},
+                "options": {},
+                "executable_search_paths": [str(pathlib.Path(sys.executable).parent)]
+            }
+        })
+
+
+        env = copy.deepcopy(os.environ)
+        env["PYTHONPATH"] = self.python_path
+
+        stdout_data = None
+        if flavor == "request_through_stdin":
+            p = subprocess.Popen([self.command_name], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            stdout_data = p.communicate(input=request_json.encode('utf-8'))[0]
+        elif flavor == "request_through_argument":
+            p = subprocess.Popen([self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            stdout_data = p.communicate()[0]
+        else:
+            raise ValueError("Invalid test flavor")
 
+        # Validate the response object. It should be error response
+        response = json.loads(stdout_data)
+        self.assertIn('error', response)
+        self.assertEquals(response['error']['code'], 505)

tests/functional/test_utils.py

@@ -40,6 +40,7 @@ def test_must_respect_excludes_list(self):
         self.assertEquals(set(os.listdir(os.path.join(self.dest, "a"))), {"c"})
         self.assertEquals(set(os.listdir(os.path.join(self.dest, "a"))), {"c"})
 
+
 def file(*args):
     path = os.path.join(*args)
     basedir = os.path.dirname(path)