Merge pull request #41 from awslabs/develop

Release v0.0.3

Author: sanathkr

.appveyor.yml

@@ -7,7 +7,7 @@ environment:
 
   - PYTHON: "C:\\Python27-x64"
   - PYTHON: "C:\\Python36-x64"
-#  - PYTHON: "C:\\Python37-x64"
+  - PYTHON: "C:\\Python37-x64"
 
 
 build: off

.travis.yml

@@ -5,11 +5,11 @@ python:
   - "2.7"
   - "3.6"
 # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs
-#matrix:
-#  include:
-#    - python: 3.7
-#      dist: xenial
-#      sudo: true
+matrix:
+  include:
+    - python: 3.7
+      dist: xenial
+      sudo: true
 install:
   # Install the code requirements
   - make init

MANIFEST.in

@@ -1,6 +1,5 @@
 include LICENSE
-include requirements/base.txt
-include requirements/dev.txt
+include requirements/*
 recursive-include aws_lambda_builders/workflows *
 prune tests
 

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = '0.0.2'
+__version__ = '0.0.3'
 RPC_PROTOCOL_VERSION = "0.1"

aws_lambda_builders/exceptions.py

@@ -16,8 +16,10 @@ class UnsupportedManifestError(LambdaBuilderError):
 
 
 class MisMatchRuntimeError(LambdaBuilderError):
-    MESSAGE = "A runtime version mismatch was found for the given language " \
-              "'{language}', required runtime '{required_runtime}'"
+    MESSAGE = "{language} executable found in your path does not " \
+              "match runtime. " \
+              "\n Expected version: {required_runtime}, Found version: {found_runtime}. " \
+              "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30"
 
 
 class WorkflowNotFoundError(LambdaBuilderError):

aws_lambda_builders/validate.py

@@ -17,6 +17,7 @@ def validate_python_cmd(required_language, required_runtime_version):
         "python",
         "-c",
         "import sys; "
+        "sys.stdout.write('python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)); "
         "assert sys.version_info.major == {major} "
         "and sys.version_info.minor == {minor}".format(
             major=major,
@@ -32,7 +33,8 @@ def validate_python_cmd(required_language, required_runtime_version):
 class RuntimeValidator(object):
     SUPPORTED_RUNTIMES = [
         "python2.7",
-        "python3.6"
+        "python3.6",
+        "python3.7",
     ]
 
     @classmethod
@@ -62,10 +64,11 @@ def validate_runtime(cls, required_language, required_runtime):
             p = subprocess.Popen(cmd,
                                  cwd=os.getcwd(),
                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            p.communicate()
+            found_runtime, _ = p.communicate()
             if p.returncode != 0:
                 raise MisMatchRuntimeError(language=required_language,
-                                           required_runtime=required_runtime)
+                                           required_runtime=required_runtime,
+                                           found_runtime=str(found_runtime.decode('utf-8')))
         else:
             LOG.warning("'%s' runtime has not "
                         "been validated!", required_language)

aws_lambda_builders/workflows/python_pip/DESIGN.md

@@ -5,7 +5,7 @@
 This package is an effort to port the Chalice packager to a library that can
 be used to handle the dependency resilution portion of packaging Python code
 for use in AWS Lambda. The scope for this builder is to take an existing
-directory containing customer code, and a top-levle `requirements.txt` file
+directory containing customer code, and a top-level `requirements.txt` file
 specifying third party depedencies. The builder will examine the dependencies
 and use pip to build and include the dependencies in the customer code bundle
 in a way that makes them importable.
@@ -146,4 +146,4 @@ bundle has an `__init__.py` and is on the `PYTHONPATH`.
 
 The dependencies should now be succesfully installed in the target directory.
 All the temporary/intermediate files can now be deleting including all the
-wheel files and sdists.
+wheel files and sdists.

aws_lambda_builders/workflows/python_pip/actions.py

@@ -17,15 +17,14 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime):
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.package_builder = PythonPipDependencyBuilder()
+        self.package_builder = PythonPipDependencyBuilder(runtime=runtime)
 
     def execute(self):
         try:
             self.package_builder.build_dependencies(
                 self.artifacts_dir,
                 self.manifest_path,
-                self.scratch_dir,
-                self.runtime,
+                self.scratch_dir
             )
         except PackagerError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/python_pip/compat.py

@@ -1,5 +1,4 @@
 import os
-import six
 
 
 def pip_import_string():
@@ -101,9 +100,3 @@ def raise_compile_error(*args, **kwargs):
     pip_no_compile_c_env_vars = {
         'CC': '/var/false'
     }
-
-
-if six.PY3:
-    lambda_abi = 'cp36m'
-else:
-    lambda_abi = 'cp27mu'

aws_lambda_builders/workflows/python_pip/packager.py

@@ -5,15 +5,17 @@
 import sys
 import re
 import subprocess
+import logging
 from email.parser import FeedParser
 
 
-from .compat import lambda_abi
 from .compat import pip_import_string
 from .compat import pip_no_compile_c_env_vars
 from .compat import pip_no_compile_c_shim
 from .utils import OSUtils
 
+LOG = logging.getLogger(__name__)
+
 
 # TODO update the wording here
 MISSING_DEPENDENCIES_TEMPLATE = r"""
@@ -56,10 +58,36 @@ class PackageDownloadError(PackagerError):
     pass
 
 
+class UnsupportedPythonVersion(PackagerError):
+    """Generic networking error during a package download."""
+    def __init__(self, version):
+        super(UnsupportedPythonVersion, self).__init__(
+            "'%s' version of python is not supported" % version
+        )
+
+
+def get_lambda_abi(runtime):
+    supported = {
+        "python2.7": "cp27mu",
+        "python3.6": "cp36m",
+        "python3.7": "cp37m"
+    }
+
+    if runtime not in supported:
+        raise UnsupportedPythonVersion(runtime)
+
+    return supported[runtime]
+
+
 class PythonPipDependencyBuilder(object):
-    def __init__(self, osutils=None, dependency_builder=None):
+    def __init__(self, runtime, osutils=None, dependency_builder=None):
         """Initialize a PythonPipDependencyBuilder.
 
+        :type runtime: str
+        :param runtime: Python version to build dependencies for. This can
+            either be python2.7, python3.6 or python3.7. These are currently the
+            only supported values.
+
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
@@ -73,11 +101,11 @@ def __init__(self, osutils=None, dependency_builder=None):
             self.osutils = OSUtils()
 
         if dependency_builder is None:
-            dependency_builder = DependencyBuilder(self.osutils)
+            dependency_builder = DependencyBuilder(self.osutils, runtime)
         self._dependency_builder = dependency_builder
 
     def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
-                           requirements_path, runtime, ui=None, config=None):
+                           requirements_path, ui=None, config=None):
         """Builds a python project's dependencies into an artifact directory.
 
         :type artifacts_dir_path: str
@@ -90,11 +118,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
         :param requirements_path: Path to a requirements.txt file to inspect
             for a list of dependencies.
 
-        :type runtime: str
-        :param runtime: Python version to build dependencies for. This can
-            either be python2.7 or python3.6. These are currently the only
-            supported values.
-
         :type ui: :class:`lambda_builders.utils.UI` or None
         :param ui: A class that traps all progress information such as status
             and errors. If injected by the caller, it can be used to monitor
@@ -138,13 +161,16 @@ class DependencyBuilder(object):
         'sqlalchemy'
     }
 
-    def __init__(self, osutils, pip_runner=None):
+    def __init__(self, osutils, runtime, pip_runner=None):
         """Initialize a DependencyBuilder.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
 
+        :type runtime: str
+        :param runtime: AWS Lambda Python runtime to build for
+
         :type pip_runner: :class:`PipRunner`
         :param pip_runner: This class is responsible for executing our pip
             on our behalf.
@@ -153,6 +179,7 @@ def __init__(self, osutils, pip_runner=None):
         if pip_runner is None:
             pip_runner = PipRunner(SubprocessPip(osutils))
         self._pip = pip_runner
+        self.runtime = runtime
 
     def build_site_packages(self, requirements_filepath,
                             target_directory,
@@ -229,6 +256,9 @@ def _download_dependencies(self, directory, requirements_filename):
                 else:
                     incompatible_wheels.add(package)
 
+        LOG.debug("initial compatible: %s", compatible_wheels)
+        LOG.debug("initial incompatible: %s", incompatible_wheels | sdists)
+
         # Next we need to go through the downloaded packages and pick out any
         # dependencies that do not have a compatible wheel file downloaded.
         # For these packages we need to explicitly try to download a
@@ -242,6 +272,10 @@ def _download_dependencies(self, directory, requirements_filename):
         # file ourselves.
         compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
             directory)
+        LOG.debug(
+            "compatible wheels after second download pass: %s",
+            compatible_wheels
+        )
         missing_wheels = sdists - compatible_wheels
         self._build_sdists(missing_wheels, directory, compile_c=True)
 
@@ -255,6 +289,10 @@ def _download_dependencies(self, directory, requirements_filename):
         # compiler.
         compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
             directory)
+        LOG.debug(
+            "compatible after building wheels (no C compiling): %s",
+            compatible_wheels
+        )
         missing_wheels = sdists - compatible_wheels
         self._build_sdists(missing_wheels, directory, compile_c=False)
 
@@ -264,6 +302,10 @@ def _download_dependencies(self, directory, requirements_filename):
         # compatible version directly and building from source.
         compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
             directory)
+        LOG.debug(
+            "compatible after building wheels (C compiling): %s",
+            compatible_wheels
+        )
 
         # Now there is still the case left over where the setup.py has been
         # made in such a way to be incompatible with python's setup tools,
@@ -273,6 +315,9 @@ def _download_dependencies(self, directory, requirements_filename):
         compatible_wheels, incompatible_wheels = self._apply_wheel_whitelist(
             compatible_wheels, incompatible_wheels)
         missing_wheels = deps - compatible_wheels
+        LOG.debug("Final compatible: %s", compatible_wheels)
+        LOG.debug("Final incompatible: %s", incompatible_wheels)
+        LOG.debug("Final missing wheels: %s", missing_wheels)
 
         return compatible_wheels, missing_wheels
 
@@ -285,14 +330,19 @@ def _download_all_dependencies(self, requirements_filename, directory):
         self._pip.download_all_dependencies(requirements_filename, directory)
         deps = {Package(directory, filename) for filename
                 in self._osutils.get_directory_contents(directory)}
+        LOG.debug("Full dependency closure: %s", deps)
         return deps
 
     def _download_binary_wheels(self, packages, directory):
         # Try to get binary wheels for each package that isn't compatible.
+        LOG.debug("Downloading missing wheels: %s", packages)
+        lambda_abi = get_lambda_abi(self.runtime)
         self._pip.download_manylinux_wheels(
-            [pkg.identifier for pkg in packages], directory)
+            [pkg.identifier for pkg in packages], directory, lambda_abi)
 
     def _build_sdists(self, sdists, directory, compile_c=True):
+        LOG.debug("Build missing wheels from sdists "
+                  "(C compiling %s): %s", compile_c, sdists)
         for sdist in sdists:
             path_to_sdist = self._osutils.joinpath(directory, sdist.filename)
             self._pip.build_wheel(path_to_sdist, directory, compile_c)
@@ -316,6 +366,9 @@ def _is_compatible_wheel_filename(self, filename):
         # Verify platform is compatible
         if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM:
             return False
+
+        lambda_runtime_abi = get_lambda_abi(self.runtime)
+
         # Verify that the ABI is compatible with lambda. Either none or the
         # correct type for the python version cp27mu for py27 and cp36m for
         # py36.
@@ -326,7 +379,7 @@ def _is_compatible_wheel_filename(self, filename):
             # Deploying python 3 function which means we need cp36m abi
             # We can also accept abi3 which is the CPython 3 Stable ABI and
             # will work on any version of python 3.
-            return abi == 'cp36m' or abi == 'abi3'
+            return abi == lambda_runtime_abi or abi == 'abi3'
         elif prefix_version == 'cp2':
             # Deploying to python 2 function which means we need cp27mu abi
             return abi == 'cp27mu'
@@ -537,6 +590,7 @@ def __init__(self, pip, osutils=None):
     def _execute(self, command, args, env_vars=None, shim=None):
         """Execute a pip command with the given arguments."""
         main_args = [command] + args
+        LOG.debug("calling pip %s", ' '.join(main_args))
         rc, out, err = self._wrapped_pip.main(main_args, env_vars=env_vars,
                                               shim=shim)
         return rc, out, err
@@ -589,7 +643,7 @@ def download_all_dependencies(self, requirements_filename, directory):
             # complain at deployment time.
             self.build_wheel(wheel_package_path, directory)
 
-    def download_manylinux_wheels(self, packages, directory):
+    def download_manylinux_wheels(self, packages, directory, lambda_abi):
         """Download wheel files for manylinux for all the given packages."""
         # If any one of these dependencies fails pip will bail out. Since we
         # are only interested in all the ones we can download, we need to feed

requirements/dev.txt

@@ -2,6 +2,8 @@ coverage==4.3.4
 flake8==3.3.0
 tox==2.2.1
 pytest-cov==2.4.0
+# astroid > 2.0.4 is not compatible with pylint1.7
+astroid>=1.5.8,<2.1.0
 pylint==1.7.2
 
 # Test requirements

requirements/python_pip.txt

@@ -0,0 +1,5 @@
+
+# Following packages are required by `python_pip` workflow to run.
+# TODO: Consider moving this dependency directly into the `python_pip` workflow module
+setuptools
+wheel

setup.py

@@ -55,7 +55,7 @@ def read_version():
             '{}=aws_lambda_builders.__main__:main'.format(cmd_name)
         ]
     },
-    install_requires=read_requirements('base.txt'),
+    install_requires=read_requirements('base.txt') + read_requirements("python_pip.txt"),
     extras_require={
         'dev': read_requirements('dev.txt')
     },