feat: Python 37 support (#32)

* Python 37 support

* Making the ABI selection explicit

* Fixing the lambda_abi to a supported list

* Making tests work

Author: sanathkr

.appveyor.yml

@@ -7,7 +7,7 @@ environment:
 
   - PYTHON: "C:\\Python27-x64"
   - PYTHON: "C:\\Python36-x64"
-#  - PYTHON: "C:\\Python37-x64"
+  - PYTHON: "C:\\Python37-x64"
 
 
 build: off

.travis.yml

@@ -5,11 +5,11 @@ python:
   - "2.7"
   - "3.6"
 # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs
-#matrix:
-#  include:
-#    - python: 3.7
-#      dist: xenial
-#      sudo: true
+matrix:
+  include:
+    - python: 3.7
+      dist: xenial
+      sudo: true
 install:
   # Install the code requirements
   - make init

aws_lambda_builders/validate.py

@@ -32,7 +32,8 @@ def validate_python_cmd(required_language, required_runtime_version):
 class RuntimeValidator(object):
     SUPPORTED_RUNTIMES = [
         "python2.7",
-        "python3.6"
+        "python3.6",
+        "python3.7",
     ]
 
     @classmethod

aws_lambda_builders/workflows/python_pip/actions.py

@@ -17,15 +17,14 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime):
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.package_builder = PythonPipDependencyBuilder()
+        self.package_builder = PythonPipDependencyBuilder(runtime=runtime)
 
     def execute(self):
         try:
             self.package_builder.build_dependencies(
                 self.artifacts_dir,
                 self.manifest_path,
-                self.scratch_dir,
-                self.runtime,
+                self.scratch_dir
             )
         except PackagerError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/python_pip/compat.py

@@ -1,5 +1,4 @@
 import os
-import six
 
 
 def pip_import_string():
@@ -101,9 +100,3 @@ def raise_compile_error(*args, **kwargs):
     pip_no_compile_c_env_vars = {
         'CC': '/var/false'
     }
-
-
-if six.PY3:
-    lambda_abi = 'cp36m'
-else:
-    lambda_abi = 'cp27mu'

aws_lambda_builders/workflows/python_pip/packager.py

@@ -9,7 +9,6 @@
 from email.parser import FeedParser
 
 
-from .compat import lambda_abi
 from .compat import pip_import_string
 from .compat import pip_no_compile_c_env_vars
 from .compat import pip_no_compile_c_shim
@@ -59,10 +58,36 @@ class PackageDownloadError(PackagerError):
     pass
 
 
+class UnsupportedPythonVersion(PackagerError):
+    """Generic networking error during a package download."""
+    def __init__(self, version):
+        super(UnsupportedPythonVersion, self).__init__(
+            "'%s' version of python is not supported" % version
+        )
+
+
+def get_lambda_abi(runtime):
+    supported = {
+        "python2.7": "cp27mu",
+        "python3.6": "cp36m",
+        "python3.7": "cp37m"
+    }
+
+    if runtime not in supported:
+        raise UnsupportedPythonVersion(runtime)
+
+    return supported[runtime]
+
+
 class PythonPipDependencyBuilder(object):
-    def __init__(self, osutils=None, dependency_builder=None):
+    def __init__(self, runtime, osutils=None, dependency_builder=None):
         """Initialize a PythonPipDependencyBuilder.
 
+        :type runtime: str
+        :param runtime: Python version to build dependencies for. This can
+            either be python2.7, python3.6 or python3.7. These are currently the
+            only supported values.
+
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
@@ -76,11 +101,11 @@ def __init__(self, osutils=None, dependency_builder=None):
             self.osutils = OSUtils()
 
         if dependency_builder is None:
-            dependency_builder = DependencyBuilder(self.osutils)
+            dependency_builder = DependencyBuilder(self.osutils, runtime)
         self._dependency_builder = dependency_builder
 
     def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
-                           requirements_path, runtime, ui=None, config=None):
+                           requirements_path, ui=None, config=None):
         """Builds a python project's dependencies into an artifact directory.
 
         :type artifacts_dir_path: str
@@ -93,11 +118,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
         :param requirements_path: Path to a requirements.txt file to inspect
             for a list of dependencies.
 
-        :type runtime: str
-        :param runtime: Python version to build dependencies for. This can
-            either be python2.7 or python3.6. These are currently the only
-            supported values.
-
         :type ui: :class:`lambda_builders.utils.UI` or None
         :param ui: A class that traps all progress information such as status
             and errors. If injected by the caller, it can be used to monitor
@@ -141,13 +161,16 @@ class DependencyBuilder(object):
         'sqlalchemy'
     }
 
-    def __init__(self, osutils, pip_runner=None):
+    def __init__(self, osutils, runtime, pip_runner=None):
         """Initialize a DependencyBuilder.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
 
+        :type runtime: str
+        :param runtime: AWS Lambda Python runtime to build for
+
         :type pip_runner: :class:`PipRunner`
         :param pip_runner: This class is responsible for executing our pip
             on our behalf.
@@ -156,6 +179,7 @@ def __init__(self, osutils, pip_runner=None):
         if pip_runner is None:
             pip_runner = PipRunner(SubprocessPip(osutils))
         self._pip = pip_runner
+        self.runtime = runtime
 
     def build_site_packages(self, requirements_filepath,
                             target_directory,
@@ -312,8 +336,9 @@ def _download_all_dependencies(self, requirements_filename, directory):
     def _download_binary_wheels(self, packages, directory):
         # Try to get binary wheels for each package that isn't compatible.
         LOG.debug("Downloading missing wheels: %s", packages)
+        lambda_abi = get_lambda_abi(self.runtime)
         self._pip.download_manylinux_wheels(
-            [pkg.identifier for pkg in packages], directory)
+            [pkg.identifier for pkg in packages], directory, lambda_abi)
 
     def _build_sdists(self, sdists, directory, compile_c=True):
         LOG.debug("Build missing wheels from sdists "
@@ -341,6 +366,9 @@ def _is_compatible_wheel_filename(self, filename):
         # Verify platform is compatible
         if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM:
             return False
+
+        lambda_runtime_abi = get_lambda_abi(self.runtime)
+
         # Verify that the ABI is compatible with lambda. Either none or the
         # correct type for the python version cp27mu for py27 and cp36m for
         # py36.
@@ -351,7 +379,7 @@ def _is_compatible_wheel_filename(self, filename):
             # Deploying python 3 function which means we need cp36m abi
             # We can also accept abi3 which is the CPython 3 Stable ABI and
             # will work on any version of python 3.
-            return abi == 'cp36m' or abi == 'abi3'
+            return abi == lambda_runtime_abi or abi == 'abi3'
         elif prefix_version == 'cp2':
             # Deploying to python 2 function which means we need cp27mu abi
             return abi == 'cp27mu'
@@ -615,7 +643,7 @@ def download_all_dependencies(self, requirements_filename, directory):
             # complain at deployment time.
             self.build_wheel(wheel_package_path, directory)
 
-    def download_manylinux_wheels(self, packages, directory):
+    def download_manylinux_wheels(self, packages, directory, lambda_abi):
         """Download wheel files for manylinux for all the given packages."""
         # If any one of these dependencies fails pip will bail out. Since we
         # are only interested in all the ones we can download, we need to feed

tests/functional/workflows/python_pip/test_packager.py

@@ -1,3 +1,4 @@
+import sys
 import os
 import zipfile
 import tarfile
@@ -16,7 +17,7 @@
 from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher
 from aws_lambda_builders.workflows.python_pip.packager import \
     InvalidSourceDistributionNameError
-from aws_lambda_builders.workflows.python_pip.compat import lambda_abi
+from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi
 from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars
 from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim
 from aws_lambda_builders.workflows.python_pip.utils import OSUtils
@@ -214,7 +215,7 @@ def _write_requirements_txt(self, packages, directory):
     def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner):
         appdir = str(_create_app_structure(tmpdir))
         self._write_requirements_txt(reqs, appdir)
-        builder = DependencyBuilder(OSUtils(), runner)
+        builder = DependencyBuilder(OSUtils(), "python3.6", runner)
         return appdir, builder
 
     def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils):
@@ -644,7 +645,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', lambda_abi, '--dest', mock.ANY,
+                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
                 'bar==1.2'
             ],
             packages=[
@@ -677,7 +678,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner):
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', lambda_abi, '--dest', mock.ANY,
+                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
                 'sqlalchemy==1.1.18'
             ],
             packages=[
@@ -839,7 +840,7 @@ def test_build_into_existing_dir_with_preinstalled_packages(
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', lambda_abi, '--dest', mock.ANY,
+                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
                 'foo==1.2'
             ],
             packages=[

tests/integration/workflows/python_pip/test_python_pip.py

@@ -47,11 +47,9 @@ def test_must_build_python_project(self):
         self.assertEquals(expected_files, output_files)
 
     def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
-        self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
-                           runtime="python2.8")
-        expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
-        output_files = set(os.listdir(self.artifacts_dir))
-        self.assertEquals(expected_files, output_files)
+        with self.assertRaises(WorkflowFailedError):
+            self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
+                               runtime="python2.8")
 
     def test_must_fail_to_resolve_dependencies(self):
 

tests/unit/workflows/python_pip/test_actions.py

@@ -20,8 +20,7 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock):
 
         builder_instance.build_dependencies.assert_called_with("artifacts",
                                                                "scratch_dir",
-                                                               "manifest",
-                                                               "runtime")
+                                                               "manifest")
 
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
     def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):

tests/unit/workflows/python_pip/test_packager.py

@@ -1,4 +1,3 @@
-import sys
 from collections import namedtuple
 
 import mock
@@ -12,6 +11,7 @@
 from aws_lambda_builders.workflows.python_pip.packager import Package
 from aws_lambda_builders.workflows.python_pip.packager import PipRunner
 from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip
+from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi
 from aws_lambda_builders.workflows.python_pip.packager \
     import InvalidSourceDistributionNameError
 from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError
@@ -85,17 +85,29 @@ def popen(self, *args, **kwargs):
         return self._processes.pop()
 
 
+class TestGetLambdaAbi(object):
+    def test_get_lambda_abi_python27(self):
+        assert "cp27mu" == get_lambda_abi("python2.7")
+
+    def test_get_lambda_abi_python36(self):
+        assert "cp36m" == get_lambda_abi("python3.6")
+
+    def test_get_lambda_abi_python37(self):
+        assert "cp37m" == get_lambda_abi("python3.7")
+
+
 class TestPythonPipDependencyBuilder(object):
     def test_can_call_dependency_builder(self, osutils):
         mock_dep_builder = mock.Mock(spec=DependencyBuilder)
         osutils_mock = mock.Mock(spec=osutils)
         builder = PythonPipDependencyBuilder(
             osutils=osutils_mock,
             dependency_builder=mock_dep_builder,
+            runtime="runtime"
         )
         builder.build_dependencies(
             'artifacts/path/', 'scratch_dir/path/',
-            'path/to/requirements.txt', 'python3.6'
+            'path/to/requirements.txt'
         )
         mock_dep_builder.build_site_packages.assert_called_once_with(
             'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/')
@@ -218,14 +230,10 @@ def test_download_wheels(self, pip_factory):
         # for getting lambda compatible wheels.
         pip, runner = pip_factory()
         packages = ['foo', 'bar', 'baz']
-        runner.download_manylinux_wheels(packages, 'directory')
-        if sys.version_info[0] == 2:
-            abi = 'cp27mu'
-        else:
-            abi = 'cp36m'
+        runner.download_manylinux_wheels(packages, 'directory', "abi")
         expected_prefix = ['download', '--only-binary=:all:', '--no-deps',
                            '--platform', 'manylinux1_x86_64',
-                           '--implementation', 'cp', '--abi', abi,
+                           '--implementation', 'cp', '--abi', "abi",
                            '--dest', 'directory']
         for i, package in enumerate(packages):
             assert pip.calls[i].args == expected_prefix + [package]
@@ -234,7 +242,7 @@ def test_download_wheels(self, pip_factory):
 
     def test_download_wheels_no_wheels(self, pip_factory):
         pip, runner = pip_factory()
-        runner.download_manylinux_wheels([], 'directory')
+        runner.download_manylinux_wheels([], 'directory', "abi")
         assert len(pip.calls) == 0
 
     def test_raise_no_such_package_error(self, pip_factory):