Fix backend python issues

Author: Eric Arellano

src/python/pants/backend/python/tasks/pytest_run.py

@@ -240,7 +240,7 @@ def _cov_setup(self, workdirs, coverage_morfs, src_to_target_base):
     # Note that it's important to put the tmpfile under the workdir, because pytest
     # uses all arguments that look like paths to compute its rootdir, and we want
     # it to pick the buildroot.
-    with temporary_file(root_dir=workdirs.root_dir) as fp:
+    with temporary_file(root_dir=workdirs.root_dir, binary_mode=False) as fp:
       cp.write(fp)
       fp.close()
       coverage_rc = fp.name

src/python/pants/backend/python/tasks/python_execution_task_base.py

@@ -7,6 +7,7 @@
 import os
 from builtins import str
 
+from future.utils import binary_type, text_type
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
 
@@ -46,7 +47,7 @@ def extra_requirements(self):
     """
     return ()
 
-  class ExtraFile(datatype(['path', 'content'])):
+  class ExtraFile(datatype([('path', text_type), ('content', binary_type)])):
     """Models an extra file to place in a PEX."""
 
     @classmethod
@@ -56,7 +57,7 @@ def empty(cls, path):
       :param str path: The path this extra file should have when added to a PEX.
       :rtype: :class:`ExtraFile`
       """
-      return cls(path=path, content='')
+      return cls(path=path, content=b'')
 
     def add_to(self, builder):
       """Adds this extra file to a PEX builder.

src/python/pants/backend/python/tasks/select_interpreter.py

@@ -34,7 +34,7 @@ def compute_fingerprint(self, python_target):
       return None
     hasher = hashlib.sha1()
     for element in hash_elements_for_target:
-      hasher.update(element)
+      hasher.update(element.encode('utf-8'))
     return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8')
 
 

src/python/pants/backend/python/tasks/setup_py.py

@@ -13,7 +13,7 @@
 from builtins import map, object, str, zip
 from collections import OrderedDict, defaultdict
 
-from pex.compatibility import string, to_bytes
+from future.utils import PY2, PY3
 from pex.installer import InstallerBase, Packager
 from pex.interpreter import PythonInterpreter
 from twitter.common.collections import OrderedSet
@@ -300,7 +300,7 @@ def collect_reduced_dependencies(current):
 class SetupPy(Task):
   """Generate setup.py-based Python projects."""
 
-  SOURCE_ROOT = b'src'
+  SOURCE_ROOT = 'src' if PY3 else b'src'
 
   PYTHON_DISTS_PRODUCT = 'python_dists'
 
@@ -514,18 +514,23 @@ def write_setup(self, root_target, reduced_dependencies, chroot):
 
     setup_keywords = root_target.provides.setup_py_keywords.copy()
 
-    package_dir = {b'': self.SOURCE_ROOT}
+    package_dir = {'' if PY3 else b'': self.SOURCE_ROOT}
     packages, namespace_packages, resources = self.find_packages(chroot, self.context.log)
 
     if namespace_packages:
       setup_keywords['namespace_packages'] = list(sorted(namespace_packages))
 
     if packages:
+      normalized_package_data = (
+        resources.items()
+        if PY3 else
+        ((package.encode('utf-8'), [v.encode('utf-8') for v in rs])
+         for (package, rs) in resources.items())
+      )
       setup_keywords.update(
           package_dir=package_dir,
           packages=list(sorted(packages)),
-          package_data=dict((str(package), list(map(str, rs)))
-                            for (package, rs) in resources.items()))
+          package_data=dict(normalized_package_data))
 
     setup_keywords['install_requires'] = list(self.install_requires(reduced_dependencies))
 
@@ -546,8 +551,8 @@ def convert(input):
         return out
       elif isinstance(input, list):
         return [convert(element) for element in input]
-      elif isinstance(input, string):
-        return to_bytes(input)
+      elif PY2 and isinstance(input, str):
+        return input.encode('utf-8')
       else:
         return input
 
@@ -571,7 +576,7 @@ def convert(input):
     chroot.write(self._setup_boilerplate().format(
       setup_dict=pprint.pformat(convert(setup_keywords), indent=4),
       setup_target=repr(root_target)
-    ), 'setup.py')
+    ).encode('utf-8'), 'setup.py')
 
     # make sure that setup.py is included
     chroot.write('include *.py'.encode('utf8'), 'MANIFEST.in')

tests/python/pants_test/backend/python/tasks/test_pytest_run.py

@@ -479,7 +479,7 @@ def load_coverage_data_for(self, context, covered_path, expect_coverage=True):
       PytestRun._add_plugin_config(cp,
                                    src_chroot_path=src_chroot_path,
                                    src_to_target_base=src_to_target_base)
-      with temporary_file() as fp:
+      with temporary_file(binary_mode=False) as fp:
         cp.write(fp)
         fp.close()
 

tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py

@@ -41,7 +41,7 @@ def test_pants_binary(self):
       # Check that the pex was built.
       self.assertTrue(os.path.isfile(pex))
       # Check that the pex runs.
-      output = subprocess.check_output(pex)
+      output = subprocess.check_output(pex).decode('utf-8')
       self._assert_native_greeting(output)
       # Check that we have exact one wheel output
       self.assertEqual(len(glob.glob(wheel_glob)), 1)
@@ -141,11 +141,11 @@ def test_pants_binary_dep_isolation_with_multiple_targets(self):
       self.assertTrue(os.path.isfile(pex1))
       self.assertTrue(os.path.isfile(pex2))
       # Check that the pex 1 runs.
-      output = subprocess.check_output(pex1)
+      output = subprocess.check_output(pex1).decode('utf-8')
       self._assert_native_greeting(output)
       # Check that the pex 2 fails due to no python_dists leaked into it.
       try:
-        output = subprocess.check_output(pex2)
+        subprocess.check_output(pex2)
       except subprocess.CalledProcessError as e:
         self.assertNotEqual(0, e.returncode)
 
@@ -172,7 +172,7 @@ def test_pants_resolves_local_dists_for_current_platform_only(self):
       # Check that the pex was built.
       self.assertTrue(os.path.isfile(pex))
       # Check that the pex runs.
-      output = subprocess.check_output(pex)
+      output = subprocess.check_output(pex).decode('utf-8')
       self._assert_native_greeting(output)
 
   def _get_current_platform_string(self):
@@ -229,7 +229,7 @@ def test_python_distribution_with_setup_requires(self):
       # Check that the pex was built.
       self.assertTrue(os.path.isfile(pex))
       # Check that the pex runs.
-      output = subprocess.check_output(pex)
+      output = subprocess.check_output(pex).decode('utf-8')
       self.assertIn('Hello, world!', output)
     finally:
       if os.path.exists(pex):

tests/python/pants_test/backend/python/tasks/test_resolve_requirements.py

@@ -7,6 +7,7 @@
 import os
 from builtins import str
 
+from future.utils import PY3
 from pex.interpreter import PythonInterpreter
 
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
@@ -34,7 +35,12 @@ def test_resolve_simple_requirements(self):
     # the requirement isn't sneaking in some other way, which would render the remainder
     # of this test moot.)
     _, stderr_data = self._exercise_module(self._resolve_requirements([noreqs_tgt]), 'colors')
-    self.assertIn('ImportError: No module named colors', stderr_data)
+
+    try:
+      self.assertIn("ModuleNotFoundError: No module named 'colors'", stderr_data)
+    except AssertionError:
+      # < Python 3.6 uses ImportError instead of ModuleNotFoundError.
+      self.assertIn('ImportError: No module named colors', stderr_data)
 
     # Check that the module is available if specified as a requirement.
     stdout_data, stderr_data = self._exercise_module(self._resolve_requirements([ansicolors_tgt]),
@@ -43,7 +49,7 @@ def test_resolve_simple_requirements(self):
 
     path = stdout_data.strip()
     # Check that the requirement resolved to what we expect.
-    self.assertTrue(path.endswith('/.deps/ansicolors-1.0.2-py2-none-any.whl/colors.py'))
+    self.assertTrue(path.endswith('/.deps/ansicolors-1.0.2-{}-none-any.whl/colors.py'.format('py3' if PY3 else 'py2')))
     # Check that the path is under the test's build root, so we know the pex was created there.
     self.assertTrue(path.startswith(os.path.realpath(get_buildroot())))
 
@@ -136,9 +142,10 @@ def _resolve_requirements(self, target_roots, options=None):
       return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
 
   def _exercise_module(self, pex, expected_module):
-    with temporary_file() as f:
+    with temporary_file(binary_mode=False) as f:
       f.write('import {m}; print({m}.__file__)'.format(m=expected_module))
       f.close()
       proc = pex.run(args=[f.name], blocking=False,
                      stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      return proc.communicate()
+      stdout, stderr = proc.communicate()
+      return (stdout.decode('utf-8'), stderr.decode('utf-8'))

tests/python/pants_test/backend/python/tasks/test_setup_py.py

@@ -601,7 +601,7 @@ def test_prep_command_case(self):
 
 def test_detect_namespace_packages():
   def has_ns(stmt):
-    with temporary_file() as fp:
+    with temporary_file(binary_mode=False) as fp:
       fp.write(stmt)
       fp.flush()
       return SetupPy.declares_namespace_package(fp.name)

tests/python/pants_test/backend/python/test_interpreter_cache.py

@@ -9,6 +9,7 @@
 from contextlib import contextmanager
 
 import mock
+from future.utils import PY3
 from pex.package import EggPackage, Package, SourcePackage
 from pex.resolver import Unsatisfiable, resolve
 
@@ -28,7 +29,7 @@ def _make_bad_requirement(requirement):
 
     E.g. 'CPython==2.7.5' becomes 'CPython==99.7.5'
     """
-    return str(requirement).replace('==2.', '==99.')
+    return str(requirement).replace('==3', '==99') if PY3 else str(requirement).replace('==2.', '==99')
 
   def setUp(self):
     super(TestInterpreterCache, self).setUp()