[MRG] turn on Python 3 on travis CI

.travis.yml

@@ -2,17 +2,17 @@ sudo: false
 language: python
 python:
   - 2.7
-    #  - 3.4
+  - 3.3
+  - 3.4
   - 3.5
-    #  - 3.6
+  - 3.6
 os:
   - linux
-
 install:
   - pip install tox-travis
 script: tox
-notifications:
-  email: false
 branches:
   only:
     - master
+notifications:
+  email: false

Makefile

@@ -161,7 +161,7 @@ mypy2: ${PYSOURCES}
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
 	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores \
+		 --warn-redundant-casts \
 		 cwltool
 
 mypy3: ${PYSOURCES}
@@ -172,7 +172,7 @@ mypy3: ${PYSOURCES}
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
 	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores \
+		 --warn-redundant-casts \
 		 cwltool
 
 FORCE:

cwltool/builder.py

@@ -60,8 +60,8 @@ def __init__(self):  # type: () -> None
         self.job_script_provider = None  # type: Any
 
     def build_job_script(self, commands):
-        # type: (List[str]) -> Text
-        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[str]], Text]
+        # type: (List[bytes]) -> Text
+        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[bytes]], Text]
         if build_job_script_method:
             return build_job_script_method(self, commands)
         else:

cwltool/docker.py

@@ -5,6 +5,7 @@
 import subprocess
 import sys
 import tempfile
+from io import open
 from typing import Dict, List, Text
 
 import requests
@@ -56,7 +57,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                 found = True
         elif "dockerFile" in dockerRequirement:
             dockerfile_dir = str(tempfile.mkdtemp())
-            with open(os.path.join(dockerfile_dir, "Dockerfile"), str("w")) as df:
+            with open(os.path.join(dockerfile_dir, "Dockerfile"), "wb") as df:
                 df.write(dockerRequirement["dockerFile"].encode('utf-8'))
             cmd = ["docker", "build", "--tag=%s" %
                    str(dockerRequirement["dockerImageId"]), dockerfile_dir]
@@ -70,7 +71,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
                     _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
-                    with open(dockerRequirement["dockerLoad"], str("rb")) as f:
+                    with open(dockerRequirement["dockerLoad"], "rb") as f:
                         loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)

cwltool/expression.py

@@ -22,7 +22,7 @@ def jshead(engineConfig, rootvars):
     # TODO: need to make sure the `rootvars dict`
     # contains no bytes type in the first place.
     if six.PY3:
-        rootvars = bytes2str_in_dicts(rootvars)  # type -> ignore
+        rootvars = bytes2str_in_dicts(rootvars)  # type: ignore
 
     return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 

cwltool/job.py

@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import tempfile
+from io import open
 from typing import (IO, Any, Callable, Dict, Iterable, List, MutableMapping, Text,
                     Tuple, Union, cast)
 
@@ -22,6 +23,7 @@
 from .pathmapper import PathMapper
 from .process import (UnsupportedRequirement, empty_subtree, get_feature,
                       stageFiles)
+from .utils import bytes2str_in_dicts
 
 _logger = logging.getLogger("cwltool")
 
@@ -222,6 +224,7 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
                 relink_initialworkdir(self.generatemapper, inplace_update=self.inplace_update)
 
             outputs = self.collect_outputs(self.outdir)
+            outputs = bytes2str_in_dicts(outputs)  # type: ignore
 
         except OSError as e:
             if e.errno == 2:
@@ -400,7 +403,7 @@ def run(self, pull_image=True, rm_container=True,
 
 
 def _job_popen(
-        commands,  # type: List[str]
+        commands,  # type: List[bytes]
         stdin_path,  # type: Text
         stdout_path,  # type: Text
         stderr_path,  # type: Text
@@ -463,9 +466,8 @@ def _job_popen(
             job_script_contents = SHELL_COMMAND_TEMPLATE
 
         env_copy = {}
-        key = None  # type: Union[Text, bytes]
+        key = None  # type: Any
         for key in env:
-            key = key.encode("utf-8")
             env_copy[key] = env[key]
 
         job_description = dict(

cwltool/load_tool.py

@@ -41,8 +41,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     # type: (...) -> Tuple[Loader, CommentedMap, Text]
     """Retrieve a CWL document."""
 
-    document_loader = Loader(jobloaderctx,
-                             fetcher_constructor=fetcher_constructor)
+    document_loader = Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap

cwltool/main.py

@@ -504,7 +504,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
 
     _jobloaderctx = jobloaderctx.copy()
     _jobloaderctx.update(t.metadata.get("$namespaces", {}))
-    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
+    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     if len(args.job_order) == 1 and args.job_order[0][0] != "-":
         job_order_file = args.job_order[0]

cwltool/process.py

@@ -13,6 +13,7 @@
 import tempfile
 import uuid
 from collections import Iterable
+from io import open
 from functools import cmp_to_key
 from typing import (Any, Callable, Dict, Generator, List, Set, Text,
                     Tuple, Union, cast)

cwltool/stdfsaccess.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import glob
 import os
-from typing import BinaryIO, List, Text, IO
+from typing import BinaryIO, List, Union, Text, IO, overload
 
 import six
 from six.moves import urllib
@@ -24,7 +24,17 @@ def _abs(self, p):  # type: (Text) -> Text
     def glob(self, pattern):  # type: (Text) -> List[Text]
         return [file_uri(str(self._abs(l))) for l in glob.glob(self._abs(pattern))]
 
-    def open(self, fn, mode):  # type: (Text, str) -> IO[bytes]
+    # overload is related to mypy type checking and in no way
+    # modifies the behaviour of the function.
+    @overload
+    def open(self, fn, mode='rb'):  # type: (Text, str) -> IO[bytes]
+        pass
+
+    @overload
+    def open(self, fn, mode='r'):  # type: (Text, str) -> IO[str]
+        pass
+
+    def open(self, fn, mode):
         return open(self._abs(fn), mode)
 
     def exists(self, fn):  # type: (Text) -> bool

tests/test_examples.py

@@ -118,7 +118,7 @@ class TestFactory(unittest.TestCase):
     def test_factory(self):
         f = cwltool.factory.Factory()
         echo = f.make(get_data("tests/echo.cwl"))
-        self.assertEqual(echo(inp="foo"), {"out": b"foo\n"})
+        self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 
     def test_partial_scatter(self):
         f = cwltool.factory.Factory(on_error="continue")

tox.ini

@@ -1,14 +1,33 @@
 [tox]
-envlist = py27-lint, py27-unit, py27-pipconflictchecker, py36-lint, py35-lint, py34-lint, py33-lint, py35-mypy2, py27-lint-readme, py27-pydocstyle
+envlist =
+  py{27,33,34,35,36}-lint,
+  py{27,33,34,35,36}-unit,
+  py35-mypy{2,3},
+  py27-pipconflictchecker,
+  py27-lint-readme,
+  py27-pydocstyle
+
 skipsdist = True
+skip_missing_interpreters = True
 
-[travis]
-python =
- 2.7: py27
- 3.5: py35-mypy2
+[tox:travis]
+2.7 = py27
+3.3 = py33
+3.4 = py34
+3.5 = py35
+3.6 = py36
 
 [testenv]
-deps = -rrequirements.txt
+deps =
+  -rrequirements.txt
+  py{27,33,34,35,36}-lint: flake8
+
+commands =
+  py{27,33,34,35,36}-unit: python setup.py test
+  py{27,33,34,35,36}-lint: flake8 schema_salad setup.py
+
+whitelist_externals =
+  py{27,33,34,35,36}-lint: flake8
 
 [testenv:py35-mypy2]
 commands = make mypy2
@@ -24,11 +43,6 @@ deps =
   mypy==0.520
   -rrequirements.txt
 
-[testenv:py27-lint]
-commands = flake8 cwltool setup.py
-whitelist_externals = flake8
-deps = flake8
-
 [testenv:py27-pipconflictchecker]
 commands = pipconflictchecker
 whitelist_externals = pipconflictchecker

typeshed/2.7/avro/schema.pyi → typeshed/2and3/avro/schema.pyi

@@ -2,7 +2,7 @@
 #
 # NOTE: This dynamically typed stub was automatically generated by stubgen.
 
-from typing import Any, Union, Dict, List
+from typing import Any, Union, Dict, List, Text
 
 PRIMITIVE_TYPES = ... # type: Any
 NAMED_TYPES = ... # type: Any
@@ -32,9 +32,9 @@ class Name:
 class Names:
     names = ... # type: Any
     default_namespace = ... # type: Any
-    def __init__(self, default_namespace: Union[str, unicode] = None) -> None: ...
-    def has_name(self, name_attr: Union[str, unicode], space_attr: Union[str, unicode]) -> bool: ...
-    def get_name(self, name_attr: Union[str, unicode], space_attr: Union[str, unicode]) -> Schema: ...
+    def __init__(self, default_namespace: Union[str, Text] = None) -> None: ...
+    def has_name(self, name_attr: Union[str, Text], space_attr: Union[str, Text]) -> bool: ...
+    def get_name(self, name_attr: Union[str, Text], space_attr: Union[str, Text]) -> Schema: ...
     def prune_namespace(self, properties): ...
     def add_name(self, name_attr, space_attr, new_schema): ...