cwltool: misc annotation improvements to fix mypy failing tests

includes few ignore types which I plan to remove in the future iteration

Author: manu-chroma

cwltool/builder.py

@@ -60,8 +60,8 @@ def __init__(self):  # type: () -> None
         self.job_script_provider = None  # type: Any
 
     def build_job_script(self, commands):
-        # type: (List[str]) -> Text
-        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[str]], Text]
+        # type: (List[bytes]) -> Text
+        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[bytes]], Text]
         if build_job_script_method:
             return build_job_script_method(self, commands)
         else:

cwltool/expression.py

@@ -22,7 +22,7 @@ def jshead(engineConfig, rootvars):
     # TODO: need to make sure the `rootvars dict`
     # contains no bytes type in the first place.
     if six.PY3:
-        rootvars = bytes2str_in_dicts(rootvars)  # type -> ignore
+        rootvars = bytes2str_in_dicts(rootvars)  # type: ignore
 
     return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 

cwltool/job.py

@@ -23,6 +23,7 @@
 from .pathmapper import PathMapper
 from .process import (UnsupportedRequirement, empty_subtree, get_feature,
                       stageFiles)
+from .utils import bytes2str_in_dicts
 
 _logger = logging.getLogger("cwltool")
 
@@ -223,6 +224,7 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
                 relink_initialworkdir(self.generatemapper, inplace_update=self.inplace_update)
 
             outputs = self.collect_outputs(self.outdir)
+            outputs = bytes2str_in_dicts(outputs)  # type: ignore
 
         except OSError as e:
             if e.errno == 2:
@@ -401,7 +403,7 @@ def run(self, pull_image=True, rm_container=True,
 
 
 def _job_popen(
-        commands,  # type: List[str]
+        commands,  # type: List[bytes]
         stdin_path,  # type: Text
         stdout_path,  # type: Text
         stderr_path,  # type: Text
@@ -464,9 +466,8 @@ def _job_popen(
             job_script_contents = SHELL_COMMAND_TEMPLATE
 
         env_copy = {}
-        key = None  # type: Union[Text, bytes]
+        key = None  # type: Any
         for key in env:
-            key = key.encode("utf-8")
             env_copy[key] = env[key]
 
         job_description = dict(

cwltool/load_tool.py

@@ -41,8 +41,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     # type: (...) -> Tuple[Loader, CommentedMap, Text]
     """Retrieve a CWL document."""
 
-    document_loader = Loader(jobloaderctx,
-                             fetcher_constructor=fetcher_constructor)
+    document_loader = Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap

cwltool/main.py

@@ -448,7 +448,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
 
     _jobloaderctx = jobloaderctx.copy()
     _jobloaderctx.update(t.metadata.get("$namespaces", {}))
-    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
+    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     if len(args.job_order) == 1 and args.job_order[0][0] != "-":
         job_order_file = args.job_order[0]

cwltool/stdfsaccess.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import glob
 import os
-from typing import BinaryIO, List, Text, IO
+from typing import BinaryIO, List, Union, Text, IO, overload
 
 import six
 from six.moves import urllib
@@ -24,7 +24,17 @@ def _abs(self, p):  # type: (Text) -> Text
     def glob(self, pattern):  # type: (Text) -> List[Text]
         return [file_uri(str(self._abs(l))) for l in glob.glob(self._abs(pattern))]
 
-    def open(self, fn, mode):  # type: (Text, str) -> IO[bytes]
+    # overload is related to mypy type checking and in no way
+    # modifies the behaviour of the function.
+    @overload
+    def open(self, fn, mode='rb'):  # type: (Text, str) -> IO[bytes]
+        pass
+
+    @overload
+    def open(self, fn, mode='r'):  # type: (Text, str) -> IO[str]
+        pass
+
+    def open(self, fn, mode):
         return open(self._abs(fn), mode)
 
     def exists(self, fn):  # type: (Text) -> bool