Merge pull request #306 from common-workflow-language/gijzelaerr-py3

py3

Author: mr-c

.travis.yml

@@ -2,7 +2,9 @@ sudo: false
 language: python
 python:
   - 2.7
+    #  - 3.4
   - 3.5
+    #  - 3.6
 os:
   - linux
 

Makefile

@@ -158,4 +158,15 @@ mypy: ${PYSOURCES}
 		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
 		 cwltool
 
+mypy3: ${PYSOURCES}
+	rm -Rf typeshed/3/ruamel/yaml
+	ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
+		typeshed/3/ruamel/yaml
+	rm -Rf typeshed/3/schema_salad
+	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
+		typeshed/3/schema_salad
+	MYPYPATH=typeshed/3 mypy --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		 cwltool
+
 FORCE:

cwltool/builder.py

@@ -4,6 +4,7 @@
 import schema_salad.validate as validate
 from schema_salad.sourceline import SourceLine
 from typing import Any, Callable, Text, Type, Union
+from six import string_types, iteritems
 
 from . import expression
 from .errors import WorkflowException
@@ -121,7 +122,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
                     for sf in aslist(schema["secondaryFiles"]):
                         if isinstance(sf, dict) or "$(" in sf or "${" in sf:
                             secondary_eval = self.do_eval(sf, context=datum)
-                            if isinstance(secondary_eval, basestring):
+                            if isinstance(secondary_eval, string_types):
                                 sfpath = {"location": secondary_eval,
                                           "class": "File"}
                             else:
@@ -203,7 +204,7 @@ def do_eval(self, ex, context=None, pull_image=True, recursive=False):
         # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
         if recursive:
             if isinstance(ex, dict):
-                return {k: self.do_eval(v, context, pull_image, recursive) for k, v in ex.iteritems()}
+                return {k: self.do_eval(v, context, pull_image, recursive) for k, v in iteritems(ex)}
             if isinstance(ex, list):
                 return [self.do_eval(v, context, pull_image, recursive) for v in ex]
 

cwltool/cwlrdf.py

@@ -1,9 +1,8 @@
-import urlparse
-
 from rdflib import Graph
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import ContextType
 from typing import Any, Dict, IO, Text
+from six.moves import urllib
 
 from .process import Process
 
@@ -133,7 +132,7 @@ def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
                 currentwf = None
 
         if Text(runtype) != "https://w3id.org/cwl/cwl#Workflow":
-            stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(Text(step))[1]))
+            stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urllib.parse.urldefrag(Text(step))[1]))
 
     if currentwf is not None:
         stdout.write("}\n")

cwltool/docker_uid.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import subprocess
 
 from typing import Text
@@ -111,4 +112,4 @@ def docker_machine_uid():  # type: () -> int
 
 
 if __name__ == '__main__':
-    print docker_vm_uid()
+    print(docker_vm_uid())

cwltool/draft2tool.py

@@ -6,7 +6,8 @@
 import re
 import shutil
 import tempfile
-import urlparse
+from six.moves import urllib
+from six import string_types, u
 from functools import partial
 
 import schema_salad.validate as validate
@@ -95,7 +96,7 @@ def revmap_file(builder, outdir, f):
     internal output directories to the external directory.
     """
 
-    split = urlparse.urlsplit(outdir)
+    split = urllib.parse.urlsplit(outdir)
     if not split.scheme:
         outdir = file_uri(str(outdir))
 
@@ -346,7 +347,7 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                         ls.append(builder.do_eval(t))
             for i, t in enumerate(ls):
                 if "entry" in t:
-                    if isinstance(t["entry"], basestring):
+                    if isinstance(t["entry"], string_types):
                         ls[i] = {
                             "class": "File",
                             "basename": t["entryname"],
@@ -415,7 +416,7 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
                                 % shortname(port["id"]), exc_info=True)
                             raise WorkflowException(
                                 u"Error collecting output for parameter '%s':\n%s"
-                                % (shortname(port["id"]), indent(unicode(e))))
+                                % (shortname(port["id"]), indent(u(str(e)))))
 
             if ret:
                 adjustFileObjs(ret,
@@ -527,7 +528,7 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                             for sf in aslist(schema["secondaryFiles"]):
                                 if isinstance(sf, dict) or "$(" in sf or "${" in sf:
                                     sfpath = builder.do_eval(sf, context=primary)
-                                    if isinstance(sfpath, basestring):
+                                    if isinstance(sfpath, string_types):
                                         sfpath = revmap({"location": sfpath, "class": "File"})
                                 else:
                                     sfpath = {"location": substitute(primary["location"], sf), "class": "File"}

cwltool/expression.py

@@ -4,6 +4,7 @@
 import re
 
 from typing import Any, AnyStr, Union, Text, Dict, List
+from six import u
 
 from . import sandboxjs
 from .errors import WorkflowException
@@ -24,7 +25,7 @@ def jshead(engineConfig, rootvars):
 segment_re = re.compile(segments, flags=re.UNICODE)
 param_re = re.compile(r"\((%s)%s*\)$" % (seg_symbol, segments), flags=re.UNICODE)
 
-JSON = Union[Dict[Any, Any], List[Any], Text, int, long, float, bool, None]
+JSON = Union[Dict[Any, Any], List[Any], Text, int, float, bool, None]
 
 
 class SubstitutionError(Exception):
@@ -126,7 +127,7 @@ def next_seg(remain, obj):  # type: (Text, Any) -> Any
             try:
                 key = int(m.group(0)[1:-1])
             except ValueError as v:
-                raise WorkflowException(unicode(v))
+                raise WorkflowException(u(str(v)))
             if not isinstance(obj, list):
                 raise WorkflowException(" is a %s, cannot index on int '%s'" % (type(obj).__name__, key))
             if key >= len(obj):

cwltool/load_tool.py

@@ -4,7 +4,6 @@
 import logging
 import os
 import re
-import urlparse
 import uuid
 
 import requests.sessions
@@ -15,6 +14,8 @@
 from schema_salad.sourceline import cmap
 from schema_salad.validate import ValidationException
 from typing import Any, Callable, cast, Dict, Text, Tuple, Union
+from six.moves import urllib
+from six import itervalues, string_types
 
 from . import process
 from . import update
@@ -37,8 +38,8 @@ def fetch_document(argsworkflow,  # type: Union[Text, dict[Text, Any]]
 
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap
-    if isinstance(argsworkflow, basestring):
-        split = urlparse.urlsplit(argsworkflow)
+    if isinstance(argsworkflow, string_types):
+        split = urllib.parse.urlsplit(argsworkflow)
         if split.scheme:
             uri = argsworkflow
         elif os.path.exists(os.path.abspath(argsworkflow)):
@@ -52,7 +53,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, dict[Text, Any]]
         if argsworkflow != uri:
             _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
 
-        fileuri = urlparse.urldefrag(uri)[0]
+        fileuri = urllib.parse.urldefrag(uri)[0]
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
         uri = "#" + Text(id(argsworkflow))
@@ -101,7 +102,7 @@ def _convert_stdstreams_to_files(workflowobj):
                                 inp['id'].rpartition('#')[2]
                             inp['type'] = 'File'
         else:
-            for entry in workflowobj.itervalues():
+            for entry in itervalues(workflowobj):
                 _convert_stdstreams_to_files(entry)
     if isinstance(workflowobj, list):
         for entry in workflowobj:
@@ -131,11 +132,11 @@ def validate_document(document_loader,  # type: Loader
     jobobj = None
     if "cwl:tool" in workflowobj:
         jobobj, _ = document_loader.resolve_all(workflowobj, uri)
-        uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
+        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
         del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
         workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]
 
-    fileuri = urlparse.urldefrag(uri)[0]
+    fileuri = urllib.parse.urldefrag(uri)[0]
 
     if "cwlVersion" in workflowobj:
         if not isinstance(workflowobj["cwlVersion"], (str, Text)):
@@ -213,7 +214,7 @@ def make_tool(document_loader,  # type: Loader
             raise WorkflowException(
                 u"Tool file contains graph of multiple objects, must specify "
                 "one of #%s" % ", #".join(
-                    urlparse.urldefrag(i["id"])[1] for i in resolveduri
+                    urllib.parse.urldefrag(i["id"])[1] for i in resolveduri
                     if "id" in i))
     elif isinstance(resolveduri, dict):
         processobj = resolveduri

cwltool/main.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 
 import argparse
 import functools
@@ -469,7 +470,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
 
     if not job_order_object and len(t.tool["inputs"]) > 0:
         if toolparser:
-            print u"\nOptions for %s " % args.workflow
+            print(u"\nOptions for {} ".format(args.workflow))
             toolparser.print_help()
         _logger.error("")
         _logger.error("Input object required, use --help for details")
@@ -619,7 +620,7 @@ def main(argsl=None,  # type: List[str]
             _logger.setLevel(logging.DEBUG)
 
         if args.version:
-            print versionfunc()
+            print(versionfunc())
             return 0
         else:
             _logger.info(versionfunc())

cwltool/pack.py

@@ -1,8 +1,8 @@
 import copy
-import urlparse
 
 from schema_salad.ref_resolver import Loader
 from typing import Union, Any, cast, Callable, Dict, Text
+from six.moves import urllib
 
 from .process import shortname, uniquename
 
@@ -82,7 +82,7 @@ def loadref(b, u):
     names = set()  # type: Set[Text]
     rewrite = {}  # type: Dict[Text, Text]
 
-    mainpath, _ = urlparse.urldefrag(uri)
+    mainpath, _ = urllib.parse.urldefrag(uri)
 
     def rewrite_id(r, mainuri):
         # type: (Text, Text) -> None
@@ -91,7 +91,7 @@ def rewrite_id(r, mainuri):
         elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"):
             pass
         else:
-            path, frag = urlparse.urldefrag(r)
+            path, frag = urllib.parse.urldefrag(r)
             if path == mainpath:
                 rewrite[r] = "#" + uniquename(frag, names)
             else:

cwltool/pathmapper.py

@@ -2,15 +2,14 @@
 import logging
 import os
 import stat
-import urllib
-import urlparse
 import uuid
 from functools import partial
 
 import schema_salad.validate as validate
 from schema_salad.ref_resolver import uri_file_path
 from schema_salad.sourceline import SourceLine
 from typing import Any, Callable, Set, Text, Tuple, Union
+from six.moves import urllib
 
 _logger = logging.getLogger("cwltool")
 
@@ -71,8 +70,8 @@ def addLocation(d):
                 d["basename"] = Text(uuid.uuid4())
 
         if "basename" not in d:
-            parse = urlparse.urlparse(d["location"])
-            d["basename"] = os.path.basename(urllib.url2pathname(parse.path))
+            parse = urllib.parse.urlparse(d["location"])
+            d["basename"] = os.path.basename(urllib.request.url2pathname(parse.path))
 
     adjustFileObjs(job, addLocation)
     adjustDirObjs(job, addLocation)

cwltool/process.py

@@ -181,7 +181,7 @@ def stageFiles(pm, stageFunc, ignoreWritable=False):
     # type: (PathMapper, Callable[..., Any], bool) -> None
     for f, p in pm.items():
         if not os.path.exists(os.path.dirname(p.target)):
-            os.makedirs(os.path.dirname(p.target), 0755)
+            os.makedirs(os.path.dirname(p.target), 0o0755)
         if p.type == "File":
             stageFunc(p.resolved, p.target)
         elif p.type == "WritableFile" and not ignoreWritable:

cwltool/sandboxjs.py

@@ -1,24 +1,22 @@
-import cStringIO
 import errno
 import json
 import logging
 import os
 import select
 import subprocess
 import threading
-from cStringIO import StringIO
+from io import BytesIO
 
 from pkg_resources import resource_stream
 from typing import Any, Dict, List, Mapping, Text, Union
 
-
 class JavascriptException(Exception):
     pass
 
 
 _logger = logging.getLogger("cwltool")
 
-JSON = Union[Dict[Text, Any], List[Any], Text, int, long, float, bool, None]
+JSON = Union[Dict[Text, Any], List[Any], Text, int, float, bool, None]
 
 localdata = threading.local()
 
@@ -101,11 +99,11 @@ def term():
     tm = threading.Timer(timeout, term)
     tm.start()
 
-    stdin_buf = StringIO(json.dumps(fn) + "\n")
-    stdout_buf = StringIO()
-    stderr_buf = StringIO()
+    stdin_buf = BytesIO(json.dumps(fn) + "\n")
+    stdout_buf = BytesIO()
+    stderr_buf = BytesIO()
 
-    completed = []  # type: List[Union[cStringIO.InputType, cStringIO.OutputType]]
+    completed = []  # type: List[BytesIO]
     while len(completed) < 3:
         rready, wready, _ = select.select([nodejs.stdout, nodejs.stderr], [nodejs.stdin], [])
         if nodejs.stdin in wready:

setup.py

@@ -51,7 +51,9 @@
           'rdflib >= 4.2.2, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
           'schema-salad >= 2.2.20170222151604, < 3',
-          'typing >= 3.5.2, < 3.6'
+          'typing >= 3.5.2, < 3.6',
+          'six >= 1.10.0',
+
       ],
       setup_requires=[] + pytest_runner,
       test_suite='tests',