Use unicode strings instead of ASCII strings.

Author: Peter Amstutz

cwltool/builder.py

@@ -54,7 +54,7 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
                     return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
-            raise validate.ValidationException("'%s' is not a valid union %s" % (datum, schema["type"]))
+            raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
         elif isinstance(schema["type"], dict):
             st = copy.deepcopy(schema["type"])
             if binding and "inputBinding" not in st and "itemSeparator" not in binding and st["type"] in ("array", "map"):
@@ -129,7 +129,7 @@ def _capture_files(f):
     def tostr(self, value):
         if isinstance(value, dict) and value.get("class") == "File":
             if "path" not in value:
-                raise WorkflowException("File object must have \"path\": %s" % (value))
+                raise WorkflowException(u"File object must have \"path\": %s" % (value))
             return value["path"]
         else:
             return str(value)

cwltool/cwlrdf.py

@@ -11,7 +11,7 @@ def makerdf(workflow, wf, ctx):
         doc_url, frg = urlparse.urldefrag(v)
         if "/" in frg:
             p, _ = frg.split("/")
-            prefixes[p] = "%s#%s/" % (doc_url, p)
+            prefixes[p] = u"%s#%s/" % (doc_url, p)
 
     wf["@context"] = ctx
     g = Graph().parse(data=json.dumps(wf), format='json-ld', location=workflow)
@@ -45,7 +45,7 @@ def dot_with_parameters(g, stdout):
            }""")
 
     for step, run, runtype in qres:
-        stdout.write('"%s" [label="%s"]\n' % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run))))
+        stdout.write(u'"%s" [label="%s"]\n' % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run))))
 
     qres = g.query(
         """SELECT ?step ?inp ?source
@@ -56,9 +56,9 @@ def dot_with_parameters(g, stdout):
            }""")
 
     for step, inp, source in qres:
-        stdout.write('"%s" [shape=box]\n' % (lastpart(inp)))
-        stdout.write('"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(inp), ""))
-        stdout.write('"%s" -> "%s" [label="%s"]\n' % (lastpart(inp), lastpart(step), ""))
+        stdout.write(u'"%s" [shape=box]\n' % (lastpart(inp)))
+        stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(inp), ""))
+        stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(inp), lastpart(step), ""))
 
     qres = g.query(
         """SELECT ?step ?out
@@ -68,8 +68,8 @@ def dot_with_parameters(g, stdout):
            }""")
 
     for step, out in qres:
-        stdout.write('"%s" [shape=box]\n' % (lastpart(out)))
-        stdout.write('"%s" -> "%s" [label="%s"]\n' % (lastpart(step), lastpart(out), ""))
+        stdout.write(u'"%s" [shape=box]\n' % (lastpart(out)))
+        stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(step), lastpart(out), ""))
 
     qres = g.query(
         """SELECT ?out ?source
@@ -79,8 +79,8 @@ def dot_with_parameters(g, stdout):
            }""")
 
     for out, source in qres:
-        stdout.write('"%s" [shape=octagon]\n' % (lastpart(out)))
-        stdout.write('"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(out), ""))
+        stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(out)))
+        stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(out), ""))
 
     qres = g.query(
         """SELECT ?inp
@@ -90,7 +90,7 @@ def dot_with_parameters(g, stdout):
            }""")
 
     for (inp,) in qres:
-        stdout.write('"%s" [shape=octagon]\n' % (lastpart(inp)))
+        stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp)))
 
 def dot_without_parameters(g, stdout):
     dotname = {}
@@ -130,14 +130,14 @@ def dot_without_parameters(g, stdout):
             if wf in subworkflows:
                 if wf not in dotname:
                     dotname[wf] = "cluster_" + lastpart(wf)
-                stdout.write('subgraph "%s" { label="%s"\n' % (dotname[wf], lastpart(wf)))
+                stdout.write(u'subgraph "%s" { label="%s"\n' % (dotname[wf], lastpart(wf)))
                 currentwf = wf
                 clusternode[wf] = step
             else:
                 currentwf = None
 
         if str(runtype) != "https://w3id.org/cwl/cwl#Workflow":
-            stdout.write('"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(str(step))[1]))
+            stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(str(step))[1]))
 
     if currentwf is not None:
         stdout.write("}\n")
@@ -157,12 +157,12 @@ def dot_without_parameters(g, stdout):
     for src, sink, srcrun, sinkrun in qres:
         attr = ""
         if srcrun in clusternode:
-            attr += 'ltail="%s"' % dotname[srcrun]
+            attr += u'ltail="%s"' % dotname[srcrun]
             src = clusternode[srcrun]
         if sinkrun in clusternode:
-            attr += ' lhead="%s"' % dotname[sinkrun]
+            attr += u' lhead="%s"' % dotname[sinkrun]
             sink = clusternode[sinkrun]
-        stdout.write('"%s" -> "%s" [%s]\n' % (dotname[src], dotname[sink], attr))
+        stdout.write(u'"%s" -> "%s" [%s]\n' % (dotname[src], dotname[sink], attr))
 
 
 def printdot(workflow, wf, ctx, stdout, include_parameters=False):

cwltool/cwltest.py

@@ -26,29 +26,29 @@ def compare(a, b):
         if isinstance(a, dict):
             if a.get("class") == "File":
                 if not (b["path"].endswith("/" + a["path"]) or ("/" not in b["path"] and a["path"] == b["path"])):
-                    raise CompareFail("%s does not end with %s" %(b["path"], a["path"]))
+                    raise CompareFail(u"%s does not end with %s" %(b["path"], a["path"]))
                 # ignore empty collections
                 b = {k: v for k, v in b.iteritems()
                      if not isinstance(v, (list, dict)) or len(v) > 0}
             if len(a) != len(b):
-                raise CompareFail("expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
+                raise CompareFail(u"expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
             for c in a:
                 if a.get("class") != "File" or c != "path":
                     if c not in b:
-                        raise CompareFail("%s not in %s" % (c, b))
+                        raise CompareFail(u"%s not in %s" % (c, b))
                     if not compare(a[c], b[c]):
                         return False
             return True
         elif isinstance(a, list):
             if len(a) != len(b):
-                raise CompareFail("expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
+                raise CompareFail(u"expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
             for c in xrange(0, len(a)):
                 if not compare(a[c], b[c]):
                     return False
             return True
         else:
             if a != b:
-                raise CompareFail("%s != %s" % (a, b))
+                raise CompareFail(u"%s != %s" % (a, b))
             else:
                 return True
     except Exception as e:
@@ -90,14 +90,14 @@ def run_test(args, i, t):
         if err.returncode == UNSUPPORTED_FEATURE:
             return UNSUPPORTED_FEATURE
         else:
-            _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+            _logger.error(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
             _logger.error(t.get("doc"))
             _logger.error("Returned non-zero")
             return 1
     except yaml.scanner.ScannerError as e:
-        _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+        _logger.error(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
         _logger.error(outstr)
-        _logger.error("Parse error %s", str(e))
+        _logger.error(u"Parse error %s", str(e))
 
     pwd = os.path.abspath(os.path.dirname(t["job"]))
     # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"])
@@ -114,12 +114,12 @@ def run_test(args, i, t):
         try:
             compare(t.get(key), out.get(key))
         except CompareFail as ex:
-            _logger.warn("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+            _logger.warn(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
             _logger.warn(t.get("doc"))
-            _logger.warn("%s expected %s\n got %s", key,
+            _logger.warn(u"%s expected %s\n got %s", key,
                                                             json.dumps(t.get(key), indent=4, sort_keys=True),
                                                             json.dumps(out.get(key), indent=4, sort_keys=True))
-            _logger.warn("Compare failure %s", ex)
+            _logger.warn(u"Compare failure %s", ex)
             failed = True
 
     if outdir:
@@ -164,7 +164,7 @@ def main():
 
     if args.l:
         for i, t in enumerate(tests):
-            print "[%i] %s" % (i+1, t["doc"].strip())
+            print u"[%i] %s" % (i+1, t["doc"].strip())
         return 0
 
     if args.n is not None:

cwltool/docker.py

@@ -49,12 +49,12 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             _logger.info(str(cmd))
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
-                    _logger.info("Loading docker image from %s", dockerRequirement["dockerLoad"])
+                    _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
                     with open(dockerRequirement["dockerLoad"], "rb") as f:
                         loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)
-                    _logger.info("Sending GET request to %s", dockerRequirement["dockerLoad"])
+                    _logger.info(u"Sending GET request to %s", dockerRequirement["dockerLoad"])
                     req = requests.get(dockerRequirement["dockerLoad"], stream=True)
                     n = 0
                     for chunk in req.iter_content(1024*1024):
@@ -96,6 +96,6 @@ def get_from_requirements(r, req, pull_image, dry_run=False):
             return r["dockerImageId"]
         else:
             if req:
-                raise process.WorkflowException("Docker image %s not found" % r["dockerImageId"])
+                raise process.WorkflowException(u"Docker image %s not found" % r["dockerImageId"])
 
     return None

cwltool/draft2tool.py

@@ -34,7 +34,7 @@ def run(self, **kwargs):
             try:
                 self.output_callback(self.builder.do_eval(self.script), "success")
             except Exception as e:
-                _logger.warn("Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False))
+                _logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False))
                 self.output_callback({}, "permanentFail")
 
     def job(self, joborder, input_basedir, output_callback, **kwargs):
@@ -75,7 +75,7 @@ def revmap_file(builder, outdir, f):
         f["hostfs"] = True
         return f
     else:
-        raise WorkflowException("Output file path %s must be within designated output directory (%s) or an input file pass through." % (f["path"], builder.outdir))
+        raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input file pass through." % (f["path"], builder.outdir))
 
 
 class CommandLineTool(Process):
@@ -94,7 +94,7 @@ def makePathMapper(self, reffiles, input_basedir, **kwargs):
                 return PathMapper(reffiles, input_basedir)
         except OSError as e:
             if e.errno == errno.ENOENT:
-                raise WorkflowException("Missing input file %s" % e)
+                raise WorkflowException(u"Missing input file %s" % e)
 
     def job(self, joborder, input_basedir, output_callback, **kwargs):
         builder = self._init_job(joborder, input_basedir, **kwargs)
@@ -139,11 +139,11 @@ def job(self, joborder, input_basedir, output_callback, **kwargs):
         j.hints = self.hints
         j.name = uniquename(kwargs.get("name", str(id(j))))
 
-        _logger.debug("[job %s] initializing from %s%s",
+        _logger.debug(u"[job %s] initializing from %s%s",
                      j.name,
                      self.tool.get("id", ""),
-                     " as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "")
-        _logger.debug("[job %s] %s", j.name, json.dumps(joborder, indent=4))
+                     u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "")
+        _logger.debug(u"[job %s] %s", j.name, json.dumps(joborder, indent=4))
 
 
         builder.pathmapper = None
@@ -170,12 +170,12 @@ def _check_adjust(f):
                 f["containerfs"] = True
             return f
 
-        _logger.debug("[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
+        _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
 
         adjustFileObjs(builder.files, _check_adjust)
         adjustFileObjs(builder.bindings, _check_adjust)
 
-        _logger.debug("[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
+        _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
 
         dockerReq, _ = self.get_requirement("DockerRequirement")
         if dockerReq and kwargs.get("use_container"):
@@ -224,7 +224,7 @@ def collect_output_ports(self, ports, builder, outdir):
             if builder.fs_access.exists(custom_output):
                 with builder.fs_access.open(custom_output, "r") as f:
                     ret = yaml.load(f)
-                _logger.debug("Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
+                _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
                 adjustFileObjs(ret, remove_hostfs)
                 adjustFileObjs(ret, functools.partial(revmap_file, builder, outdir))
                 adjustFileObjs(ret, remove_hostfs)
@@ -236,7 +236,7 @@ def collect_output_ports(self, ports, builder, outdir):
                 try:
                     ret[fragment] = self.collect_output(port, builder, outdir)
                 except Exception as e:
-                    raise WorkflowException("Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e))
+                    raise WorkflowException(u"Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e))
             if ret:
                 adjustFileObjs(ret, remove_hostfs)
             validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
@@ -303,7 +303,7 @@ def collect_output(self, schema, builder, outdir):
                     if optional and r is None:
                         pass
                     elif (r is None or len(r) != 1 or not isinstance(r[0], dict) or "path" not in r[0]):
-                        raise WorkflowException("Expression must return a file object for %s." % schema["id"])
+                        raise WorkflowException(u"Expression must return a file object for %s." % schema["id"])
 
             if singlefile:
                 if not r and not optional:

cwltool/expression.py

@@ -15,15 +15,15 @@
 _logger = logging.getLogger("cwltool")
 
 def jshead(engineConfig, rootvars):
-    return "\n".join(engineConfig + ["var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
+    return "\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 
 def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
     if ex["engine"] == "https://w3id.org/cwl/cwl#JsonPointer":
         try:
             obj = {"job": jobinput, "context": context, "outdir": outdir, "tmpdir": tmpdir}
             return schema_salad.ref_resolver.resolve_json_pointer(obj, ex["script"])
         except ValueError as v:
-            raise WorkflowException("%s in %s" % (v,  obj))
+            raise WorkflowException(u"%s in %s" % (v,  obj))
 
     if ex["engine"] == "https://w3id.org/cwl/cwl#JavascriptEngine":
         engineConfig = []
@@ -59,7 +59,7 @@ class DR(object):
                 "tmpdir": tmpdir,
             }
 
-            _logger.debug("Invoking expression engine %s with %s",
+            _logger.debug(u"Invoking expression engine %s with %s",
                           runtime + aslist(r["engineCommand"]),
                                            json.dumps(inp, indent=4))
 
@@ -71,11 +71,11 @@ class DR(object):
 
             (stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n")
             if sp.returncode != 0:
-                raise WorkflowException("Expression engine returned non-zero exit code on evaluation of\n%s" % json.dumps(inp, indent=4))
+                raise WorkflowException(u"Expression engine returned non-zero exit code on evaluation of\n%s" % json.dumps(inp, indent=4))
 
             return json.loads(stdoutdata)
 
-    raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
+    raise WorkflowException(u"Unknown expression engine '%s'" % ex["engine"])
 
 seg_symbol = r"""\w+"""
 seg_single = r"""\['([^']|\\')+'\]"""
@@ -112,7 +112,7 @@ def param_interpolate(ex, obj, strip=True):
             return ex[0:m.start(0)] + leaf + param_interpolate(ex[m.end(0):], obj, False)
     else:
         if "$(" in ex or "${" in ex:
-            _logger.warn("Warning possible workflow bug: found '$(' or '${' in '%s' but did not match valid parameter reference and InlineJavascriptRequirement not specified.", ex)
+            _logger.warn(u"Warning possible workflow bug: found '$(' or '${' in '%s' but did not match valid parameter reference and InlineJavascriptRequirement not specified.", ex)
         return ex