Merge branch 'master' into udocker

Author: mr-c

MANIFEST.in

@@ -2,6 +2,7 @@ include gittaggers.py Makefile cwltool.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
+include tests/override/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml
 include cwltool/schemas/draft-3/*.yml

Makefile

@@ -160,7 +160,7 @@ mypy2: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 
@@ -171,7 +171,7 @@ mypy3: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 

README.rst

@@ -65,7 +65,7 @@ Running tests locally
 To run the basis tests after installing `cwltool` execute the following:
 
 .. code:: bash
-  
+
   pip install pytest mock
   py.test --ignore cwltool/schemas/ --pyarg cwltool
 
@@ -235,7 +235,7 @@ the correct module environment before executing the above tool would simply be:
 
 .. code:: yaml
 
-  - type: module
+  - type: modules
 
 The outer list indicates that one plugin is being enabled, the plugin parameters are
 defined as a dictionary for this one list item. There is only one required parameter
@@ -410,17 +410,16 @@ environment or with a particular dataset.  To avoid the need to modify the
 underlying workflow, cwltool supports requirement "overrides".
 
 The format of the "overrides" object is a mapping of item identifier (workflow,
-workflow step, or command line tool) followed by a list of ProcessRequirements
-that should be applied.
+workflow step, or command line tool) to the process requirements that should be applied.
 
 .. code:: yaml
 
   cwltool:overrides:
     echo.cwl:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
-
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 Overrides can be specified either on the command line, or as part of the job
 input document.  Workflow steps are identified using the name of the workflow
@@ -437,9 +436,10 @@ Override identifiers are relative to the toplevel workflow document.
   input_parameter2: value2
   cwltool:overrides:
     workflow.cwl#step1:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 .. code:: bash
 

cwltool/draft2tool.py

@@ -1,14 +1,16 @@
 from __future__ import absolute_import
 import copy
 import hashlib
+import locale
 import json
 import logging
 import os
 import re
 import shutil
 import tempfile
-from functools import partial
-from typing import Any, Callable, Dict, Generator, List, Optional, Set, Text, Union, cast
+from functools import partial, cmp_to_key
+from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Text,
+    Union, cast)
 
 from six import string_types, u
 
@@ -208,7 +210,7 @@ def makeJobRunner(self, use_container=True, **kwargs):  # type: (Optional[bool],
                     })
                     dockerReq = self.requirements[0]
                     if default_container == windows_default_container_id and use_container and onWindows():
-                        _logger.warning(DEFAULT_CONTAINER_MSG%(windows_default_container_id, windows_default_container_id))
+                        _logger.warning(DEFAULT_CONTAINER_MSG % (windows_default_container_id, windows_default_container_id))
 
         if dockerReq and use_container:
             return DockerCommandLineJob()
@@ -523,8 +525,8 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jo
                 for i, port in enumerate(ports):
                     def makeWorkflowException(msg):
                         return WorkflowException(
-                                u"Error collecting output for parameter '%s':\n%s"
-                                % (shortname(port["id"]), msg))
+                            u"Error collecting output for parameter '%s':\n%s"
+                            % (shortname(port["id"]), msg))
                     with SourceLine(ports, i, makeWorkflowException, debug):
                         fragment = shortname(port["id"])
                         ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
@@ -575,16 +577,25 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                         elif gb == ".":
                             gb = outdir
                         elif gb.startswith("/"):
-                            raise WorkflowException("glob patterns must not start with '/'")
+                            raise WorkflowException(
+                                "glob patterns must not start with '/'")
                         try:
                             prefix = fs_access.glob(outdir)
                             r.extend([{"location": g,
-                                       "path": fs_access.join(builder.outdir, g[len(prefix[0])+1:]),
+                                       "path": fs_access.join(builder.outdir,
+                                           g[len(prefix[0])+1:]),
                                        "basename": os.path.basename(g),
-                                       "nameroot": os.path.splitext(os.path.basename(g))[0],
-                                       "nameext": os.path.splitext(os.path.basename(g))[1],
-                                       "class": "File" if fs_access.isfile(g) else "Directory"}
-                                      for g in fs_access.glob(fs_access.join(outdir, gb))])
+                                       "nameroot": os.path.splitext(
+                                           os.path.basename(g))[0],
+                                       "nameext": os.path.splitext(
+                                           os.path.basename(g))[1],
+                                       "class": "File" if fs_access.isfile(g)
+                                       else "Directory"}
+                                      for g in sorted(fs_access.glob(
+                                          fs_access.join(outdir, gb)),
+                                          key=cmp_to_key(cast(
+                                              Callable[[Text, Text],
+                                                  int], locale.strcoll)))])
                         except (OSError, IOError) as e:
                             _logger.warning(Text(e))
                         except:

cwltool/job.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import codecs
 import functools
 import io
 import json
@@ -549,14 +550,14 @@ def _job_popen(
             stdin_path=stdin_path,
         )
         with open(os.path.join(job_dir, "job.json"), "wb") as f:
-            json.dump(job_description, f)
+            json.dump(job_description, codecs.getwriter('utf-8')(f), ensure_ascii=False) # type: ignore
         try:
             job_script = os.path.join(job_dir, "run_job.bash")
             with open(job_script, "wb") as f:
                 f.write(job_script_contents.encode('utf-8'))
             job_run = os.path.join(job_dir, "run_job.py")
             with open(job_run, "wb") as f:
-                f.write(PYTHON_RUN_SCRIPT)
+                f.write(PYTHON_RUN_SCRIPT.encode('utf-8'))
             sp = subprocess.Popen(
                 ["bash", job_script.encode("utf-8")],
                 shell=False,