Merge branch 'master' into fix-648

Author: mr-c

.travis.yml

@@ -1,18 +1,22 @@
 sudo: false
 language: python
+cache: pip
 python:
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
   - 3.6
 os:
   - linux
 install:
   - pip install tox-travis
+jobs:
+  include:
+    - stage: release-test
+      script: RELEASE_SKIP=head ./release-test.sh
 script: tox
 branches:
   only:
     - master
 notifications:
-  email: false
+  email: false

Makefile

@@ -27,8 +27,9 @@ MODULE=cwltool
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
 DEVPKGS=pep8 diff_cover autopep8 pylint coverage pydocstyle flake8 pytest isort mock
-DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount python-flake8 python-mock
-VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
+	   python-flake8 python-mock shellcheck
+VERSION=1.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
@@ -56,7 +57,7 @@ install: FORCE
 dist: dist/${MODULE}-$(VERSION).tar.gz
 
 dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
-	./setup.py sdist
+	./setup.py sdist bdist_wheel
 
 ## clean       : clean up all temporary / machine-generated files
 clean: FORCE
@@ -189,4 +190,3 @@ FORCE:
 # Example `make print-VERSION`
 # From https://www.cmcrossroads.com/article/printing-value-makefile-variable
 print-%  : ; @echo $* = $($*)
-

README.rst

@@ -139,6 +139,16 @@ or
 
   cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
 
+``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ as a Docker container runtime, an experimental feature.
+Singularity will run software containers specified in ``DockerRequirement`` and therefore works with Docker images only,
+native Singularity images are not supported.
+To use Singularity as the Docker container runtime, provide ``--singularity`` command line option to ``cwltool``.
+
+
+.. code:: bash
+
+  cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/cat-job.json
+
 Tool or workflow loading from remote or local locations
 -------------------------------------------------------
 

appveyor.yml

@@ -12,10 +12,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python33-x64"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "64"
-
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "64"

cwltool/argparser.py

@@ -7,40 +7,37 @@
 
 from typing import (Any, AnyStr, Dict, List, Sequence, Text, Union, cast)
 
+from . import loghandler
 from schema_salad.ref_resolver import file_uri
 from .process import (Process, shortname)
 from .resolver import ga4gh_tool_registries
 from .software_requirements import (SOFTWARE_REQUIREMENTS_ENABLED)
 
 _logger = logging.getLogger("cwltool")
 
-defaultStreamHandler = logging.StreamHandler()
-_logger.addHandler(defaultStreamHandler)
-_logger.setLevel(logging.INFO)
+DEFAULT_TMP_PREFIX = "tmp"
 
 
 def arg_parser():  # type: () -> argparse.ArgumentParser
-    parser = argparse.ArgumentParser(description='Reference executor for Common Workflow Language')
+    parser = argparse.ArgumentParser(
+        description='Reference executor for Common Workflow Language standards.')
     parser.add_argument("--basedir", type=Text)
     parser.add_argument("--outdir", type=Text, default=os.path.abspath('.'),
                         help="Output directory, default current directory")
 
-    parser.add_argument("--no-container", action="store_false", default=True,
-                        help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool",
-                        dest="use_container")
     parser.add_argument("--parallel", action="store_true", default=False,
                         help="[experimental] Run jobs in parallel. "
                              "Does not currently keep track of ResourceRequirements like the number of cores"
                              "or memory and can overload this system")
-    parser.add_argument("--preserve-environment", type=Text, action="append",
-                        help="Preserve specific environment variable when running CommandLineTools.  May be provided multiple times.",
-                        metavar="ENVVAR",
-                        default=["PATH"],
+    envgroup = parser.add_mutually_exclusive_group()
+    envgroup.add_argument("--preserve-environment", type=Text, action="append",
+                        help="Preserve specific environment variable when "
+                        "running CommandLineTools.  May be provided multiple "
+                        "times.", metavar="ENVVAR", default=["PATH"],
                         dest="preserve_environment")
-
-    parser.add_argument("--preserve-entire-environment", action="store_true",
-                        help="Preserve entire parent environment when running CommandLineTools.",
-                        default=False,
+    envgroup.add_argument("--preserve-entire-environment", action="store_true",
+                        help="Preserve all environment variable when running "
+                        "CommandLineTools.", default=False,
                         dest="preserve_entire_environment")
 
     exgroup = parser.add_mutually_exclusive_group()
@@ -75,12 +72,12 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 
     parser.add_argument("--tmpdir-prefix", type=Text,
                         help="Path prefix for temporary directories",
-                        default="tmp")
+                        default=DEFAULT_TMP_PREFIX)
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--tmp-outdir-prefix", type=Text,
                          help="Path prefix for intermediate output directories",
-                         default="tmp")
+                         default=DEFAULT_TMP_PREFIX)
 
     exgroup.add_argument("--cachedir", type=Text, default="",
                          help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
@@ -155,10 +152,22 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         "timestamps to the errors, warnings, and "
                         "notifications.")
     parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
-    parser.add_argument("--user-space-docker-cmd",
+    dockergroup = parser.add_mutually_exclusive_group()
+    dockergroup.add_argument("--user-space-docker-cmd", metavar="CMD",
                         help="(Linux/OS X only) Specify a user space docker "
                         "command (like udocker or dx-docker) that will be "
                         "used to call 'pull' and 'run'")
+    dockergroup.add_argument("--singularity", action="store_true",
+                             default=False, help="[experimental] Use "
+                             "Singularity runtime for running containers. "
+                             "Requires Singularity v2.3.2+ and Linux with kernel "
+                             "version v3.18+ or with overlayfs support "
+                             "backported.")
+    dockergroup.add_argument("--no-container", action="store_false",
+                             default=True, help="Do not execute jobs in a "
+                             "Docker container, even when `DockerRequirement` "
+                             "is specified under `hints`.",
+                             dest="use_container")
 
     dependency_resolvers_configuration_help = argparse.SUPPRESS
     dependencies_directory_help = argparse.SUPPRESS
@@ -193,7 +202,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--default-container",
                         help="Specify a default docker container that will be used if the workflow fails to specify one.")
     parser.add_argument("--no-match-user", action="store_true",
-                        help="Disable passing the current uid to 'docker run --user`")
+                        help="Disable passing the current uid to `docker run --user`")
     parser.add_argument("--disable-net", action="store_true",
                         help="Use docker's default networking for containers;"
                              " the default is to enable networking.")
@@ -238,8 +247,15 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--overrides", type=str,
                         default=None, help="Read process requirement overrides from file.")
 
-    parser.add_argument("workflow", type=Text, nargs="?", default=None)
-    parser.add_argument("job_order", nargs=argparse.REMAINDER)
+    parser.add_argument("workflow", type=Text, nargs="?", default=None,
+            metavar='cwl_document', help="path or URL to a CWL Workflow, "
+            "CommandLineTool, or ExpressionTool. If the `inputs_object` has a "
+            "`cwl:tool` field indicating the path or URL to the cwl_document, "
+            " then the `workflow` argument is optional.")
+    parser.add_argument("job_order", nargs=argparse.REMAINDER,
+            metavar='inputs_object', help="path or URL to a YAML or JSON "
+            "formatted description of the required input values for the given "
+            "`cwl_document`.")
 
     return parser
 

cwltool/builder.py

@@ -78,6 +78,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
             lead_pos = []
         bindings = []  # type: List[Dict[Text,Text]]
         binding = None  # type: Dict[Text,Any]
+        value_from_expression = False
         if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
             binding = copy.copy(schema["inputBinding"])
 
@@ -87,29 +88,33 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
                 binding["position"] = aslist(lead_pos) + [0] + aslist(tail_pos)
 
             binding["datum"] = datum
+            if "valueFrom" in binding:
+                value_from_expression = True
 
         # Handle union types
         if isinstance(schema["type"], list):
-            for t in schema["type"]:
-                if isinstance(t, (str, Text)) and self.names.has_name(t, ""):
-                    avsc = self.names.get_name(t, "")
-                elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
-                    avsc = self.names.get_name(t["name"], "")
-                else:
-                    avsc = AvroSchemaFromJSONData(t, self.names)
-                if validate.validate(avsc, datum):
-                    schema = copy.deepcopy(schema)
-                    schema["type"] = t
-                    return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
-            raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
+            if not value_from_expression:
+                for t in schema["type"]:
+                    if isinstance(t, (str, Text)) and self.names.has_name(t, ""):
+                        avsc = self.names.get_name(t, "")
+                    elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
+                        avsc = self.names.get_name(t["name"], "")
+                    else:
+                        avsc = AvroSchemaFromJSONData(t, self.names)
+                    if validate.validate(avsc, datum):
+                        schema = copy.deepcopy(schema)
+                        schema["type"] = t
+                        return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
+                raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"]))
         elif isinstance(schema["type"], dict):
-            st = copy.deepcopy(schema["type"])
-            if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
-                st["inputBinding"] = {}
-            for k in ("secondaryFiles", "format", "streamable"):
-                if k in schema:
-                    st[k] = schema[k]
-            bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos))
+            if not value_from_expression:
+                st = copy.deepcopy(schema["type"])
+                if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
+                    st["inputBinding"] = {}
+                for k in ("secondaryFiles", "format", "streamable"):
+                    if k in schema:
+                        st[k] = schema[k]
+                bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos))
         else:
             if schema["type"] in self.schemaDefs:
                 schema = self.schemaDefs[schema["type"]]
@@ -212,15 +217,18 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
+        if prefix is None and not sep:
+            with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
+                raise WorkflowException("'separate' option can not be specified without prefix")
 
         l = []  # type: List[Dict[Text,Text]]
         if isinstance(value, list):
-            if binding.get("itemSeparator"):
+            if binding.get("itemSeparator") and value:
                 l = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
             elif binding.get("valueFrom"):
                 value = [self.tostr(v) for v in value]
                 return ([prefix] if prefix else []) + value
-            elif prefix:
+            elif prefix and value:
                 return [prefix]
             else:
                 return []
@@ -244,8 +252,8 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         return [a for a in args if a is not None]
 
-    def do_eval(self, ex, context=None, pull_image=True, recursive=False):
-        # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
+    def do_eval(self, ex, context=None, pull_image=True, recursive=False, strip_whitespace=True):
+        # type: (Union[Dict[Text, Text], Text], Any, bool, bool, bool) -> Any
         if recursive:
             if isinstance(ex, dict):
                 return {k: self.do_eval(v, context, pull_image, recursive) for k, v in iteritems(ex)}
@@ -260,4 +268,5 @@ def do_eval(self, ex, context=None, pull_image=True, recursive=False):
                                   timeout=self.timeout,
                                   debug=self.debug,
                                   js_console=self.js_console,
-                                  force_docker_pull=self.force_docker_pull)
+                                  force_docker_pull=self.force_docker_pull,
+                                  strip_whitespace=strip_whitespace)