Merge branch 'master' into patch-1

Author: tetron

.coveragerc

@@ -1,3 +1,13 @@
 [run]
 branch = True
 source = cwltool
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    tests/*

.gitignore

@@ -30,3 +30,16 @@ typeshed/2and3/ruamel/yaml
 
 #mypy
 .mypy_cache/
+
+# Files generated by Makefile
+.cache/
+cache/
+.coverage
+coverage.xml
+htmlcov
+output
+output.txt
+pydocstyle_report.txt
+response.txt
+test.txt
+value

.travis.yml

@@ -1,18 +1,22 @@
 sudo: false
 language: python
+cache: pip
 python:
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
   - 3.6
 os:
   - linux
 install:
   - pip install tox-travis
+jobs:
+  include:
+    - stage: release-test
+      script: RELEASE_SKIP=head ./release-test.sh
 script: tox
 branches:
   only:
     - master
 notifications:
-  email: false
+  email: false

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+In order to contribute to the development of ``cwltool``, you need to install cwltool from source (preferably in a virtual environment):
+Here's a rough 
+- Install virtualenv via pip: ``pip install virtualenv``
+- Clone the cwltool: ``git clone https://github.com/common-workflow-language/cwltool.git``
+- Switch to cwltool directory: ``cd cwltool``
+- Create a virtual environment: ``virtualenv cwltool``
+- To begin using the virtual environment, it needs to be activated: ``source bin/activate``
+- To check if you have the virtual environment set up: ``which python`` and it should point to python executable in your virtual env
+- Install cwltool in the virtual environment: ``pip install .``
+- Check the version which might be different from the version installed in general on any system: ``cwltool --version``
+- After you've made the changes, you can the complete test suite via tox: ``tox``
+	- If you want to run specific tests, say ``unit tests`` in Python 3.5, then: ``tox -e py35-unit``.
+	- Look at ``tox.ini`` for all available tests and runtimes
+- If tests are passing, you can simply commit and create a PR on ``cwltool`` repo:
+- After you're done working on the ``cwltool``, you can deactivate the virtual environment: ``deactivate``

MANIFEST.in

@@ -2,6 +2,7 @@ include gittaggers.py Makefile cwltool.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
+include tests/override/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml
 include cwltool/schemas/draft-3/*.yml
@@ -17,6 +18,10 @@ include cwltool/schemas/v1.1.0-dev1/*.md
 include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md
 include cwltool/cwlNodeEngine.js
+include cwltool/cwlNodeEngineJSConsole.js
+include cwltool/cwlNodeEngineWithContext.js
 include cwltool/extensions.yml
+include cwltool/jshint/jshint_wrapper.js
+include cwltool/jshint/jshint.js
 global-exclude *~
 global-exclude *.pyc

Makefile

@@ -27,8 +27,9 @@ MODULE=cwltool
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
 DEVPKGS=pep8 diff_cover autopep8 pylint coverage pydocstyle flake8 pytest isort mock
-DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount python-flake8 python-mock
-VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
+	   python-flake8 python-mock shellcheck
+VERSION=1.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
@@ -56,7 +57,7 @@ install: FORCE
 dist: dist/${MODULE}-$(VERSION).tar.gz
 
 dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
-	./setup.py sdist
+	./setup.py sdist bdist_wheel
 
 ## clean       : clean up all temporary / machine-generated files
 clean: FORCE
@@ -89,7 +90,7 @@ pydocstyle_report.txt: $(PYSOURCES)
 	pydocstyle setup.py $^ > pydocstyle_report.txt 2>&1 || true
 
 diff_pydocstyle_report: pydocstyle_report.txt
-	diff-quality --violations=pep8 $^
+	diff-quality --violations=pycodestyle $^
 
 ## autopep8    : fix most Python code indentation and formatting
 autopep8: $(PYSOURCES)
@@ -112,23 +113,18 @@ pylint_report.txt: ${PYSOURCES}
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
-.coverage: $(PYSOURCES) all
-	export COVERAGE_PROCESS_START=${mkfile_dir}.coveragerc; \
-	       cd ${CWL}; ./run_test.sh RUNNER=cwltool
-	coverage run setup.py test
-	coverage combine ${CWL} ${CWL}/draft-3/ ./
+.coverage: tests
+
+coverage: .coverage
+	coverage report
 
 coverage.xml: .coverage
-	python-coverage xml
+	coverage xml
 
 coverage.html: htmlcov/index.html
 
 htmlcov/index.html: .coverage
-	python-coverage html
-	@echo Test coverage of the Python code is now in htmlcov/index.html
-
-coverage-report: .coverage
-	python-coverage report
+	coverage html
 
 diff-cover: coverage-gcovr.xml coverage.xml
 	diff-cover coverage-gcovr.xml coverage.xml
@@ -138,8 +134,8 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
 		--html-report diff-cover.html
 
 ## test        : run the ${MODULE} test suite
-test: FORCE
-	./setup.py test
+test: $(PYSOURCES)
+	python setup.py test --addopts "--cov cwltool"
 
 sloccount.sc: ${PYSOURCES} Makefile
 	sloccount --duplicates --wide --details $^ > sloccount.sc
@@ -160,7 +156,7 @@ mypy2: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 
@@ -171,8 +167,21 @@ mypy3: ${PYSOURCES}
 	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
-	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
 
+release: FORCE
+	./release-test.sh
+	. testenv2/bin/activate && \
+		testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
+		pip install twine && \
+		twine upload testenv2/src/${MODULE}/dist/* && \
+		git tag ${VERSION} && git push --tags
+
 FORCE:
+
+# Use this to print the value of a Makefile variable
+# Example `make print-VERSION`
+# From https://www.cmcrossroads.com/article/printing-value-makefile-variable
+print-%  : ; @echo $* = $($*)

README.rst

@@ -31,7 +31,7 @@ It is highly recommended to setup virtual environment before installing `cwltool
   virtualenv -p python2 venv   # Create a virtual environment, can use `python3` as well
   source venv/bin/activate     # Activate environment before installing `cwltool`
 
-1. Installing the official package from PyPi (will install "cwltool" package as
+Installing the official package from PyPi (will install "cwltool" package as
 well)
 
 .. code:: bash
@@ -44,7 +44,7 @@ If installing alongside another CWL implementation then
 
   pip install cwltool
 
-2. To install from source
+Or you can install from source:
 
 .. code:: bash
 
@@ -62,9 +62,16 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-We use `tox <https://github.com/common-workflow-language/cwltool/tree/master/tox.ini>`_
-to run various tests in all supported Python environments.
-You can run the test suite by simply running the following in the terminal:
+To run the basis tests after installing `cwltool` execute the following:
+
+.. code:: bash
+
+  pip install pytest mock
+  py.test --ignore cwltool/schemas/ --pyarg cwltool
+
+To run various tests in all supported Python environments we use `tox <https://github.com/common-workflow-language/cwltool/tree/master/tox.ini>`_. To run the test suite in all supported Python environments
+first downloading the complete code repository (see the ``git clone`` instructions above) and then run
+the following in the terminal:
 ``pip install tox; tox``
 
 List of all environment can be seen using:
@@ -106,6 +113,42 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
+Using user-space replacements for Docker
+----------------------------------------
+
+Some shared computing environments don't support Docker software containers for technical or policy reasons.
+As a work around, the CWL reference runner supports using a alternative ``docker`` implementations on Linux
+with the ``--user-space-docker-cmd`` option.
+
+One such "user space" friendly docker replacement is ``udocker`` https://github.com/indigo-dc/udocker and another
+is ``dx-docker`` https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images
+
+udocker installation: https://github.com/indigo-dc/udocker/blob/master/doc/installation_manual.md#22-install-from-indigo-datacloud-repositories
+
+dx-docker installation: start with the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
+
+Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
+
+.. code:: bash
+
+  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
+  
+or 
+
+.. code:: bash
+
+  cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
+
+``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ as a Docker container runtime, an experimental feature.
+Singularity will run software containers specified in ``DockerRequirement`` and therefore works with Docker images only,
+native Singularity images are not supported.
+To use Singularity as the Docker container runtime, provide ``--singularity`` command line option to ``cwltool``.
+
+
+.. code:: bash
+
+  cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/cat-job.json
+
 Tool or workflow loading from remote or local locations
 -------------------------------------------------------
 
@@ -164,7 +207,7 @@ The easiest way to use cwltool to run a tool or workflow from Python is to use a
 Leveraging SoftwareRequirements (Beta)
 --------------------------------------
 
-CWL tools may be decoarated with ``SoftwareRequirement`` hints that cwltool
+CWL tools may be decorated with ``SoftwareRequirement`` hints that cwltool
 may in turn use to resolve to packages in various package managers or
 dependency management systems such as `Environment Modules
 <http://modules.sourceforge.net/>`__.
@@ -193,16 +236,16 @@ following ``hint`` definition for an example CWL tool.
       - r93
 
 Now imagine deploying cwltool on a cluster with Software Modules installed
-and that a ``seqtk`` module is avaialble at version ``r93``. This means cluster
-users likely won't have the ``seqtk`` the binary on their ``PATH`` by default but after
+and that a ``seqtk`` module is available at version ``r93``. This means cluster
+users likely won't have the binary ``seqtk`` on their ``PATH`` by default, but after
 sourcing this module with the command ``modulecmd sh load seqtk/r93`` ``seqtk`` is
 available on the ``PATH``. A simple dependency resolvers configuration file, called
 ``dependency-resolvers-conf.yml`` for instance, that would enable cwltool to source
 the correct module environment before executing the above tool would simply be:
 
 .. code:: yaml
 
-  - type: module
+  - type: modules
 
 The outer list indicates that one plugin is being enabled, the plugin parameters are
 defined as a dictionary for this one list item. There is only one required parameter
@@ -369,6 +412,50 @@ at the following links:
 - `Specifications - Implementation <https://github.com/galaxyproject/galaxy/commit/81d71d2e740ee07754785306e4448f8425f890bc>`__
 - `Initial cwltool Integration Pull Request <https://github.com/common-workflow-language/cwltool/pull/214>`__
 
+Overriding workflow requirements at load time
+---------------------------------------------
+
+Sometimes a workflow needs additional requirements to run in a particular
+environment or with a particular dataset.  To avoid the need to modify the
+underlying workflow, cwltool supports requirement "overrides".
+
+The format of the "overrides" object is a mapping of item identifier (workflow,
+workflow step, or command line tool) to the process requirements that should be applied.
+
+.. code:: yaml
+
+  cwltool:overrides:
+    echo.cwl:
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
+
+Overrides can be specified either on the command line, or as part of the job
+input document.  Workflow steps are identified using the name of the workflow
+file followed by the step name as a document fragment identifier "#id".
+Override identifiers are relative to the toplevel workflow document.
+
+.. code:: bash
+
+  cwltool --overrides overrides.yml my-tool.cwl my-job.yml
+
+.. code:: yaml
+
+  input_parameter1: value1
+  input_parameter2: value2
+  cwltool:overrides:
+    workflow.cwl#step1:
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
+
+.. code:: bash
+
+  cwltool my-tool.cwl my-job-with-overrides.yml
+
+
 CWL Tool Control Flow
 ---------------------
 

appveyor.yml

@@ -1,6 +1,8 @@
 version: .{build}-{branch}
 
 environment:
+  nodejs_version: "8"
+
   SYSTEMROOT: "C:\\WINDOWS"
 
   matrix:
@@ -12,10 +14,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python33-x64"
-      PYTHON_VERSION: "3.3.x"
-      PYTHON_ARCH: "64"
-
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "64"
@@ -26,17 +24,18 @@ environment:
 
 
 install:
+  # Get the latest stable version of Node.js or io.js
+  - ps: Install-Product node $env:nodejs_version
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
 build_script:
+  - "%CMD_IN_ENV% python -m pip install -U setuptools pip"
+  - "%CMD_IN_ENV% pip install pytest mock"
   - "%CMD_IN_ENV% pip install ."
 
 
 test_script:
-
-  - "%CMD_IN_ENV% python setup.py test"
+  - "%CMD_IN_ENV% py.test --verbose -p no:cacheprovider"
 
 branches:
   only: