Merge branch 'master' into defaultcheck

Author: mr-c

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+This issue tracker is for reporting bugs in cwltool (the CWL reference implementation) itself. Please use the [Gitter channel](https://gitter.im/common-workflow-language/common-workflow-language) or [Biostars forum](https://www.biostars.org/) for general questions about using cwltool to create/run workflows or issues not related to cwltool. Don't forget to use cwl tag when posting on Biostars Forum.
+
+If you'd like to report a bug, fill out the template below and provide any extra information that may be useful / related to your problem. Ideally, you create an [a Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) reproducing the problem before opening an issue to ensure it's not caused by something in your code.
+
+Before you submit, please delete this help text above the ---
+Thanks!
+---
+
+## Expected Behavior
+Tell us what should happen
+
+## Actual Behavior
+Tell us what happens instead
+
+## Workflow Code
+```
+Paste the template code (ideally a minimal example) that causes the issue
+
+```
+
+## Full Traceback
+```pytb
+Paste the full traceback in case there is an exception
+Run the workflow with ``--debug`` flag for more verbose logging 
+```
+
+## Your Environment
+* cwltool version: 
+Check using ``cwltool --version``

.travis.yml

@@ -13,3 +13,6 @@ install:
 script: tox
 notifications:
   email: false
+branches:
+  only:
+    - master

MANIFEST.in

@@ -1,5 +1,6 @@
 include gittaggers.py Makefile cwltool.py
 include tests/*
+include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/wf/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/draft-2/*.yml

Makefile

@@ -26,8 +26,8 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest
-DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest isort mock
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8 python-mock
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -65,6 +65,11 @@ clean: FORCE
 	rm -Rf .coverage
 	rm -f diff-cover.html
 
+# Linting and code style related targets
+## sorting imports using isort: https://github.com/timothycrosley/isort
+sort_imports:
+	isort ${MODULE}/*.py tests/*.py setup.py
+
 ## pep8        : check Python code style
 pep8: $(PYSOURCES)
 	pep8 --exclude=_version.py  --show-source --show-pep8 $^ || true

README.rst

@@ -4,6 +4,11 @@ Common workflow language tool description reference implementation
 
 CWL Conformance test: |Build Status|
 
+Travis: |Unix Build Status|
+
+.. |Unix Build Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/master.svg?label=unix%20build
+   :target: https://travis-ci.org/common-workflow-language/cwltool
+
 This is the reference implementation of the Common Workflow Language.  It is
 intended to be feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
@@ -26,7 +31,7 @@ well)::
 
   pip install cwlref-runner
 
-If installling alongside another CWL implementation then::
+If installing alongside another CWL implementation then::
 
   pip install cwltool
 
@@ -72,7 +77,7 @@ the default cwl-runner use::
 Use with boot2docker
 --------------------
 boot2docker is running docker inside a virtual machine and it only mounts ``Users``
-on it. The default behavoir of CWL is to create temporary directories under e.g.
+on it. The default behavior of CWL is to create temporary directories under e.g.
 ``/Var`` which is not accessible to Docker containers.
 
 To run CWL successfully with boot2docker you need to set the ``--tmpdir-prefix``

cwltool/builder.py

@@ -1,14 +1,17 @@
 import copy
+from typing import Any, Callable, Text, Type, Union
+
+from six import iteritems, string_types
 
 import avro
 import schema_salad.validate as validate
 from schema_salad.sourceline import SourceLine
-from typing import Any, Callable, Text, Type, Union
-from six import string_types, iteritems
 
 from . import expression
 from .errors import WorkflowException
-from .pathmapper import PathMapper, adjustFileObjs, normalizeFilesDirs, get_listing
+from .mutation import MutationManager
+from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs,
+                         visit_class)
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
 
@@ -41,6 +44,7 @@ def __init__(self):  # type: () -> None
         self.make_fs_access = None  # type: Type[StdFsAccess]
         self.build_job_script = None  # type: Callable[[List[str]], Text]
         self.debug = False  # type: bool
+        self.mutation_manager = None  # type: MutationManager
 
         # One of "no_listing", "shallow_listing", "deep_listing"
         # Will be default "no_listing" for CWL v1.1
@@ -144,7 +148,7 @@ def _capture_files(f):
                     self.files.append(f)
                     return f
 
-                adjustFileObjs(datum.get("secondaryFiles", []), _capture_files)
+                visit_class(datum.get("secondaryFiles", []), ("File", "Directory"), _capture_files)
 
             if schema["type"] == "Directory":
                 ll = self.loadListing or (binding and binding.get("loadListing"))

cwltool/cwlrdf.py

@@ -1,7 +1,9 @@
+from typing import IO, Any, Dict, Text
+
 from rdflib import Graph
+
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import ContextType
-from typing import Any, Dict, IO, Text
 from six.moves import urllib
 
 from .process import Process

cwltool/docker.py

@@ -4,9 +4,9 @@
 import subprocess
 import sys
 import tempfile
+from typing import Text
 
 import requests
-from typing import Text
 
 from .errors import WorkflowException
 
@@ -27,6 +27,17 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             sp = dockerRequirement["dockerImageId"].split(":")
             if len(sp) == 1:
                 sp.append("latest")
+            elif len(sp) == 2:
+                #  if sp[1] doesn't  match valid tag names, it is a part of repository
+                if not re.match(r'[\w][\w.-]{0,127}', sp[1]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = "latest"
+            elif len(sp) == 3:
+                if re.match(r'[\w][\w.-]{0,127}', sp[2]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = sp[2]
+                    del sp[2]
+
             # check for repository:tag match or image id match
             if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)):
                 found = True

cwltool/docker_uid.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
-import subprocess
 
+import subprocess
 from typing import Text