Merge branch 'master' into no-container-fix

Author: mr-c

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+This issue tracker is for reporting bugs in cwltool (the CWL reference implementation) itself. Please use the [Gitter channel](https://gitter.im/common-workflow-language/common-workflow-language) or [Biostars forum](https://www.biostars.org/) for general questions about using cwltool to create/run workflows or issues not related to cwltool. Don't forget to use cwl tag when posting on Biostars Forum.
+
+If you'd like to report a bug, fill out the template below and provide any extra information that may be useful / related to your problem. Ideally, you create an [a Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) reproducing the problem before opening an issue to ensure it's not caused by something in your code.
+
+Before you submit, please delete this help text above the ---
+Thanks!
+---
+
+## Expected Behavior
+Tell us what should happen
+
+## Actual Behavior
+Tell us what happens instead
+
+## Workflow Code
+```
+Paste the template code (ideally a minimal example) that causes the issue
+
+```
+
+## Full Traceback
+```pytb
+Paste the full traceback in case there is an exception
+Run the workflow with ``--debug`` flag for more verbose logging 
+```
+
+## Your Environment
+* cwltool version: 
+Check using ``cwltool --version``

cwltool/docker.py

@@ -27,6 +27,17 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             sp = dockerRequirement["dockerImageId"].split(":")
             if len(sp) == 1:
                 sp.append("latest")
+            elif len(sp) == 2:
+                #  if sp[1] doesn't  match valid tag names, it is a part of repository
+                if not re.match(r'[\w][\w.-]{0,127}', sp[1]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = "latest"
+            elif len(sp) == 3:
+                if re.match(r'[\w][\w.-]{0,127}', sp[2]):
+                    sp[0] = sp[0] + ":" + sp[1]
+                    sp[1] = sp[2]
+                    del sp[2]
+
             # check for repository:tag match or image id match
             if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)):
                 found = True

cwltool/main.py

@@ -431,8 +431,8 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
     if len(args.job_order) == 1 and args.job_order[0][0] != "-":
         job_order_file = args.job_order[0]
     elif len(args.job_order) == 1 and args.job_order[0] == "-":
-        job_order_object = yaml.load(stdin)
-        job_order_object, _ = loader.resolve_all(job_order_object, "")
+        job_order_object = yaml.round_trip_load(stdin)  # type: ignore
+        job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/")
     else:
         job_order_file = None
 

cwltool/sandboxjs.py

@@ -8,7 +8,8 @@
 from io import BytesIO
 
 from pkg_resources import resource_stream
-from typing import Any, Dict, List, Mapping, Text, Union
+from typing import Any, Dict, List, Mapping, Text, Union, Tuple
+
 
 class JavascriptException(Exception):
     pass
@@ -21,6 +22,27 @@ class JavascriptException(Exception):
 localdata = threading.local()
 
 have_node_slim = False
+# minimum acceptable version of nodejs engine
+minimum_node_version_str = '0.10.26'
+
+def check_js_threshold_version(working_alias):
+    # type: (str) -> bool
+
+    """Checks if the nodeJS engine version on the system
+    with the allowed minimum version.
+    https://github.com/nodejs/node/blob/master/CHANGELOG.md#nodejs-changelog
+    """
+    # parse nodejs version into int Tuple: 'v4.2.6\n' -> [4, 2, 6]
+    current_version_str = subprocess.check_output(
+        [working_alias, "-v"]).decode('ascii')
+
+    current_version = [int(v) for v in current_version_str.strip().strip('v').split('.')]
+    minimum_node_version = [int(v) for v in minimum_node_version_str.split('.')]
+
+    if current_version >= minimum_node_version:
+        return True
+    else:
+        return False
 
 
 def new_js_proc():
@@ -29,17 +51,21 @@ def new_js_proc():
     res = resource_stream(__name__, 'cwlNodeEngine.js')
     nodecode = res.read()
 
+    required_node_version, docker = (False,)*2
     nodejs = None
     trynodes = ("nodejs", "node")
     for n in trynodes:
         try:
             if subprocess.check_output([n, "--eval", "process.stdout.write('t')"]) != "t":
                 continue
-            nodejs = subprocess.Popen([n, "--eval", nodecode],
-                                      stdin=subprocess.PIPE,
-                                      stdout=subprocess.PIPE,
-                                      stderr=subprocess.PIPE)
-            break
+            else:
+                nodejs = subprocess.Popen([n, "--eval", nodecode],
+                                          stdin=subprocess.PIPE,
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.PIPE)
+
+                required_node_version = check_js_threshold_version(n)
+                break
         except subprocess.CalledProcessError:
             pass
         except OSError as e:
@@ -48,7 +74,7 @@ def new_js_proc():
             else:
                 raise
 
-    if nodejs is None:
+    if nodejs is None or nodejs is not None and required_node_version is False:
         try:
             nodeimg = "node:slim"
             global have_node_slim
@@ -63,6 +89,7 @@ def new_js_proc():
                                        "--sig-proxy=true", "--interactive",
                                        "--rm", nodeimg, "node", "--eval", nodecode],
                                       stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            docker = True
         except OSError as e:
             if e.errno == errno.ENOENT:
                 pass
@@ -71,12 +98,19 @@ def new_js_proc():
         except subprocess.CalledProcessError:
             pass
 
+    # docker failed and nodejs not on system
     if nodejs is None:
         raise JavascriptException(
             u"cwltool requires Node.js engine to evaluate Javascript "
             "expressions, but couldn't find it.  Tried %s, docker run "
             "node:slim" % u", ".join(trynodes))
 
+    # docker failed, but nodejs is installed on system but the version is below the required version
+    if docker is False and required_node_version is False:
+        raise JavascriptException(
+            u'cwltool requires minimum v{} version of Node.js engine.'.format(minimum_node_version_str),
+            u'Try updating: https://docs.npmjs.com/getting-started/installing-node')
+
     return nodejs
 
 

cwltool/workflow.py

@@ -245,6 +245,9 @@ def job(self, joborder, output_callback, **kwargs):
         # type: (Dict[Text, Text], functools.partial[None], **Any) -> Generator
         kwargs["part_of"] = self.name
         kwargs["name"] = shortname(self.id)
+
+        _logger.info(u"[%s] start", self.name)
+
         for j in self.step.job(joborder, output_callback, **kwargs):
             yield j
 
@@ -257,6 +260,8 @@ def __init__(self, workflow, **kwargs):
         self.steps = [WorkflowJobStep(s) for s in workflow.steps]
         self.state = None  # type: Dict[Text, WorkflowStateItem]
         self.processStatus = None  # type: Text
+        self.did_callback = False
+
         if "outdir" in kwargs:
             self.outdir = kwargs["outdir"]
         elif "tmp_outdir_prefix" in kwargs:
@@ -270,8 +275,27 @@ def __init__(self, workflow, **kwargs):
         _logger.debug(u"[%s] initialized from %s", self.name,
                       self.tool.get("id", "workflow embedded in %s" % kwargs.get("part_of")))
 
-    def receive_output(self, step, outputparms, jobout, processStatus):
-        # type: (WorkflowJobStep, List[Dict[Text,Text]], Dict[Text,Text], Text) -> None
+    def do_output_callback(self, final_output_callback):
+        # type: (Callable[[Any, Any], Any]) -> None
+
+        supportsMultipleInput = bool(self.workflow.get_requirement("MultipleInputFeatureRequirement")[0])
+
+        try:
+            wo = object_from_state(self.state, self.tool["outputs"], True, supportsMultipleInput, "outputSource",
+                                   incomplete=True)
+        except WorkflowException as e:
+            _logger.error(u"[%s] Cannot collect workflow output: %s", self.name, e)
+            wo = {}
+            self.processStatus = "permanentFail"
+
+        _logger.info(u"[%s] completed %s", self.name, self.processStatus)
+
+        self.did_callback = True
+
+        final_output_callback(wo, self.processStatus)
+
+    def receive_output(self, step, outputparms, final_output_callback, jobout, processStatus):
+        # type: (WorkflowJobStep, List[Dict[Text,Text]], Callable[[Any, Any], Any], Dict[Text,Text], Text) -> None
 
         for i in outputparms:
             if "id" in i:
@@ -295,8 +319,12 @@ def receive_output(self, step, outputparms, jobout, processStatus):
         step.completed = True
         self.made_progress = True
 
-    def try_make_job(self, step, **kwargs):
-        # type: (WorkflowJobStep, **Any) -> Generator
+        completed = sum(1 for s in self.steps if s.completed)
+        if completed == len(self.steps):
+            self.do_output_callback(final_output_callback)
+
+    def try_make_job(self, step, final_output_callback, **kwargs):
+        # type: (WorkflowJobStep, Callable[[Any, Any], Any], **Any) -> Generator
         inputparms = step.tool["inputs"]
         outputparms = step.tool["outputs"]
 
@@ -315,7 +343,7 @@ def try_make_job(self, step, **kwargs):
 
             _logger.debug(u"[%s] starting %s", self.name, step.name)
 
-            callback = functools.partial(self.receive_output, step, outputparms)
+            callback = functools.partial(self.receive_output, step, outputparms, final_output_callback)
 
             valueFrom = {
                 i["id"]: i["valueFrom"] for i in step.tool["inputs"]
@@ -394,7 +422,7 @@ def valueFromFunc(k, v):  # type: (Any, Any) -> Any
             step.completed = True
 
     def run(self, **kwargs):
-        _logger.debug(u"[%s] workflow starting", self.name)
+        _logger.info(u"[%s] start", self.name)
 
     def job(self, joborder, output_callback, **kwargs):
         # type: (Dict[Text, Any], Callable[[Any, Any], Any], **Any) -> Generator
@@ -429,7 +457,7 @@ def job(self, joborder, output_callback, **kwargs):
 
                 if not step.submitted:
                     try:
-                        step.iterable = self.try_make_job(step, **kwargs)
+                        step.iterable = self.try_make_job(step, output_callback, **kwargs)
                     except WorkflowException as e:
                         _logger.error(u"[%s] Cannot make job: %s", step.name, e)
                         _logger.debug("", exc_info=True)
@@ -458,19 +486,8 @@ def job(self, joborder, output_callback, **kwargs):
                 else:
                     yield None
 
-        supportsMultipleInput = bool(self.workflow.get_requirement("MultipleInputFeatureRequirement")[0])
-
-        try:
-            wo = object_from_state(self.state, self.tool["outputs"], True, supportsMultipleInput, "outputSource",
-                                   incomplete=True)
-        except WorkflowException as e:
-            _logger.error(u"[%s] Cannot collect workflow output: %s", self.name, e)
-            wo = {}
-            self.processStatus = "permanentFail"
-
-        _logger.info(u"[%s] outdir is %s", self.name, self.outdir)
-
-        output_callback(wo, self.processStatus)
+        if not self.did_callback:
+            self.do_output_callback(output_callback)
 
 
 class Workflow(Process):

setup.py

@@ -8,6 +8,12 @@
 SETUP_DIR = os.path.dirname(__file__)
 README = os.path.join(SETUP_DIR, 'README.rst')
 
+# if python3 runtime and `setup.py install` is called
+if sys.version_info.major == 3 and sys.argv[1] == 'install':
+    print("Aborting installation. CWL Tool doesn't support Python 3 currently.")
+    print("Install using Python 2 pip.")
+    exit(1)
+
 try:
     import gittaggers
 
@@ -54,14 +60,19 @@
           'schema-salad >= 2.4.20170308171942, < 3',
           'typing >= 3.5.2, < 3.6',
           'six >= 1.8.0',
-
       ],
       setup_requires=[] + pytest_runner,
       test_suite='tests',
-      tests_require=['pytest'],
+      tests_require=['pytest', 'mock >= 2.0.0',],
       entry_points={
           'console_scripts': ["cwltool=cwltool.main:main"]
       },
       zip_safe=True,
       cmdclass={'egg_info': tagger},
+      classifiers=[
+          'Development Status :: 5 - Production/Stable',
+          'Operating System :: POSIX',
+          'Operating System :: OS Independent',
+          'Programming Language :: Python :: 2 :: Only',
+      ]
       )

tests/test_js_sandbox.py

@@ -0,0 +1,32 @@
+import unittest
+from mock import Mock, patch
+
+# we should modify the subprocess imported from cwltool.sandboxjs
+from cwltool.sandboxjs import check_js_threshold_version, subprocess, minimum_node_version_str
+
+class Javascript_Sanity_Checks(unittest.TestCase):
+
+    def setUp(self):
+        self.check_output = subprocess.check_output 
+
+    def tearDown(self):
+        subprocess.check_output = self.check_output
+
+    def test_node_version(self):
+        subprocess.check_output = Mock(return_value=b'v0.8.26\n')
+        self.assertEquals(check_js_threshold_version('node'), False)
+
+        subprocess.check_output = Mock(return_value=b'v0.10.25\n')
+        self.assertEquals(check_js_threshold_version('node'), False)
+
+        subprocess.check_output = Mock(return_value=b'v0.10.26\n')
+        self.assertEquals(check_js_threshold_version('node'), True)
+
+        subprocess.check_output = Mock(return_value=b'v4.4.2\n')
+        self.assertEquals(check_js_threshold_version('node'), True)
+
+        subprocess.check_output = Mock(return_value=b'v7.7.3\n')
+        self.assertEquals(check_js_threshold_version('node'), True)
+
+    def test_is_javascript_installed(self):
+            pass