Merge branch 'master' into py3_window

Author: kapilkd13

cwltool/builder.py

@@ -49,6 +49,7 @@ def __init__(self):  # type: () -> None
         self.make_fs_access = None  # type: Type[StdFsAccess]
         self.debug = False  # type: bool
         self.mutation_manager = None  # type: MutationManager
+        self.force_docker_pull = False  # type: bool
 
         # One of "no_listing", "shallow_listing", "deep_listing"
         # Will be default "no_listing" for CWL v1.1
@@ -245,4 +246,5 @@ def do_eval(self, ex, context=None, pull_image=True, recursive=False):
                                   self.resources,
                                   context=context, pull_image=pull_image,
                                   timeout=self.timeout,
+                                  force_docker_pull=self.force_docker_pull,
                                   debug=self.debug)

cwltool/expression.py

@@ -153,8 +153,8 @@ def next_seg(remain, obj):  # type: (Text, Any) -> Any
         return obj
 
 
-def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False):
-    # type: (Text, Text, Dict[Text, Any], bool, int, bool) -> JSON
+def evaluator(ex, jslib, obj, fullJS=False, timeout=None, force_docker_pull=False, debug=False):
+    # type: (Text, Text, Dict[Text, Any], bool, int, bool, bool) -> JSON
     m = param_re.match(ex)
     if m:
         if m.end(1)+1 == len(ex) and m.group(1) == "null":
@@ -164,16 +164,17 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False):
         except Exception as w:
             raise WorkflowException("%s%s" % (m.group(1), w))
     elif fullJS:
-        return sandboxjs.execjs(ex, jslib, timeout=timeout, debug=debug)
+        return sandboxjs.execjs(ex, jslib, timeout=timeout, force_docker_pull=force_docker_pull, debug=debug)
     else:
         raise sandboxjs.JavascriptException(
             "Syntax error in parameter reference '%s' or used Javascript code without specifying InlineJavascriptRequirement.",
             ex)
 
 
 def interpolate(scan, rootvars,
-                timeout=None, fullJS=None, jslib="", debug=False):
-    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool) -> JSON
+                timeout=None, fullJS=None, jslib="",force_docker_pull=False,
+                debug=False):
+    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool) -> JSON
     scan = scan.strip()
     parts = []
     w = scanner(scan)
@@ -182,7 +183,8 @@ def interpolate(scan, rootvars,
 
         if scan[w[0]] == '$':
             e = evaluator(scan[w[0] + 1:w[1]], jslib, rootvars, fullJS=fullJS,
-                          timeout=timeout, debug=debug)
+                          timeout=timeout, force_docker_pull=force_docker_pull,
+                          debug=debug)
             if w[0] == 0 and w[1] == len(scan):
                 return e
             leaf = json.dumps(e, sort_keys=True)
@@ -200,8 +202,8 @@ def interpolate(scan, rootvars,
 
 
 def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
-            context=None, pull_image=True, timeout=None, debug=False):
-    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool) -> Any
+            context=None, pull_image=True, timeout=None, force_docker_pull=False, debug=False):
+    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool) -> Any
 
     runtime = copy.copy(resources)
     runtime["tmpdir"] = docker_windows_path_adjust(tmpdir)
@@ -227,6 +229,7 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
                                timeout=timeout,
                                fullJS=fullJS,
                                jslib=jslib,
+                               force_docker_pull=force_docker_pull,
                                debug=debug)
         except Exception as e:
             raise WorkflowException("Expression evaluation error:\n%s" % e)

cwltool/main.py

@@ -226,7 +226,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--make-template", action="store_true",
                          help="Generate a template input object")
 
-
+    parser.add_argument("--force-docker-pull", action="store_true",
+                        default=False, help="Pull latest docker image even if"
+                                            " it is locally present", dest="force_docker_pull")
     parser.add_argument("workflow", type=Text, nargs="?", default=None)
     parser.add_argument("job_order", nargs=argparse.REMAINDER)
 

cwltool/pack.py

@@ -157,4 +157,9 @@ def rewrite_id(r, mainuri):
 
     import_embed(packed, set())
 
+    if len(packed["$graph"]) == 1:
+        # duplicate 'cwlVersion' inside $graph when there is a single item
+        # because we're printing contents inside '$graph' rather than whole dict
+        packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"]
+
     return packed

cwltool/process.py

@@ -560,6 +560,7 @@ def _init_job(self, joborder, **kwargs):
 
         builder.make_fs_access = kwargs.get("make_fs_access") or StdFsAccess
         builder.fs_access = builder.make_fs_access(kwargs["basedir"])
+        builder.force_docker_pull = kwargs.get("force_docker_pull")
 
         loadListingReq, _ = self.get_requirement("http://commonwl.org/cwltool#LoadListingRequirement")
         if loadListingReq:

cwltool/sandboxjs.py

@@ -53,8 +53,8 @@ def check_js_threshold_version(working_alias):
         return False
 
 
-def new_js_proc():
-    # type: () -> subprocess.Popen
+def new_js_proc(force_docker_pull=False):
+    # type: (bool) -> subprocess.Popen
 
     res = resource_stream(__name__, 'cwlNodeEngine.js')
     nodecode = res.read().decode('utf-8')
@@ -86,10 +86,11 @@ def new_js_proc():
         try:
             nodeimg = "node:slim"
             global have_node_slim
+
             if not have_node_slim:
                 dockerimgs = subprocess.check_output(["docker", "images", "-q", nodeimg]).decode('utf-8')
                 # if output is an empty string
-                if len(dockerimgs.split("\n")) <= 1:
+                if (len(dockerimgs.split("\n")) <= 1) or force_docker_pull:
                     # pull node:slim docker container
                     nodejsimg = subprocess.check_output(["docker", "pull", nodeimg]).decode('utf-8')
                     _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg)
@@ -124,10 +125,10 @@ def new_js_proc():
     return nodejs
 
 
-def execjs(js, jslib, timeout=None, debug=False):  # type: (Union[Mapping, Text], Any, int, bool) -> JSON
+def execjs(js, jslib, timeout=None, force_docker_pull=False, debug=False):  # type: (Union[Mapping, Text], Any, int, bool, bool) -> JSON
 
     if not hasattr(localdata, "proc") or localdata.proc.poll() is not None or onWindows():
-        localdata.proc = new_js_proc()
+        localdata.proc = new_js_proc(force_docker_pull=force_docker_pull)
 
     nodejs = localdata.proc
 

jenkins.bash

@@ -14,29 +14,40 @@ cloneorpull() {
 }
 venv() {
         if ! test -d "$1" ; then
-                virtualenv "$1"
+                virtualenv -p python${PYTHON_VERSION} "$1"
         fi
 	# shellcheck source=/dev/null
         source "$1"/bin/activate
 }
-git clean --force -d -x || /bin/true
+
 cloneorpull common-workflow-language https://github.com/common-workflow-language/common-workflow-language.git
-venv cwltool-venv
 docker pull node:slim
-export PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/
-pip install -U setuptools wheel pip
-python setup.py install
-pip install "cwltest>=1.0.20160825151655"
-pushd common-workflow-language
+# clean both the repos before the loop
 git clean --force -d -x || /bin/true
-# shellcheck disable=SC2154
-if [[ "$version" = *dev* ]]
-then
-	EXTRA="EXTRA=--enable-dev"
-fi
-./run_test.sh --junit-xml=result.xml RUNNER=cwltool -j4 DRAFT="${version}" ${EXTRA}
-CODE=$?
-popd
+git -C common-workflow-language clean --force -d -x || /bin/true
+
+# Test for Python 2.7 and Python 3
+for PYTHON_VERSION in 2.7 3
+do
+	venv cwltool-venv${PYTHON_VERSION}
+	export PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/
+	# use pip2.7 and pip3 in separate loop runs
+	pip${PYTHON_VERSION} install -U setuptools wheel pip
+	pip${PYTHON_VERSION} install .
+	pip${PYTHON_VERSION} install "cwltest>=1.0.20160825151655"
+	pushd common-workflow-language
+	# shellcheck disable=SC2154
+	if [[ "$version" = *dev* ]]
+	then
+		EXTRA="EXTRA=--enable-dev"
+	fi
+	./run_test.sh --junit-xml=result${PYTHON_VERSION}.xml RUNNER=cwltool -j4 DRAFT=${version}
+	CODE=$(($CODE+$?)) # capture return code of ./run_test.sh
+	deactivate
+	popd
+done
+
+# build new docker container
 if [ "$GIT_BRANCH" = "origin/master" ] && [[ "$version" = "v1.0" ]]
 then
   ./build-cwl-docker.sh && docker push commonworkflowlanguage/cwltool_module && docker push commonworkflowlanguage/cwltool

requirements.txt

@@ -1,4 +1,4 @@
-requests>=1.0
+requests>=2.12.4
 ruamel.yaml>=0.12.4,<0.15
 rdflib==4.2.2
 rdflib-jsonld==0.4.0

setup.py

@@ -48,7 +48,7 @@
       include_package_data=True,
       install_requires=[
           'setuptools',
-          'requests >= 1.0',
+          'requests >= 2.12.4',
           'ruamel.yaml >= 0.12.4, < 0.15',
           'rdflib >= 4.2.2, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',

tests/test_pack.py

@@ -5,6 +5,7 @@
 from functools import partial
 
 import cwltool.pack
+from cwltool.main import print_pack as print_pack
 import cwltool.workflow
 from cwltool.load_tool import fetch_document, validate_document
 from cwltool.main import makeRelative
@@ -33,3 +34,29 @@ def test_pack(self):
         del expect_packed["$schemas"]
 
         self.assertEqual(expect_packed, packed)
+
+    def test_pack_missing_cwlVersion(self):
+        """Test to ensure the generated pack output is not missing
+        the `cwlVersion` in case of single tool workflow and single step workflow"""
+        # Since diff is longer than 3174 characters
+        self.maxDiff = None
+
+        # Testing single tool workflow
+        document_loader, workflowobj, uri = fetch_document(
+            get_data("tests/wf/hello_single_tool.cwl"))
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+            document_loader, workflowobj, uri)
+        # generate pack output dict
+        packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
+
+        self.assertEqual('v1.0', packed["cwlVersion"])
+
+        # Testing single step workflow
+        document_loader, workflowobj, uri = fetch_document(
+            get_data("tests/wf/hello-workflow.cwl"))
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+            document_loader, workflowobj, uri)
+        # generate pack output dict
+        packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
+
+        self.assertEqual('v1.0', packed["cwlVersion"])

tests/wf/hello-workflow.cwl

@@ -0,0 +1,38 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+
+label: "Hello World"
+doc: "Outputs a message using echo"
+
+inputs:
+  usermessage: string
+
+outputs:
+  response:
+    outputSource: step0/response
+    type: File
+
+steps:
+  step0:
+    run:
+      class: CommandLineTool
+      inputs:
+        message:
+          type: string
+          doc: "The message to print"
+          default: "Hello World"
+          inputBinding:
+            position: 1
+      baseCommand: echo
+      arguments:
+         - "-n"
+         - "-e"
+      stdout: response.txt
+      outputs:
+        response:
+          type: stdout
+    in:
+      message: usermessage
+    out: [response]

tests/wf/hello_single_tool.cwl

@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+inputs:
+  message:
+    type: string
+    inputBinding:
+      position: 1
+outputs: []