Merge branch 'master' into udocker

Author: mr-c

cwltool/load_tool.py

@@ -281,10 +281,13 @@ def make_tool(document_loader,  # type: Loader
     """Make a Python CWL object."""
     resolveduri = document_loader.resolve_ref(uri)[0]
 
+    processobj = None
     if isinstance(resolveduri, list):
-        if len(resolveduri) == 1:
-            processobj = resolveduri[0]
-        else:
+        for obj in resolveduri:
+            if obj['id'].endswith('#main'):
+                processobj = obj
+                break
+        if not processobj:
             raise WorkflowException(
                 u"Tool file contains graph of multiple objects, must specify "
                 "one of #%s" % ", #".join(

cwltool/main.py

@@ -790,7 +790,8 @@ def main(argsl=None,  # type: List[str]
                      'enable_ga4gh_tool_registry': False,
                      'ga4gh_tool_registries': [],
                      'find_default_container': None,
-                     'make_template': False
+                                   'make_template': False,
+                                   'overrides': None
         }):
             if not hasattr(args, k):
                 setattr(args, k, v)
@@ -869,10 +870,6 @@ def main(argsl=None,  # type: List[str]
                                     skip_schemas=args.skip_schemas,
                                     overrides=overrides)
 
-            if args.pack:
-                stdout.write(print_pack(document_loader, processobj, uri, metadata))
-                return 0
-
             if args.print_pre:
                 stdout.write(json.dumps(processobj, indent=4))
                 return 0
@@ -901,6 +898,11 @@ def main(argsl=None,  # type: List[str]
                 return 0
 
             if args.validate:
+                _logger.info("Tool definition is valid")
+                return 0
+
+            if args.pack:
+                stdout.write(print_pack(document_loader, processobj, uri, metadata))
                 return 0
 
             if args.print_rdf:

cwltool/pack.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import
 import copy
+import re
 from typing import Any, Callable, Dict, List, Set, Text, Union, cast
 
-from schema_salad.ref_resolver import Loader
+from schema_salad.ref_resolver import Loader, SubLoader
 from six.moves import urllib
+from ruamel.yaml.comments import CommentedSeq, CommentedMap
 
 from .process import shortname, uniquename
 import six
@@ -64,7 +66,12 @@ def replace_refs(d, rewrite, stem, newstem):
                 if v in rewrite:
                     d[s] = rewrite[v]
                 elif v.startswith(stem):
-                    d[s] = newstem + v[len(stem):]
+                    id_ = v[len(stem):]
+                    # prevent appending newstems if tool is already packed
+                    if id_.startswith(newstem.strip("#")):
+                        d[s] = "#" + id_
+                    else:
+                        d[s] = newstem + id_
             replace_refs(v, rewrite, stem, newstem)
 
 def import_embed(d, seen):
@@ -84,12 +91,25 @@ def import_embed(d, seen):
                     seen.add(this)
                     break
 
-        for v in d.values():
-            import_embed(v, seen)
+        for k in sorted(d.keys()):
+            import_embed(d[k], seen)
 
 
 def pack(document_loader, processobj, uri, metadata):
     # type: (Loader, Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Dict[Text, Text]) -> Dict[Text, Any]
+
+    document_loader = SubLoader(document_loader)
+    document_loader.idx = {}
+    if isinstance(processobj, dict):
+        document_loader.idx[processobj["id"]] = CommentedMap(six.iteritems(processobj))
+    elif isinstance(processobj, list):
+        path, frag = urllib.parse.urldefrag(uri)
+        for po in processobj:
+            if not frag:
+                if po["id"].endswith("#main"):
+                    uri = po["id"]
+            document_loader.idx[po["id"]] = CommentedMap(six.iteritems(po))
+
     def loadref(b, u):
         # type: (Text, Text) -> Union[Dict, List, Text]
         return document_loader.resolve_ref(u, base_url=b)[0]
@@ -111,7 +131,8 @@ def rewrite_id(r, mainuri):
         if r == mainuri:
             rewrite[r] = "#main"
         elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"):
-            pass
+            path, frag = urllib.parse.urldefrag(r)
+            rewrite[r] = "#"+frag
         else:
             path, frag = urllib.parse.urldefrag(r)
             if path == mainpath:
@@ -132,6 +153,9 @@ def rewrite_id(r, mainuri):
     schemas = set()  # type: Set[Text]
     for r in sorted(runs):
         dcr, metadata = document_loader.resolve_ref(r)
+        if isinstance(dcr, CommentedSeq):
+            dcr = dcr[0]
+            dcr = cast(CommentedMap, dcr)
         if not isinstance(dcr, dict):
             continue
         for doc in (dcr, metadata):

cwltool/workflow.py

@@ -625,8 +625,7 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs)
     all_exception_msg = "\n".join(exception_msgs)
 
     if warnings:
-        _logger.warning("Workflow checker warning:")
-        _logger.warning(all_warning_msg)
+        _logger.warning("Workflow checker warning:\n%s" % all_warning_msg)
     if exceptions:
         raise validate.ValidationException(all_exception_msg)
 

tests/test_pack.py

@@ -1,22 +1,27 @@
 from __future__ import absolute_import
+
 import json
-import os
 import unittest
+
+import os
 from functools import partial
 
+import pytest
+from six import StringIO
+
 import cwltool.pack
-from cwltool.main import print_pack as print_pack
 import cwltool.workflow
 from cwltool.load_tool import fetch_document, validate_document
-from cwltool.main import makeRelative
+from cwltool.main import makeRelative, main, print_pack
 from cwltool.pathmapper import adjustDirObjs, adjustFileObjs
-
+from cwltool.utils import onWindows
 from .util import get_data
 
 
 class TestPack(unittest.TestCase):
+    maxDiff = None
+
     def test_pack(self):
-        self.maxDiff = None
 
         document_loader, workflowobj, uri = fetch_document(
             get_data("tests/wf/revsort.cwl"))
@@ -38,8 +43,6 @@ def test_pack(self):
     def test_pack_missing_cwlVersion(self):
         """Test to ensure the generated pack output is not missing
         the `cwlVersion` in case of single tool workflow and single step workflow"""
-        # Since diff is longer than 3174 characters
-        self.maxDiff = None
 
         # Testing single tool workflow
         document_loader, workflowobj, uri = fetch_document(
@@ -60,3 +63,56 @@ def test_pack_missing_cwlVersion(self):
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
 
         self.assertEqual('v1.0', packed["cwlVersion"])
+
+    def test_pack_idempotence_tool(self):
+        """Test to ensure that pack produces exactly the same document for
+           an already packed document"""
+
+        # Testing single tool
+        self._pack_idempotently("tests/wf/hello_single_tool.cwl")
+
+    def test_pack_idempotence_workflow(self):
+        """Test to ensure that pack produces exactly the same document for
+           an already packed document"""
+
+        # Testing workflow
+        self._pack_idempotently("tests/wf/count-lines1-wf.cwl")
+
+    def _pack_idempotently(self, document):
+        document_loader, workflowobj, uri = fetch_document(
+            get_data(document))
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+            document_loader, workflowobj, uri)
+        # generate pack output dict
+        packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
+
+        document_loader, workflowobj, uri2 = fetch_document(packed)
+        document_loader, avsc_names, processobj, metadata, uri2 = validate_document(
+            document_loader, workflowobj, uri)
+        double_packed = json.loads(print_pack(document_loader, processobj, uri2, metadata))
+        self.assertEqual(packed, double_packed)
+
+    @pytest.mark.skipif(onWindows(),
+                        reason="Instance of cwltool is used, on Windows it invokes a default docker container"
+                               "which is not supported on AppVeyor")
+    def test_packed_workflow_execution(self):
+        test_wf = "tests/wf/count-lines1-wf.cwl"
+        test_wf_job = "tests/wf/wc-job.json"
+        document_loader, workflowobj, uri = fetch_document(
+            get_data(test_wf))
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+            document_loader, workflowobj, uri)
+        packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
+        temp_packed_path = "/tmp/packedwf"
+        with open(temp_packed_path, 'w') as f:
+            json.dump(packed, f)
+        normal_output = StringIO()
+        packed_output = StringIO()
+        self.assertEquals(main(['--debug', get_data(temp_packed_path),
+                                get_data(test_wf_job)],
+                               stdout=packed_output), 0)
+        self.assertEquals(main([get_data(test_wf),
+                                get_data(test_wf_job)],
+                               stdout=normal_output), 0)
+        self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
+

tests/wf/count-lines1-wf.cwl

@@ -0,0 +1,25 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  file1:
+    type: File
+
+outputs:
+  count_output:
+    type: int
+    outputSource: step2/output
+
+steps:
+  step1:
+    run: wc-tool.cwl
+    in:
+      file1: file1
+    out: [output]
+
+  step2:
+    run: parseInt-tool.cwl
+    in:
+      file1: step1/output
+    out: [output]

tests/wf/parseInt-tool.cwl

@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+
+class: ExpressionTool
+requirements:
+  - class: InlineJavascriptRequirement
+cwlVersion: v1.0
+
+inputs:
+  file1:
+    type: File
+    inputBinding: { loadContents: true }
+
+outputs:
+  output: int
+
+expression: "$({'output': parseInt(inputs.file1.contents)})"

tests/wf/wc-job.json

@@ -0,0 +1,6 @@
+{
+    "file1": {
+        "class": "File",
+        "location": "whale.txt"
+    }
+}

tests/wf/wc-tool.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+
+inputs:
+  file1: File
+
+outputs:
+  output:
+    type: File
+    outputBinding: { glob: output }
+
+baseCommand: [wc, -l]
+
+stdin: $(inputs.file1.path)
+stdout: output

tests/wf/whale.txt

@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.