Merge remote-tracking branch 'origin/master' into stagefiles

Author: mr-c

cwltool/cwltest.py

@@ -25,15 +25,15 @@ class CompareFail(Exception):
 
 
 def compare(a, b):  # type: (Any, Any) -> bool
+    if a == "Any" or b == "Any":
+        return True
     try:
         if isinstance(a, dict):
             if a.get("class") == "File":
-                if "path" in a:
-                    comp = "path"
-                else:
-                    comp = "location"
-                if a[comp] and (not (b[comp].endswith("/" + a[comp]) or ("/" not in b[comp] and a[comp] == b[comp]))):
-                    raise CompareFail(u"%s does not end with %s" %(b[comp], a[comp]))
+                if a["path"] == "Any" or b["path"] == "Any":
+                    return True
+                if not (b["path"].endswith("/" + a["path"]) or ("/" not in b["path"] and a["path"] == b["path"])):
+                    raise CompareFail(u"%s does not end with %s" %(b["path"], a["path"]))
                 # ignore empty collections
                 b = {k: v for k, v in b.iteritems()
                      if not isinstance(v, (list, dict)) or len(v) > 0}

cwltool/main.py

@@ -182,6 +182,16 @@ def output_callback(out, processStatus):
     kwargs["outdir"] = tempfile.mkdtemp()
     output_dirs.add(kwargs["outdir"])
 
+    jobReqs = None
+    if "cwl:requirements" in job_order_object:
+        jobReqs = job_order_object["cwl:requirements"]
+    elif ("cwl:defaults" in t.metadata and "cwl:requirements" in
+            t.metadata["cwl:defaults"]):
+        jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"]
+    if jobReqs:
+        for req in jobReqs:
+            t.requirements.append(req)
+
     jobiter = t.job(job_order_object,
                     output_callback,
                     **kwargs)
@@ -297,7 +307,8 @@ def generate_parser(toolparser, tool, namemap):
                 action = cast(argparse.Action, FileAppendAction)
             else:
                 action = "append"
-
+        elif isinstance(inptype, dict) and inptype["type"] == "enum":
+            atype = str
         if inptype == "string":
             atype = str
         elif inptype == "int":

cwltool/process.py

@@ -286,6 +286,22 @@ def fillInDefaults(inputs, job):
         else:
             raise validate.ValidationException("Missing input parameter `%s`" % shortname(inp["id"]))
 
+
+def avroize_type(field_type, name_prefix=""):
+    """
+    adds missing information to a type so that CWL types are valid in schema_salad.
+    """
+    if type(field_type) == list:
+        field_type_result = []
+        for idx, field_type_item in enumerate(field_type):
+            field_type_result.append(avroize_type(field_type_item, name_prefix+"_"+str(idx)))
+        return field_type_result
+    elif type(field_type) == dict and "type" in field_type and field_type["type"] == "enum":
+        if "name" not in field_type:
+            field_type["name"] = name_prefix+"_type_enum"
+    return field_type
+
+
 class Process(object):
     __metaclass__ = abc.ABCMeta
 
@@ -328,7 +344,7 @@ def __init__(self, toolpath_object, **kwargs):
 
         if sd:
             sdtypes = sd["types"]
-            av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in sdtypes}, set())
+            av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
             for i in av:
                 self.schemaDefs[i["name"]] = i
             avro.schema.make_avsc_object(av, self.names)
@@ -354,7 +370,7 @@ def __init__(self, toolpath_object, **kwargs):
                     c["type"] = ["null"] + aslist(c["type"])
                 else:
                     c["type"] = c["type"]
-
+                c["type"] = avroize_type(c["type"],c["name"])
                 if key == "inputs":
                     self.inputs_record_schema["fields"].append(c)  # type: ignore
                 elif key == "outputs":

cwltool/workflow.py

@@ -82,27 +82,29 @@ def match_types(sinktype, src, iid, inputobj, linkMerge, valueFrom):
             else:
                 raise WorkflowException(u"Unrecognized linkMerge enum '%s'" % linkMerge)
             return True
-    elif valueFrom is not None or are_same_type(src.parameter["type"], sinktype) or sinktype == "Any":
+    elif valueFrom is not None or can_assign_src_to_sink(src.parameter["type"], sinktype) or sinktype == "Any":
         # simply assign the value from state to input
         inputobj[iid] = copy.deepcopy(src.value)
         return True
     return False
 
-def are_same_type(src, sink):  # type: (Any, Any) -> bool
+def can_assign_src_to_sink(src, sink):  # type: (Any, Any) -> bool
     """Check for identical type specifications, ignoring extra keys like inputBinding.
     """
     if isinstance(src, dict) and isinstance(sink, dict):
         if src["type"] == "array" and sink["type"] == "array":
-            if 'null' in sink["items"]:
-                return are_same_type([src["items"]], [it for it in sink["items"] if it != 'null'])
-            return are_same_type(src["items"], sink["items"])
-        elif src["type"] == sink["type"]:
-            return True
-        else:
-            return False
+            return can_assign_src_to_sink(src["items"], sink["items"])
+    elif isinstance(src, list):
+        for t in src:
+            if can_assign_src_to_sink(t, sink):
+                return True
+    elif isinstance(sink, list):
+        for t in sink:
+            if can_assign_src_to_sink(src, t):
+                return True
     else:
         return src == sink
-
+    return False
 
 def object_from_state(state, parms, frag_only, supportsMultipleInput, sourceField):
     # type: (Dict[unicode, WorkflowStateItem], List[Dict[unicode, Any]], bool, bool, unicode) -> Dict[unicode, Any]

tests/echo-cwlrun-job.yaml

@@ -0,0 +1,6 @@
+cwl:tool: echo.cwl
+cwl:requirements:
+ - class: DockerRequirement
+   dockerPull: debian
+
+inp: "Hoopla!"

tests/echo-job.yaml

@@ -0,0 +1,5 @@
+cwl:requirements:
+ - class: DockerRequirement
+   dockerPull: debian
+
+inp: "Howdy!"

tests/test_examples.py

@@ -4,7 +4,7 @@
 import cwltool.expression as expr
 import cwltool.factory
 import cwltool.process
-
+import cwltool.workflow
 
 class TestParamMatching(unittest.TestCase):
 
@@ -150,6 +150,24 @@ def loadref(base, p):
                                        set(("$include", "$schemas", "path")),
                                                   loadref), indent=4)
 
+class TestTypeCompare(unittest.TestCase):
+    def test_typecompare(self):
+        self.assertTrue(cwltool.workflow.can_assign_src_to_sink(
+            {'items': ['string', 'null'], 'type': 'array'},
+            {'items': ['string', 'null'], 'type': 'array'}))
+
+        self.assertTrue(cwltool.workflow.can_assign_src_to_sink(
+            {'items': ['string'], 'type': 'array'},
+            {'items': ['string', 'null'], 'type': 'array'}))
+
+        self.assertTrue(cwltool.workflow.can_assign_src_to_sink(
+            {'items': ['string', 'null'], 'type': 'array'},
+            {'items': ['string'], 'type': 'array'}))
+
+        self.assertFalse(cwltool.workflow.can_assign_src_to_sink(
+            {'items': ['string'], 'type': 'array'},
+            {'items': ['int'], 'type': 'array'}))
+
 
 if __name__ == '__main__':
     unittest.main()