Tests

Author: Peter Amstutz

cwltool/load_tool.py

@@ -251,7 +251,7 @@ def validate_document(document_loader,  # type: Loader
         metadata[u"cwl:defaults"] = jobobj
 
     if overrides:
-        metadata[u"cwl:overrides"] = overrides
+        metadata[u"cwltool:overrides"] = overrides
 
     return document_loader, avsc_names, processobj, metadata, uri
 

cwltool/main.py

@@ -868,6 +868,8 @@ def main(argsl=None,  # type: List[str]
                 stdout.write(json.dumps(processobj, indent=4))
                 return 0
 
+            overrides.extend(metadata.get("cwltool:overrides", []))
+
             conf_file = getattr(args, "beta_dependency_resolvers_configuration", None)  # Text
             use_conda_dependencies = getattr(args, "beta_conda_dependencies", None)  # Text
 
@@ -879,6 +881,7 @@ def main(argsl=None,  # type: List[str]
                 make_tool_kwds["job_script_provider"] = dependencies_configuration
 
             make_tool_kwds["find_default_container"] = functools.partial(find_default_container, args)
+            make_tool_kwds["overrides"] = overrides
 
             tool = make_tool(document_loader, avsc_names, metadata, uri,
                              makeTool, make_tool_kwds)

cwltool/process.py

@@ -422,9 +422,8 @@ def avroize_type(field_type, name_prefix=""):
             avroize_type(field_type["items"], name_prefix)
     return field_type
 
-def get_overrides(metadata, toolid):  # type: (Dict[Text, Any], Text) -> List[Dict[Text, Any]]
+def get_overrides(overrides, toolid):  # type: (List[Dict[Text, Any]], Text) -> List[Dict[Text, Any]]
     req = []  # type: List[Dict[Text, Any]]
-    overrides = metadata.get("cwl:overrides", [])
     if not isinstance(overrides, list):
         raise validate.ValidationException("Expected overrides to be a list, but was %s" % type(overrides))
     for ov in overrides:
@@ -468,7 +467,7 @@ def __init__(self, toolpath_object, **kwargs):
         self.tool = toolpath_object
         self.requirements = (kwargs.get("requirements", []) +
                              self.tool.get("requirements", []) +
-                             get_overrides(self.metadata, self.tool["id"]))
+                             get_overrides(kwargs.get("overrides", []), self.tool["id"]))
         self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])
         self.formatgraph = None  # type: Graph
         if "loader" in kwargs:

cwltool/workflow.py

@@ -521,6 +521,8 @@ def __init__(self, toolpath_object, **kwargs):
             try:
                 self.steps.append(WorkflowStep(step, n, **kwargs))
             except validate.ValidationException as v:
+                if _logger.isEnabledFor(logging.DEBUG):
+                    _logger.exception("Validation failed at")
                 validation_errors.append(v)
 
         if validation_errors:
@@ -668,7 +670,7 @@ def __init__(self, toolpath_object, pos, **kwargs):
 
         kwargs["requirements"] = (kwargs.get("requirements", []) +
                                   toolpath_object.get("requirements", []) +
-                                  get_overrides(kwargs["metadata"], self.id))
+                                  get_overrides(kwargs["overrides"], self.id))
         kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get("hints", [])
 
         try:
@@ -681,7 +683,7 @@ def __init__(self, toolpath_object, pos, **kwargs):
                     strict=kwargs.get("strict"),
                     fetcher_constructor=kwargs.get("fetcher_constructor"),
                     resolver=kwargs.get("resolver"),
-                    overrides=kwargs.get("metadata", {}).get("overrides"))
+                    overrides=kwargs.get("overrides"))
         except validate.ValidationException as v:
             raise WorkflowException(
                 u"Tool definition %s failed validation:\n%s" %

tests/override/echo-wf.cwl

@@ -0,0 +1,14 @@
+cwlVersion: v1.0
+class: Workflow
+inputs:
+  m1: string
+outputs:
+  out:
+    type: string
+    outputSource: step1/out
+steps:
+  step1:
+    in:
+      m1: m1
+    out: [out]
+    run: echo.cwl

tests/override/ov2.yml

@@ -0,0 +1,5 @@
+cwltool:overrides:
+  "echo-wf.cwl#step1":
+    - class: EnvVarRequirement
+      envDef:
+        MESSAGE: hello5

tests/override/ov3.yml

@@ -0,0 +1,5 @@
+cwltool:overrides:
+  "echo-wf.cwl":
+    - class: EnvVarRequirement
+      envDef:
+        MESSAGE: hello6

tests/test_override.py

@@ -43,9 +43,24 @@ def test_overrides(self):
                                stdout=sio), 0)
         self.assertEquals({"out": "zing hello4"}, json.loads(sio.getvalue()))
 
+
+        sio = StringIO()
+        self.assertEquals(main(["--overrides", get_data('tests/override/ov.yml'),
+                                get_data('tests/override/echo-wf.cwl'),
+                                get_data('tests/override/echo-job.yml')],
+                               stdout=sio), 0)
+        self.assertEquals({"out": "zing hello2"}, json.loads(sio.getvalue()))
+
         sio = StringIO()
         self.assertEquals(main(["--overrides", get_data('tests/override/ov2.yml'),
                                 get_data('tests/override/echo-wf.cwl'),
                                 get_data('tests/override/echo-job.yml')],
                                stdout=sio), 0)
         self.assertEquals({"out": "zing hello5"}, json.loads(sio.getvalue()))
+
+        sio = StringIO()
+        self.assertEquals(main(["--overrides", get_data('tests/override/ov3.yml'),
+                                get_data('tests/override/echo-wf.cwl'),
+                                get_data('tests/override/echo-job.yml')],
+                               stdout=sio), 0)
+        self.assertEquals({"out": "zing hello6"}, json.loads(sio.getvalue()))