Refactor requirements checking. Add visitor pattern for walking over workflows.

Author: Peter Amstutz

cwltool/process.py

@@ -36,7 +36,8 @@
                                 "MultipleInputFeatureRequirement",
                                 "InlineJavascriptRequirement",
                                 "ShellCommandRequirement",
-                                "StepInputExpressionRequirement"]
+                                "StepInputExpressionRequirement",
+                                "ResourceRequirement"]
 
 cwl_files = ("Workflow.yml",
               "CommandLineTool.yml",
@@ -114,15 +115,15 @@ def open(self, fn, mode):
     def exists(self, fn):
         return os.path.exists(self._abs(fn))
 
+class UnsupportedRequirement(Exception):
+    pass
+
 def checkRequirements(rec, supportedProcessRequirements):
     if isinstance(rec, dict):
         if "requirements" in rec:
             for r in rec["requirements"]:
                 if r["class"] not in supportedProcessRequirements:
-                    raise Exception("Unsupported requirement %s" % r["class"])
-        if "scatter" in rec:
-            if isinstance(rec["scatter"], list) and rec["scatter"] > 1:
-                raise Exception("Unsupported complex scatter type '%s'" % rec.get("scatterMethod"))
+                    raise UnsupportedRequirement("Unsupported requirement %s" % r["class"])
         for d in rec:
             checkRequirements(rec[d], supportedProcessRequirements)
     if isinstance(rec, list):
@@ -211,6 +212,7 @@ def __init__(self, toolpath_object, **kwargs):
         else:
             self.formatgraph = None
 
+        checkRequirements(self.tool, supportedProcessRequirements)
         self.validate_hints(self.tool.get("hints", []), strict=kwargs.get("strict"))
 
         self.schemaDefs = {}
@@ -270,10 +272,6 @@ def _init_job(self, joborder, input_basedir, **kwargs):
             if d not in builder.job and "default" in i:
                 builder.job[d] = i["default"]
 
-        for r in self.requirements:
-            if r["class"] not in supportedProcessRequirements:
-                raise WorkflowException("Unsupported process requirement %s" % (r["class"]))
-
         # Validate job order
         try:
             validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job)
@@ -363,6 +361,9 @@ def validate_hints(self, hints, strict):
     def get_requirement(self, feature):
         return get_feature(self, feature)
 
+    def visit(self, op):
+        self.tool = op(self.tool)
+
 def empty_subtree(dirpath):
     # Test if a directory tree contains any files (does not count empty
     # subdirectories)

cwltool/workflow.py

@@ -373,6 +373,9 @@ def job(self, joborder, basedir, output_callback, **kwargs):
         for w in wj.job(builder.job, basedir, output_callback, **kwargs):
             yield w
 
+    def visit(self, op):
+        self.tool = op(self.tool)
+        self.steps = [op(s) for s in self.steps]
 
 class WorkflowStep(Process):
     def __init__(self, toolpath_object, pos, **kwargs):