break cyclic dependency; remove unused engine code

mr-c · mr-c · commit 7599c4948edc · 2016-03-11T15:36:16.000-08:00
diff --git a/cwltool/builder.py b/cwltool/builder.py
@@ -5,7 +5,7 @@
 import schema_salad.validate as validate
 from typing import Any, Union
 from .errors import WorkflowException
-from .process import StdFsAccess
+from .stdfsaccess import StdFsAccess
 
 CONTENT_LIMIT = 64 * 1024
 
@@ -40,6 +40,7 @@ def __init__(self):  # type: () -> None
         self.tmpdir = None  # type: str
         self.resources = None  # type: Dict[str,str]
         self.bindings = None  # type: List[Dict[str,str]]
+        self.timeout = None  # type: int
 
     def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
         # type: (Dict[str,Any], Any, List[int], List[int]) -> List[Dict[str,str]]
diff --git a/cwltool/docker.py b/cwltool/docker.py
@@ -3,7 +3,7 @@
 import sys
 import requests
 import os
-from . import process
+from .errors import ProcessWorkflowException
 import re
 import tempfile
 from typing import Any, Union
@@ -76,7 +76,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                     loadproc.stdin.close()
                 rcode = loadproc.wait()
                 if rcode != 0:
-                    raise process.WorkflowException(
+                    raise ProcessWorkflowException(
                         "Docker load returned non-zero exit status %i"
                         % (rcode))
                 found = True
@@ -104,15 +104,15 @@ def get_from_requirements(r, req, pull_image, dry_run=False):
 
         if errmsg:
             if req:
-                raise process.WorkflowException(errmsg)
+                raise ProcessWorkflowException(errmsg)
             else:
                 return None
 
         if get_image(r, pull_image, dry_run):
             return r["dockerImageId"]
         else:
             if req:
-                raise process.WorkflowException(
+                raise ProcessWorkflowException(
                     "Docker image %s not found" % r["dockerImageId"])
 
     return None
diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py
@@ -102,9 +102,6 @@ class CommandLineTool(Process):
     def __init__(self, toolpath_object, **kwargs):
         super(CommandLineTool, self).__init__(toolpath_object, **kwargs)
 
-    def makeJobRunner(self):
-        return CommandLineJob()
-
     def makePathMapper(self, reffiles, input_basedir, **kwargs):
         dockerReq, _ = self.get_requirement("DockerRequirement")
         try:
@@ -147,7 +144,7 @@ def job(self, joborder, input_basedir, output_callback, **kwargs):
 
         reffiles = set((f["path"] for f in builder.files))
 
-        j = self.makeJobRunner()
+        j = CommandLineJob()
         j.builder = builder
         j.joborder = builder.job
         j.stdin = None
diff --git a/cwltool/errors.py b/cwltool/errors.py
@@ -1,2 +1,5 @@
 class WorkflowException(Exception):
     pass
+
+class ProcessWorkflowException(WorkflowException):
+    pass
diff --git a/cwltool/expression.py b/cwltool/expression.py
@@ -1,10 +1,8 @@
-from . import docker
 import subprocess
 import json
 from .aslist import aslist
 import logging
 from .errors import WorkflowException
-from . import process
 import schema_salad.ref_resolver
 from . import sandboxjs
 import re
@@ -19,79 +17,6 @@ def jshead(engineConfig, rootvars):
                                      for k, v in rootvars.items()])
 
 
-def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
-    if ex["engine"] == "https://w3id.org/cwl/cwl#JsonPointer":
-        try:
-            obj = {"job": jobinput, "context": context,
-                   "outdir": outdir, "tmpdir": tmpdir}
-            return schema_salad.ref_resolver.resolve_json_pointer(
-                obj, ex["script"])
-        except ValueError as v:
-            raise WorkflowException("%s in %s" % (v, obj))
-
-    if ex["engine"] == "https://w3id.org/cwl/cwl#JavascriptEngine":
-        engineConfig = []  # type: List[str]
-        for r in reversed(requirements):
-            if (r["class"] == "ExpressionEngineRequirement" and
-                    r["id"] == "https://w3id.org/cwl/cwl#JavascriptEngine"):
-                engineConfig = r.get("engineConfig", [])
-                break
-        return sandboxjs.execjs(ex["script"], jshead(
-            engineConfig, (jobinput, context, tmpdir, outdir)))
-
-    for r in reversed(requirements):
-        if (r["class"] == "ExpressionEngineRequirement" and
-                r["id"] == ex["engine"]):
-            runtime = []  # type: List[str]
-
-            class DR(object):
-
-                def __init__(self):
-                    self.requirements = None  # type: str
-                    self.hints = None  # type: str
-
-            dr = DR()
-            dr.requirements = r.get("requirements", [])
-            dr.hints = r.get("hints", [])
-
-            (docker_req, docker_is_req) = process.get_feature(
-                dr, "DockerRequirement")
-            img_id = None
-            if docker_req:
-                img_id = docker.get_from_requirements(
-                    docker_req, docker_is_req, pull_image)
-            if img_id:
-                runtime = ["docker", "run", "-i", "--rm", img_id]
-
-            inp = {
-                "script": ex["script"],
-                "engineConfig": r.get("engineConfig", []),
-                "job": jobinput,
-                "context": context,
-                "outdir": outdir,
-                "tmpdir": tmpdir,
-            }
-
-            _logger.debug("Invoking expression engine %s with %s",
-                          runtime + aslist(r["engineCommand"]),
-                          json.dumps(inp, indent=4))
-
-            sp = subprocess.Popen(runtime + aslist(r["engineCommand"]),
-                                  shell=False,
-                                  close_fds=True,
-                                  stdin=subprocess.PIPE,
-                                  stdout=subprocess.PIPE)
-
-            (stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n")
-            if sp.returncode != 0:
-                raise WorkflowException("Expression engine returned non-zero "
-                                        "exit code on evaluation of\n%s"
-                                        % json.dumps(inp, indent=4))
-
-            return json.loads(stdoutdata)
-
-    raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
-
 seg_symbol = r"""\w+"""
 seg_single = r"""\['([^']|\\')+'\]"""
 seg_double = r"""\["([^"]|\\")+"\]"""
@@ -150,9 +75,6 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
         "runtime": runtime
     }
 
-    if isinstance(ex, dict) and "engine" in ex and "script" in ex:
-        return exeval(ex, jobinput, requirements, outdir, tmpdir, context,
-                      pull_image)
     if isinstance(ex, basestring):
         for r in requirements:
             if r["class"] == "InlineJavascriptRequirement":
diff --git a/cwltool/main.py b/cwltool/main.py
@@ -6,7 +6,8 @@
 import os
 import sys
 import logging
-from . import workflow
+from .default_make_tool import defaultMakeTool
+from .errors import WorkflowException
 import schema_salad.validate as validate
 import tempfile
 import schema_salad.jsonld_context
@@ -198,17 +199,16 @@ def output_callback(out, processStatus):
                 if r:
                     r.run(**kwargs)
                 else:
-                    raise workflow.WorkflowException(
-                        "Workflow cannot make any more progress.")
-        except workflow.WorkflowException:
+                    raise WorkflowException(
+                            "Workflow cannot make any more progress.")
+        except WorkflowException:
             raise
         except Exception as e:
             _logger.exception("Got workflow error")
-            raise workflow.WorkflowException("%s" % e, )
+            raise WorkflowException("%s" % e, )
 
         if final_status[0] != "success":
-            raise workflow.WorkflowException(
-                "Process status is %s" % (final_status))
+            raise WorkflowException("Process status is %s" % (final_status))
 
         return final_output[0]
 
@@ -382,7 +382,7 @@ def load_tool(argsworkflow, updateonly, strict, makeTool, debug,
         _logger.error("Tool definition failed validation:\n%s",
                       e, exc_info=(e if debug else False))
         return 1
-    except (RuntimeError, workflow.WorkflowException) as e:
+    except (RuntimeError, WorkflowException) as e:
         _logger.error("Tool definition failed initialization:\n%s",
                       e, exc_info=(e if debug else False))
         return 1
@@ -493,7 +493,7 @@ def print_deps(fn):
 
 def main(args=None,
          executor=single_job_executor,
-         makeTool=workflow.defaultMakeTool,
+         makeTool=defaultMakeTool,
          selectResources=None,
          parser=None,
          stdin=sys.stdin,
@@ -600,7 +600,7 @@ def main(args=None,
         _logger.error("Input object failed validation:\n%s",
                       e, exc_info=(e if args.debug else False))
         return 1
-    except workflow.WorkflowException as e:
+    except WorkflowException as e:
         _logger.error("Workflow error, try again with --debug for more "
                       "information:\n  %s", e,
                       exc_info=(e if args.debug else False))
diff --git a/cwltool/process.py b/cwltool/process.py
@@ -3,6 +3,7 @@
 import json
 import schema_salad.validate as validate
 import copy
+import yaml
 import logging
 from .aslist import aslist
 import schema_salad.schema
diff --git a/cwltool/workflow.py b/cwltool/workflow.py
@@ -1,4 +1,4 @@
-from . import draft2tool
+from .draft2tool import CommandLineTool, ExpressionTool
 from .aslist import aslist
 from .process import Process, empty_subtree, shortname, uniquename
 from .errors import WorkflowException
@@ -21,20 +21,6 @@
 WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value'])
 
 
-def defaultMakeTool(toolpath_object, **kwargs):
-    if not isinstance(toolpath_object, dict):
-        raise WorkflowException("Not a dict: `%s`" % toolpath_object)
-    if "class" in toolpath_object:
-        if toolpath_object["class"] == "CommandLineTool":
-            return draft2tool.CommandLineTool(toolpath_object, **kwargs)
-        elif toolpath_object["class"] == "ExpressionTool":
-            return draft2tool.ExpressionTool(toolpath_object, **kwargs)
-        elif toolpath_object["class"] == "Workflow":
-            return Workflow(toolpath_object, **kwargs)
-
-    raise WorkflowException(
-        "Missing or invalid 'class' field in %s, expecting one of: "
-        "CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"])
 
 
 def findfiles(wo, fn=None):

-Original file line number
+Diff line change
@@ @@ -1,2 +1,5 @@ @@
 class WorkflowException(Exception):
     pass
++
 +class ProcessWorkflowException(WorkflowException):
 +    pass