Removed cwltool deps (#135)

* Removed javascript cwltool deps

Before this commit, JS evaluation was taken from the cwltool codebase.
Now, all the logic has been migrated here. Plus, the code has been made
more modular, so that JSEngines can be substituted at runtime without
having to rewrite the entire logic. A basic NodeJSEngine has been added
and loaded by default, which contains the original logic.

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: GlassOfWhiskey

MANIFEST.in

@@ -3,6 +3,9 @@ include Makefile
 include MANIFEST.in
 include LICENSE.txt
 include *requirements.txt
+include cwl_utils/cwlNodeEngine.js
+include cwl_utils/cwlNodeEngineJSConsole.js
+include cwl_utils/cwlNodeEngineWithContext.js
 include testdata/*.cwl
 include testdata/*.yaml
 include testdata/*.input

cwl_utils/cwlNodeEngine.js

@@ -0,0 +1,21 @@
+"use strict";
+process.stdin.setEncoding("utf8");
+var incoming = "";
+process.stdin.on("data", function(chunk) {
+  incoming += chunk;
+  var i = incoming.indexOf("\n");
+  if (i > -1) {
+    try{
+      var fn = JSON.parse(incoming.substr(0, i));
+      incoming = incoming.substr(i+1);
+      process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {})) + "\n");
+    }
+    catch(e){
+      console.error(e)
+    }
+    /*strings to indicate the process has finished*/
+    console.log("r1cepzbhUTxtykz5XTC4");
+    console.error("r1cepzbhUTxtykz5XTC4");
+  }
+});
+process.stdin.on("end", process.exit);

cwl_utils/cwlNodeEngineJSConsole.js

@@ -0,0 +1,32 @@
+"use strict";
+function js_console_log(){
+    console.error("[log] "+require("util").format.apply(this, arguments).split("\n").join("\n[log] "));
+}
+function js_console_err(){
+    console.error("[err] "+require("util").format.apply(this, arguments).split("\n").join("\n[err] "));
+}
+process.stdin.setEncoding("utf8");
+var incoming = "";
+process.stdin.on("data", function(chunk) {
+  incoming += chunk;
+  var i = incoming.indexOf("\n");
+  if (i > -1) {
+      try{
+        var fn = JSON.parse(incoming.substr(0, i));
+        incoming = incoming.substr(i+1);
+        process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {
+        console: {
+            log: js_console_log, 
+            error: js_console_err
+        }
+        })) + "\n");
+    }
+    catch(e){
+        console.error(e)
+    }
+    /*strings to indicate the process has finished*/
+    console.log("r1cepzbhUTxtykz5XTC4");
+    console.error("r1cepzbhUTxtykz5XTC4");
+  }
+});
+process.stdin.on("end", process.exit);

cwl_utils/cwlNodeEngineWithContext.js

@@ -0,0 +1,37 @@
+"use strict";
+process.stdin.setEncoding("utf8");
+var incoming = "";
+var firstInput = true;
+var context = {};
+
+process.stdin.on("data", function(chunk) {
+  incoming += chunk;
+  var i = incoming.indexOf("\n");
+  while (i > -1) {
+    try{
+      var input = incoming.substr(0, i);
+      incoming = incoming.substr(i+1);
+      var fn = JSON.parse(input);
+      if(firstInput){
+        context = require("vm").runInNewContext(fn, {});
+      }
+      else{
+        process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, context)) + "\n");
+      }
+    }
+    catch(e){
+      console.error(e);
+    }
+    if(firstInput){
+      firstInput = false;
+    }
+    else{
+      /*strings to indicate the process has finished*/
+      console.log("r1cepzbhUTxtykz5XTC4");
+      console.error("r1cepzbhUTxtykz5XTC4");
+    }
+
+    i = incoming.indexOf("\n");
+  }
+});
+process.stdin.on("end", process.exit);

cwl_utils/cwl_expression_refactor.py

@@ -19,9 +19,10 @@
     Union,
 )
 
-from cwltool.loghandler import _logger as _cwltoollogger
 from ruamel import yaml
 
+from cwl_utils.loghandler import _logger as _cwlutilslogger
+
 if TYPE_CHECKING:
     from typing_extensions import Protocol
 else:
@@ -31,7 +32,7 @@
 defaultStreamHandler = logging.StreamHandler()  # pylint: disable=invalid-name
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
-_cwltoollogger.setLevel(100)
+_cwlutilslogger.setLevel(100)
 
 from cwl_utils import (
     cwl_v1_0_expression_refactor,

cwl_utils/cwl_normalizer.py

@@ -9,24 +9,18 @@
 from pathlib import Path
 from typing import List, MutableSequence, Optional, Set
 
-from cwltool.context import LoadingContext, RuntimeContext
-from cwltool.load_tool import (
-    fetch_document,
-    resolve_and_validate_document,
-    resolve_tool_uri,
-)
-from cwltool.loghandler import _logger as _cwltoollogger
-from cwltool.main import print_pack
-from cwltool.process import use_standard_schema
 from cwlupgrader import main as cwlupgrader
 from ruamel import yaml
 from schema_salad.sourceline import add_lc_filename
 
+from cwl_utils.loghandler import _logger as _cwlutilslogger
+from cwl_utils.pack import pack
+
 _logger = logging.getLogger("cwl-normalizer")  # pylint: disable=invalid-name
 defaultStreamHandler = logging.StreamHandler()  # pylint: disable=invalid-name
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
-_cwltoollogger.setLevel(100)
+_cwlutilslogger.setLevel(100)
 
 from cwl_utils import cwl_v1_2_expression_refactor
 from cwl_utils.parser.cwl_v1_2 import load_document_by_yaml, save
@@ -127,29 +121,7 @@ def run(args: argparse.Namespace) -> int:
         else:
             with tempfile.TemporaryDirectory() as tmpdirname:
                 path = Path(tmpdirname) / Path(document).name
-                with open(path, "w") as handle:
-                    yaml.main.round_trip_dump(result, handle)
-                # TODO replace the cwltool based packing with a parser_v1_2 based packer
-                runtimeContext = RuntimeContext()
-                loadingContext = LoadingContext()
-                use_standard_schema("v1.2")
-                # loadingContext.construct_tool_object = workflow.default_make_tool
-                # loadingContext.resolver = tool_resolver
-                loadingContext.do_update = False
-                uri, tool_file_uri = resolve_tool_uri(
-                    str(path),
-                    resolver=loadingContext.resolver,
-                    fetcher_constructor=loadingContext.fetcher_constructor,
-                )
-                loadingContext, workflowobj, uri = fetch_document(uri, loadingContext)
-                loadingContext, uri = resolve_and_validate_document(
-                    loadingContext,
-                    workflowobj,
-                    uri,
-                    preprocess_only=True,
-                    skip_schemas=True,
-                )
-                packed = print_pack(loadingContext, uri)
+                packed = pack(str(path))
         output = Path(args.dir) / Path(document).name
         with open(output, "w", encoding="utf-8") as output_filehandle:
             output_filehandle.write(packed)

cwl_utils/cwl_v1_0_expression_refactor.py

@@ -12,22 +12,20 @@
     MutableSequence,
     Optional,
     Sequence,
-    Text,
     Tuple,
     Type,
     Union,
     cast,
 )
 
-from cwltool.errors import WorkflowException
-from cwltool.expression import do_eval, interpolate
-from cwltool.sandboxjs import JavascriptException
-from cwltool.utils import CWLObjectType, CWLOutputType
 from ruamel import yaml
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
 import cwl_utils.parser.cwl_v1_0 as cwl
+from cwl_utils.errors import JavascriptException, WorkflowException
+from cwl_utils.expression import do_eval, interpolate
+from cwl_utils.types import CWLObjectType, CWLOutputType
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:

cwl_utils/cwl_v1_1_expression_refactor.py

@@ -12,22 +12,20 @@
     MutableSequence,
     Optional,
     Sequence,
-    Text,
     Tuple,
     Type,
     Union,
     cast,
 )
 
-from cwltool.errors import WorkflowException
-from cwltool.expression import do_eval, interpolate
-from cwltool.sandboxjs import JavascriptException
-from cwltool.utils import CWLObjectType, CWLOutputType
 from ruamel import yaml
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
 import cwl_utils.parser.cwl_v1_1 as cwl
+from cwl_utils.errors import JavascriptException, WorkflowException
+from cwl_utils.expression import do_eval, interpolate
+from cwl_utils.types import CWLObjectType, CWLOutputType
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:
@@ -184,7 +182,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-var ret = function(){"""
+    var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -390,7 +388,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-  return {"result": function(){"""
+      return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -1782,7 +1780,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-  return {"result": function(){"""
+      return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -12,22 +12,20 @@
     MutableSequence,
     Optional,
     Sequence,
-    Text,
     Tuple,
     Type,
     Union,
     cast,
 )
 
-from cwltool.errors import WorkflowException
-from cwltool.expression import do_eval, interpolate
-from cwltool.sandboxjs import JavascriptException
-from cwltool.utils import CWLObjectType, CWLOutputType
 from ruamel import yaml
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
 import cwl_utils.parser.cwl_v1_2 as cwl
+from cwl_utils.errors import JavascriptException, WorkflowException
+from cwl_utils.expression import do_eval, interpolate
+from cwl_utils.types import CWLObjectType, CWLOutputType
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:
@@ -184,7 +182,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-var ret = function(){"""
+    var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -390,7 +388,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-  return {"result": function(){"""
+      return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -1878,7 +1876,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-  return {"result": function(){"""
+      return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/errors.py

@@ -0,0 +1,26 @@
+class ArrayMissingItems(BaseException):
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+
+
+class JavascriptException(Exception):
+    pass
+
+
+class MissingKeyField(BaseException):
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+
+
+class MissingTypeName(BaseException):
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+
+
+class RecordMissingFields(BaseException):
+    """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py"""
+
+
+class SubstitutionError(Exception):
+    pass
+
+
+class WorkflowException(Exception):
+    pass