Merge pull request #97 from common-workflow-language/stderr_out_shortcut

initial stderr & stdout shortcut implementation

Author: mr-c

cwltool/cwltest.py

@@ -72,8 +72,9 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
                 outdir = tempfile.mkdtemp()
             test_command.extend(["--outdir={}".format(outdir),
                                  "--quiet",
-                                 t["tool"],
-                                 t["job"]])
+                                 t["tool"]])
+            if t["job"] != None:
+                test_command.extend([t["job"]])
             outstr = subprocess.check_output(test_command)
             out = {"output": json.loads(outstr)}
         else:
@@ -82,8 +83,9 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
                             "--basedir=" + args.basedir,
                             "--no-container",
                             "--quiet",
-                            t["tool"],
-                            t["job"]]
+                            t["tool"]]
+            if t["job"] != None:
+                test_command.extend([t["job"]])
 
             outstr = subprocess.check_output(test_command)
             out = yaml.load(outstr)
@@ -100,12 +102,12 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
             _logger.error(t.get("doc"))
             _logger.error("Returned non-zero")
             return 1
-    except yaml.scanner.ScannerError as e:
+    except (yaml.scanner.ScannerError, TypeError) as e:
         _logger.error(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
         _logger.error(outstr)
         _logger.error(u"Parse error %s", str(e))
 
-    pwd = os.path.abspath(os.path.dirname(t["job"]))
+    # pwd = os.path.abspath(os.path.dirname(t["job"]))
     # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"])
     # if "stdin" in t:
     #     t["stdin"] = t["stdin"].replace("$PWD", pwd)
@@ -118,7 +120,8 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
 
     for key in checkkeys:
         try:
-            compare(t.get(key), out.get(key))
+            if key in t and key in out:
+                compare(t.get(key), out.get(key))
         except CompareFail as ex:
             _logger.warn(u"""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
             _logger.warn(t.get("doc"))

cwltool/job.py

@@ -154,7 +154,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                      " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                      u' < %s' % (self.stdin) if self.stdin else '',
                      u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
-                     u' \2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')
+                     u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')
 
         if dry_run:
             return (self.outdir, {})

cwltool/load_tool.py

@@ -2,6 +2,7 @@
 """Loads a CWL document."""
 
 import os
+import uuid
 import logging
 import re
 import urlparse
@@ -41,6 +42,30 @@ def fetch_document(argsworkflow):
 
     return document_loader, workflowobj, uri
 
+def _convert_stdstreams_to_files(workflowobj):
+    # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]) -> None
+
+    if isinstance(workflowobj, dict):
+        if ('class' in workflowobj
+                and workflowobj['class'] == 'CommandLineTool'
+                and 'outputs' in workflowobj):
+            for out in workflowobj['outputs']:
+                for streamtype in ['stdout', 'stderr']:
+                    if out['type'] == streamtype:
+                        if 'outputBinding' in out:
+                            raise validate.ValidateException(
+                                    "Not allowed to specify outputBinding when"
+                                    " using %s shortcut." % streamtype)
+                        if streamtype in workflowobj:
+                            filename = workflowobj[streamtype]
+                        else:
+                            filename = unicode(uuid.uuid4())
+                            workflowobj[streamtype] = filename
+                        out['type'] = 'File'
+                        out['outputBinding'] = {'glob': filename}
+    else:
+        for entry in workflowobj:
+            _convert_stdstreams_to_files(entry)
 
 def validate_document(document_loader, workflowobj, uri,
                       enable_dev=False, strict=True, preprocess_only=False):
@@ -94,6 +119,8 @@ def validate_document(document_loader, workflowobj, uri,
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]}
 
+    _convert_stdstreams_to_files(workflowobj)
+
     if preprocess_only:
         return document_loader, avsc_names, processobj, metadata, uri
 

cwltool/schemas/draft-2/conformance_test_draft-2.yaml

@@ -1,63 +1,52 @@
-- args: [bwa, mem, -t, '4', -I, '1,2,3,4', -m, '3', draft-2/rabix/tests/test-files/chr20.fa,
-    draft-2/rabix/tests/test-files/example_human_Illumina.pe_1.fastq, draft-2/rabix/tests/test-files/example_human_Illumina.pe_2.fastq]
+- args:
   job: draft-2/bwa-mem-job.json
-  stdout: output.sam
   tool: draft-2/bwa-mem-tool.cwl
+  output:
+    args: [bwa, mem, -t, '4', -I, '1,2,3,4', -m, '3',
+      chr20.fa,
+      example_human_Illumina.pe_1.fastq,
+      example_human_Illumina.pe_2.fastq]
   doc: General test of command line generation
 
-- args: [bwa, mem, draft-2/rabix/tests/test-files/chr20.fa,
-    "-XXX",
-    "-YYY", draft-2/rabix/tests/test-files/example_human_Illumina.pe_1.fastq,
-    "-YYY", draft-2/rabix/tests/test-files/example_human_Illumina.pe_2.fastq]
+- output:
+    args: [bwa, mem, chr20.fa,
+      "-XXX",
+      "-YYY", example_human_Illumina.pe_1.fastq,
+      "-YYY", example_human_Illumina.pe_2.fastq]
   job: draft-2/bwa-mem-job.json
   tool: draft-2/binding-test.cwl
   doc: Test nested prefixes with arrays
 
-- args: [tmap, mapall, stage1, map1, --min-seq-length, '20', map2, --min-seq-length,
+- output:
+    args: [tmap, mapall, stage1, map1, --min-seq-length, '20', map2, --min-seq-length,
     '20', stage2, map1, --max-seq-length, '20', --min-seq-length, '10', --seed-length,
     '16', map2, --max-seed-hits, '-1', --max-seq-length, '20', --min-seq-length, '10']
   job: draft-2/tmap-job.json
-  stdin: draft-2/reads.fastq
-  stdout: output.sam
   tool: draft-2/tmap-tool.cwl
-  doc: Test nested command line bindings and stdin/stdout redirection
+  doc: Test nested command line bindings
 
-- args: [cat, draft-2/hello.txt]
+- output:
+    args: [cat, hello.txt]
   job: draft-2/cat-job.json
-  tool: draft-2/cat1-tool.cwl
+  tool: draft-2/cat1-testcli.cwl
   doc: Test command line with optional input (missing)
 
-- args: [cat, -n, draft-2/hello.txt]
+- output:
+    args: [cat, -n, hello.txt]
   job: draft-2/cat-n-job.json
-  tool: draft-2/cat1-tool.cwl
+  tool: draft-2/cat1-testcli.cwl
   doc: Test command line with optional input (provided)
 
-- args: [cat]
-  job: draft-2/cat-job.json
-  stdin: draft-2/hello.txt
-  tool: draft-2/cat2-tool.cwl
-  doc: Test command line with stdin redirection
-
-- args: [cat, draft-2/hello.txt]
-  job: draft-2/cat-job.json
-  stdout: output.txt
-  tool: draft-2/cat3-tool.cwl
-  doc: Test command line with stdout redirection
-
-- args: [cat]
-  job: draft-2/cat-job.json
-  stdin: draft-2/hello.txt
-  stdout: output.txt
-  tool: draft-2/cat4-tool.cwl
-  doc: Test command line with stdin and stdout redirection
-
-- args: [cat, foo.txt]
-  createfiles: {foo.txt: 'The file is draft-2/hello.txt
-
-      '}
+- output:
+    "foo": {
+        "checksum": "sha1$63da67422622fbf9251a046d7a34b7ea0fd4fead",
+        "class": "File",
+        "path": "foo.txt",
+        "size": 22
+    }
   job: draft-2/cat-job.json
   tool: draft-2/template-tool.cwl
-  doc: Test CreateFileRequirement ExpressionEngineRequirement.engineConfig features
+  doc: Test CreateFileRequirement ExpressionEngineRequirement.engineConfig feature
 
 - job: draft-2/cat-job.json
   output:

cwltool/schemas/draft-2/draft-2/args.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+import sys
+import json
+import os
+args = [os.path.basename(a) for a in sys.argv[1:]]
+with open("cwl.output.json", "w") as f:
+    json.dump({"args": args}, f)

cwltool/schemas/draft-2/draft-2/binding-test.cwl

@@ -14,6 +14,15 @@ inputs:
       inputBinding: { prefix: "-YYY" }
     inputBinding: { position: 3, prefix: "-XXX" }
 
+  - id: "#args.py"
+    type: File
+    default:
+      class: File
+      path: args.py
+    inputBinding:
+      position: -1
+
 outputs: []
 
-baseCommand: ["bwa", "mem"]
+baseCommand: python
+arguments: ["bwa", "mem"]

cwltool/schemas/draft-2/draft-2/bwa-mem-job.json

@@ -1,18 +1,18 @@
 {
     "reference": {
         "class": "File",
-        "path": "rabix/tests/test-files/chr20.fa",
+        "path": "chr20.fa",
         "size": 123,
         "checksum": "sha1$hash"
     },
     "reads": [
         {
             "class": "File",
-            "path": "rabix/tests/test-files/example_human_Illumina.pe_1.fastq"
+            "path": "example_human_Illumina.pe_1.fastq"
         },
         {
             "class": "File",
-            "path": "rabix/tests/test-files/example_human_Illumina.pe_2.fastq"
+            "path": "example_human_Illumina.pe_2.fastq"
         }
     ],
     "min_std_max_min": [

cwltool/schemas/draft-2/draft-2/bwa-mem-tool.cwl

@@ -4,11 +4,6 @@ class: CommandLineTool
 requirements:
   - import: node-engine.cwl
 
-hints:
-  - class: DockerRequirement
-    dockerPull: images.sbgenomics.com/rabix/bwa
-    dockerImageId: 9d3b9b0359cf
-
 inputs:
   - id: "#reference"
     type: File
@@ -31,14 +26,28 @@ inputs:
       prefix: "-I"
       itemSeparator: ","
 
+  - id: "#args.py"
+    type: File
+    default:
+      class: File
+      path: args.py
+    inputBinding:
+      position: -1
+
 outputs:
   - id: "#sam"
-    type: "File"
+    type: ["null", "File"]
     outputBinding: { "glob": "output.sam" }
+  - id: "#args"
+    type:
+      type: array
+      items: string
 
-baseCommand: ["bwa", "mem"]
+baseCommand: python
 
 arguments:
+  - "bwa"
+  - "mem"
   - valueFrom:
       engine: "node-engine.cwl"
       script: "$job.allocatedResources.cpu"

cwltool/schemas/draft-2/draft-2/cat1-testcli.cwl

@@ -0,0 +1,34 @@
+#!/usr/bin/env cwl-runner
+{
+    "class": "CommandLineTool",
+    "description": "Print the contents of a file to stdout using 'cat' running in a docker container.",
+    "inputs": [
+        {
+            "id": "#file1",
+            "type": "File",
+            "inputBinding": {"position": 1}
+        },
+        {
+            "id": "#numbering",
+            "type": ["null", "boolean"],
+            "inputBinding": {
+                "position": 0,
+                "prefix": "-n"
+            }
+        },
+        {
+        id: "#args.py",
+        type: File,
+        default: {
+          class: File,
+          path: args.py
+        },
+        inputBinding: {
+          position: -1
+        }
+      }
+    ],
+    "outputs": [],
+    "baseCommand": "python",
+    "arguments": ["cat"]
+}

cwltool/schemas/draft-2/draft-2/cat2-tool.cwl

@@ -17,9 +17,14 @@ requirements:
       - filename: foo.txt
         fileContent:
           engine: "#js"
-          script: 't("The file is <%= $job.file1.path %>\n")'
+          script: >
+            t("The file is <%= $job.file1.path.split('/').slice(-1)[0] %>\n")
 inputs:
   - id: "#file1"
     type: File
-outputs: []
+outputs:
+  - id: "#foo"
+    type: File
+    outputBinding:
+      glob: foo.txt
 baseCommand: ["cat", "foo.txt"]