add stderr support

cwltool/cwltest.py

@@ -114,7 +114,7 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
     if "output" in t:
         checkkeys = ["output"]
     else:
-        checkkeys = ["args", "stdin", "stdout", "createfiles"]
+        checkkeys = ["args", "stdin", "stderr", "stdout", "createfiles"]
 
     for key in checkkeys:
         try:

cwltool/draft2tool.py

@@ -206,6 +206,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
         j.builder = builder
         j.joborder = builder.job
         j.stdin = None
+        j.stderr = None
         j.stdout = None
         j.successCodes = self.tool.get("successCodes")
         j.temporaryFailCodes = self.tool.get("temporaryFailCodes")
@@ -227,6 +228,11 @@ def rm_pending_output_callback(output_callback, jobcachepending,
             j.stdin = builder.do_eval(self.tool["stdin"])
             reffiles.add(j.stdin)
 
+        if self.tool.get("stderr"):
+            j.stderr = builder.do_eval(self.tool["stderr"])
+            if os.path.isabs(j.stderr) or ".." in j.stderr:
+                raise validate.ValidationException("stderr must be a relative path")
+
         if self.tool.get("stdout"):
             j.stdout = builder.do_eval(self.tool["stdout"])
             if os.path.isabs(j.stdout) or ".." in j.stdout:

cwltool/job.py

@@ -44,6 +44,7 @@ def __init__(self):  # type: () -> None
         self.builder = None  # type: Builder
         self.joborder = None  # type: Dict[str,str]
         self.stdin = None  # type: str
+        self.stderr = None  # type: str
         self.stdout = None  # type: str
         self.successCodes = None  # type: Iterable[int]
         self.temporaryFailCodes = None  # type: Iterable[int]
@@ -137,6 +138,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                         env[key] = value
 
         stdin = None  # type: Union[IO[Any],int]
+        stderr = None  # type: IO[Any]
         stdout = None  # type: IO[Any]
 
         scr, _ = get_feature(self, "ShellCommandRequirement")
@@ -146,12 +148,13 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
         else:
             shouldquote = needs_shell_quoting_re.search
 
-        _logger.info(u"[job %s] %s$ %s%s%s",
+        _logger.info(u"[job %s] %s$ %s%s%s%s",
                      self.name,
                      self.outdir,
                      " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                      u' < %s' % (self.stdin) if self.stdin else '',
-                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '')
+                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
+                     u' \2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')
 
         if dry_run:
             return (self.outdir, {})
@@ -178,6 +181,15 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             else:
                 stdin = subprocess.PIPE
 
+            if self.stderr:
+                abserr = os.path.join(self.outdir, self.stderr)
+                dnerr = os.path.dirname(abserr)
+                if dnerr and not os.path.exists(dnerr):
+                    os.makedirs(dnerr)
+                stderr = open(abserr, "wb")
+            else:
+                stderr = sys.stderr
+
             if self.stdout:
                 absout = os.path.join(self.outdir, self.stdout)
                 dn = os.path.dirname(absout)
@@ -191,6 +203,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                                   shell=False,
                                   close_fds=True,
                                   stdin=stdin,
+                                  stderr=stderr,
                                   stdout=stdout,
                                   env=env,
                                   cwd=self.outdir)
@@ -203,6 +216,9 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             if isinstance(stdin, file):
                 stdin.close()
 
+            if stderr is not sys.stderr:
+                stderr.close()
+
             if stdout is not sys.stderr:
                 stdout.close()
 

cwltool/main.py

@@ -183,6 +183,8 @@ def output_callback(out, processStatus):
         a = {"args": job.command_line}
         if job.stdin:
             a["stdin"] = job.pathmapper.mapper(job.stdin)[1]
+        if job.stderr:
+            a["stderr"] = job.stderr
         if job.stdout:
             a["stdout"] = job.stdout
         if job.generatefiles:

cwltool/schemas/README.md

@@ -14,6 +14,11 @@ CWL builds on technologies such as [JSON-LD](http://json-ld.org) and
 CWL is designed to express workflows for data-intensive science, such as
 Bioinformatics, Medical Imaging, Chemistry, Physics, and Astronomy.
 
+## CWL User Guide
+
+[User guide for the current stable specification (draft-3)](http://www.commonwl.org/draft-3/UserGuide.html),
+provides a gentle introduction to writing CWL command line tool and workflow descriptions.
+
 ## CWL Specification
 
 The current stable specification is [draft 3](http://www.commonwl.org/draft-3/):
@@ -55,6 +60,10 @@ Some of the software supporting running Common Workflow Language tools or workfl
 * [Apache Taverna](http://taverna.incubator.apache.org/),
   [Apache Taverna wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Taverna)
 
+We continuously run the CWL conformance tests on several implementations:
+
+https://ci.commonwl.org
+
 ## Examples
 
 [Github repository of example tools and workflows.](https://github.com/common-workflow-language/workflows)

cwltool/schemas/draft-3/UserGuide.yml

@@ -20,7 +20,7 @@
     - |
       ```
 
-      Use a JSON object in a separate file to describe the input of a run:
+      Use a YAML object in a separate file to describe the input of a run:
 
       *echo-job.yml*
       ```
@@ -106,10 +106,11 @@
       Notice that "example_file", as a `File` type, must be provided as an
       object with the fields `class: File` and `path`.
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Next, create a whale.txt and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
 
       ```
+      $ touch whale.txt
       $ cwl-runner inp.cwl inp-job.yml
       [job 140020149614160] /home/example$ echo -f -i42 --example-string hello --file=/home/example/whale.txt
       -f -i42 --example-string hello --file=/home/example/whale.txt
@@ -218,9 +219,10 @@
     - |
       ```
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Next, create a tar file for the example and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
       ```
+      $ touch hello.txt && tar -cvf hello.tar hello.txt
       $ cwl-runner tar.cwl tar-job.yml
       [job 139868145165200] $ tar xf /home/example/hello.tar
       Final process status is success
@@ -308,9 +310,10 @@
     - |
       ```
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Create your input files and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
       ```
+      $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar
       $ cwl-runner tar-param.cwl tar-param-job.yml
       [job 139868145165200] $ tar xf /home/example/hello.tar goodbye.txt
       Final process status is success
@@ -374,11 +377,12 @@
     - |
       ```
 
-      Now invoke `cwl-runner` providing the tool wrapper and the input object
+      Provide a hello.js and invoke `cwl-runner` providing the tool wrapper and the input object
       on the command line:
 
       ```
-      $ cwl-runner example-docker.cwl example-docker-job.yml
+      $ echo "console.log(\"Hello World\");" > hello.js
+      $ cwl-runner docker.cwl docker-job.yml
       [job 140259721854416] /home/example$ docker run -i --volume=/home/example/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpDLs5hm:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp node:slim node /var/lib/cwl/job369354770_examples/hello.js
       Hello world!
       Final process status is success
@@ -417,11 +421,12 @@
     - |
       ```
 
-      Now invoke `cwl-runner` providing the tool wrapper and the input object
+      Now create a sample Java file and invoke `cwl-runner` providing the tool wrapper and the input object
       on the command line:
 
       ```
-      $ cwl-runner example-arguments.cwl example-arguments-job.yml
+      $ echo "public class Hello {}" > Hello.java
+      $ cwl-runner arguments.cwl arguments-job.yml
       [job 140051188854928] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job710906416_example/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpdlQDWi:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job710906416_examples/Hello.java
       Final process status is success
       {
@@ -855,4 +860,4 @@
       The second step `compile` depends on the results from the first step by
       connecting the input parameter `src` to the output parameter of `untar`
       using `#untar/example_out`.  The output of this step `classfile` is
-      connected to the `outputs` section for the Workflow, described above.
+      connected to the `outputs` section for the Workflow, described above.

cwltool/schemas/draft-3/cwl-runner.cwl

@@ -0,0 +1,78 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: "cwl:draft-3"
+
+description: |
+ Generic interface to run a Common Workflow Language tool or workflow from the
+ command line. To be implemented by each CWL compliant execution platform for
+ testing conformance to the standard and optionally for use by users.
+
+inputs:
+ - id: outdir
+   type: string
+   default: outdir
+   description: |
+    Output directory, defaults to the current directory
+   inputBinding:
+    prefix: "--outdir"
+
+ - id: quiet
+   type: boolean
+   description: no diagnostic output
+   inputBinding:
+    prefix: "--quiet"
+
+ - id: toolfile
+   type: [ "null", File ]
+   description: |
+    The tool or workflow description to run. Optional if the jobfile has a
+    `cwl:tool` field to indicate the tool or workflow description to run.
+   inputBinding:
+    position: 1
+
+ - id: jobfile
+   type: File
+   inputBinding:
+    position: 2
+
+ - id: conformance-test
+   type: boolean
+   inputBinding:
+    prefix: "--conformance-test"
+
+ - id: basedir
+   type: string
+   inputBinding:
+    prefix: "--basedir"
+
+ - id: no-container
+   type: boolean
+   description: |
+    Do not execute jobs in a Docker container, even when listed as a Requirement
+   inputBinding:
+    prefix: "--no-container"
+
+ - id: tmp-outdir-prefix
+   type: string
+   description: |
+    Path prefix for temporary directories. Useful for OS X so that boot2docker
+    writes to /Users
+   inputBinding:
+    prefix: "--tmp-outdir-prefix"
+
+ - id: tmpdir-prefix
+   type: string
+   description: |
+    Path prefix for temporary directories
+   inputBinding:
+    prefix: "--tmpdir-prefix"
+
+baseCommand: cwl-runner
+
+stdout: output-object.json
+
+outputs:
+ - id: output-object
+   type: File
+   outputBinding:
+     glob: output-object.json

cwltool/schemas/draft-4/CommandLineTool.yml

@@ -38,8 +38,12 @@ $graph:
     - |
       ## Introduction to draft 4
 
-      This specification represents the third milestone of the CWL group.
-      Since draft-3, this draft introduces the following major changes and additions:
+      This specification represents the fourth milestone of the CWL group.
+      Since draft-3, this draft introduces the following changes and additions:
+
+        * Preliminary support for initialWorkDir & the Directory type
+        * "id: name" is now just "name"; "class: Classname" is now just
+          "Classtname".
 
       ## Purpose
 
@@ -430,6 +434,18 @@ $graph:
       doc: |
         A path to a file whose contents must be piped into the command's
         standard input stream.
+    - name: stderr
+      type: ["null", string, "#Expression"]
+      doc: |
+        Capture the command's standard error stream to a file written to
+        the designated output directory.
+
+        If `stderr` is a string, it specifies the file name to use.
+
+        If `stderr` is an expression, the expression is evaluated and must
+        return a string with the file name to use to capture stderr.  If the
+        return value is not a string, or the resulting path contains illegal
+        characters (such as the path separator `/`) it is an error.
     - name: stdout
       type: ["null", string, "#Expression"]
       doc: |

cwltool/schemas/draft-4/UserGuide.yml

@@ -17,7 +17,7 @@
     - |
       ```
 
-      Use a JSON object in a separate file to describe the input of a run:
+      Use a YAML object in a separate file to describe the input of a run:
 
       *echo-job.yml*
       ```
@@ -103,10 +103,11 @@
       Notice that "example_file", as a `File` type, must be provided as an
       object with the fields `class: File` and `path`.
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Next, create a whale.txt and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
 
       ```
+      $ touch whale.txt
       $ cwl-runner inp.cwl inp-job.yml
       [job 140020149614160] /home/example$ echo -f -i42 --example-string hello --file=/home/example/whale.txt
       -f -i42 --example-string hello --file=/home/example/whale.txt
@@ -215,9 +216,10 @@
     - |
       ```
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Next, create a tar file for the example and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
       ```
+      $ touch hello.txt && tar -cvf hello.tar hello.txt
       $ cwl-runner tar.cwl tar-job.yml
       [job 139868145165200] $ tar xf /home/example/hello.tar
       Final process status is success
@@ -305,9 +307,10 @@
     - |
       ```
 
-      Invoke `cwl-runner` with the tool wrapper and the input object on the
+      Create your input files and invoke `cwl-runner` with the tool wrapper and the input object on the
       command line:
       ```
+      $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar
       $ cwl-runner example-tar-param.cwl example-tar-param-job.yml
       [job 139868145165200] $ tar xf /home/example/hello.tar goodbye.txt
       Final process status is success
@@ -371,11 +374,12 @@
     - |
       ```
 
-      Now invoke `cwl-runner` providing the tool wrapper and the input object
+      Provide a hello.js and invoke `cwl-runner` providing the tool wrapper and the input object
       on the command line:
 
       ```
-      $ cwl-runner example-docker.cwl example-docker-job.yml
+      $ echo "console.log(\"Hello World\");" > hello.js
+      $ cwl-runner docker.cwl docker-job.yml
       [job 140259721854416] /home/example$ docker run -i --volume=/home/example/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpDLs5hm:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp node:slim node /var/lib/cwl/job369354770_examples/hello.js
       Hello world!
       Final process status is success
@@ -414,11 +418,12 @@
     - |
       ```
 
-      Now invoke `cwl-runner` providing the tool wrapper and the input object
+      Now create a sample Java file and invoke `cwl-runner` providing the tool wrapper and the input object
       on the command line:
 
       ```
-      $ cwl-runner example-arguments.cwl example-arguments-job.yml
+      $ echo "public class Hello {}" > Hello.java
+      $ cwl-runner arguments.cwl arguments-job.yml
       [job 140051188854928] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job710906416_example/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpdlQDWi:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job710906416_examples/Hello.java
       Final process status is success
       {