Merge pull request #182 from common-workflow-language/cwl-v1.1.0-dev1

CWL v1.1.0-dev1 with stdin shortcut

Author: mr-c

cwltool/load_tool.py

@@ -47,22 +47,38 @@ def _convert_stdstreams_to_files(workflowobj):
 
     if isinstance(workflowobj, dict):
         if ('class' in workflowobj
-                and workflowobj['class'] == 'CommandLineTool'
-                and 'outputs' in workflowobj):
-            for out in workflowobj['outputs']:
-                for streamtype in ['stdout', 'stderr']:
-                    if out['type'] == streamtype:
-                        if 'outputBinding' in out:
+                and workflowobj['class'] == 'CommandLineTool'):
+            if 'outputs' in workflowobj:
+                for out in workflowobj['outputs']:
+                    for streamtype in ['stdout', 'stderr']:
+                        if out['type'] == streamtype:
+                            if 'outputBinding' in out:
+                                raise ValidationException(
+                                    "Not allowed to specify outputBinding when"
+                                    " using %s shortcut." % streamtype)
+                            if streamtype in workflowobj:
+                                filename = workflowobj[streamtype]
+                            else:
+                                filename = Text(uuid.uuid4())
+                                workflowobj[streamtype] = filename
+                            out['type'] = 'File'
+                            out['outputBinding'] = {'glob': filename}
+            if 'inputs' in workflowobj:
+                for inp in workflowobj['inputs']:
+                    if inp['type'] == 'stdin':
+                        if 'inputBinding' in inp:
                             raise ValidationException(
-                                "Not allowed to specify outputBinding when"
-                                " using %s shortcut." % streamtype)
-                        if streamtype in workflowobj:
-                            filename = workflowobj[streamtype]
+                                "Not allowed to specify inputBinding when"
+                                " using stdin shortcut.")
+                        if 'stdin' in workflowobj:
+                            raise ValidationException(
+                                "Not allowed to specify stdin path when"
+                                " using stdin type shortcut.")
                         else:
-                            filename = Text(uuid.uuid4())
-                            workflowobj[streamtype] = filename
-                        out['type'] = 'File'
-                        out['outputBinding'] = {'glob': filename}
+                            workflowobj['stdin'] = \
+                                "$(inputs.%s.path)" % \
+                                inp['id'].rpartition('#')[2]
+                            inp['type'] = 'File'
         else:
             for entry in workflowobj.itervalues():
                 _convert_stdstreams_to_files(entry)

cwltool/main.py

@@ -607,7 +607,7 @@ def main(argsl=None,
                 return 0
 
             tool = make_tool(document_loader, avsc_names, metadata, uri,
-                    makeTool, {})
+                    makeTool, vars(args))
         except (validate.ValidationException) as exc:
             _logger.error(u"Tool definition failed validation:\n%s", exc,
                           exc_info=args.debug)

cwltool/process.py

@@ -91,7 +91,7 @@ def get_schema(version):
 
     cache = {}
     version = version.split("#")[-1]
-    if 'dev' in version:
+    if '.dev' in version:
         version = ".".join(version.split(".")[:-1])
     for f in cwl_files:
         try:

cwltool/schemas/README.md

@@ -1,94 +1,80 @@
 Common Workflow Language
 ========================
 
-The Common Workflow Language (CWL) is an informal, multi-vendor working group
-consisting of various organizations and individuals that have an interest in
-portability of data analysis workflows.  Our goal is to create specifications
-that enable data scientists to describe analysis tools and workflows that are
-powerful, easy to use, portable, and support reproducibility.
+[![GitHub stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-CWL builds on technologies such as [JSON-LD](http://json-ld.org)
-for data modeling and [Docker](http://docker.com) for portable runtime\
-environments.
-
-CWL is designed to express workflows for data-intensive science, such as
-Bioinformatics, Medical Imaging, Chemistry, Physics, and Astronomy.
+The Common Workflow Language (CWL) is a specification for describing analysis
+workflows and tools that are portable and scalable across a variety of software
+and hardware environments, from workstations to cluster, cloud, and high
+performance computing (HPC) environments.  CWL is designed to meet the needs of
+data-intensive science, such as Bioinformatics, Medical Imaging, Astronomy,
+Physics, and Chemistry.
 
-## CWL User Guide
+CWL is developed by an informal, multi-vendor working group consisting of
+organizations and individuals interested enabling scientists to share data
+analysis workflows.  [The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
 
-[User guide for the current stable specification (v1.0)](http://www.commonwl.org/v1.0/UserGuide.html),
-provides a gentle introduction to writing CWL command line tool and workflow descriptions.
+CWL builds on technologies such as [JSON-LD](http://json-ld.org)
+for data modeling and [Docker](http://docker.com) for portable runtime
+environments.
 
-## CWL Specification
+## User Guide
 
-The current stable specification is [v1.0](http://www.commonwl.org/v1.0/):
+The [CWL user guide (v1.0)](http://www.commonwl.org/v1.0/UserGuide.html) provides a
+gentle introduction to learning how to write CWL command line tool and workflow
+descriptions.
 
-http://www.commonwl.org/v1.0/
+## Specification
 
-Older drafts: [draft-1](https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-1), [draft-2](http://www.commonwl.org/draft-2/), [draft-3](http://www.commonwl.org/draft-3/)
+For developers and advanced users, the current
+[CWL specification (v1.0)](http://www.commonwl.org/v1.0/) provides
+authoritative documentation of the execution of CWL documents.  Links older
+drafts:
+[draft-1](https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-1),
+[draft-2](http://www.commonwl.org/draft-2/),
+[draft-3](http://www.commonwl.org/draft-3/)
 
 ### Citation
 
 Please cite https://dx.doi.org/10.6084/m9.figshare.3115156.v2
 
 ## Implementations
 
-Some of the software supporting running Common Workflow Language tools or workflows:
-
-* [cwltool (reference implementation)](https://github.com/common-workflow-language/cwltool),
-  [cwltool wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwltool-%28reference-implementation%29)
-* [Rabix](https://github.com/rabix/rabix),
-  [Rabix wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Rabix)
-* [Arvados](https://arvados.org),
-  [Arvados wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Arvados)
-* [Galaxy](https://github.com/common-workflow-language/Galaxy),
-  [Galaxy wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Galaxy)
-* [Parallel Recipes](https://github.com/yvdriess/precipes),
-  [Parallel Recipes wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Parallel-Recipes)
-* [Toil](https://github.com/BD2KGenomics/toil),
-  [Toil wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Toil)
-* [CancerCollaboratory](https://github.com/CancerCollaboratory),
-  [CancerCollaboratory wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/CancerCollaboratory)
-* [Airflow (SciDAP)](https://github.com/SciDAP/scidap),
-  [Airflow wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/SciDAP)
-* [cwl2script](https://github.com/common-workflow-language/cwl2script),
-  [cwl2script page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwl2script)
-* [Apache Taverna](http://taverna.incubator.apache.org/),
-  [Apache Taverna wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Taverna)
-
-We continuously run the CWL conformance tests on several implementations:
-
-https://ci.commonwl.org
-
-## Examples
-
-[Github repository of example tools and workflows.](https://github.com/common-workflow-language/workflows)
-
-## Support
-
-The best place to ask a question about all things CWL is on
+|Software|Description|CWL support|Platform support|Maturity|
+|--------|-----------|-----------|--------|--------|
+|[cwltool](https://github.com/common-workflow-language/cwltool)|Reference implementation of CWL|[![Build Status](https://ci.commonwl.org/job/cwltool-conformance/badge/icon)](http://ci.commonwl.org/job/cwltool-conformance/lastBuild/testReport/)|Linux, OS X, local execution only|Production|
+|[Arvados](https://arvados.org)|Distributed computing platform for data analysis on massive data sets. [Using CWL on Arvados](http://doc.arvados.org/user/cwl/cwl-runner.html)|[![Build Status](https://ci.commonwl.org/job/arvados-conformance/badge/icon)](http://ci.commonwl.org/job/arvados-conformance/lastBuild/testReport/)|AWS, GCP, Azure, Slurm|Production|
+|[Toil](https://github.com/BD2KGenomics/toil)|Toil is a workflow engine entirely written in Python.|[![Build Status](https://ci.commonwl.org/job/toil-conformance/badge/icon)](http://ci.commonwl.org/job/toil-conformance/lastBuild/testReport/)|AWS, GCP, Azure, OpenStack, Grid Engine, Mesos|Production|
+|[Apache Taverna](http://taverna.incubator.apache.org/)|Domain-independent Workflow Management System|[alpha](https://issues.apache.org/jira/browse/TAVERNA-900)|Java|Production|
+|[Galaxy](https://galaxyproject.org/)|Web-based platform for data intensive biomedical research.|[alpha](https://github.com/common-workflow-language/galaxy)|Python|Production|
+|[AWE](https://github.com/MG-RAST/AWE)|Workflow and resource management system for bioinformatics data analysis.|[alpha](https://github.com/wgerlach/AWE)|Go|Production|
+|[Funnel](https://github.com/bmeg/funnel)|Use Google Genomics Pipeline API with CWL|alpha|GCP|alpha|
+|[Rabix Bunny](https://github.com/rabix/bunny)|Reproducible Analyses for Bioinformatics|alpha|Java|alpha|
+
+## Repositories of CWL Tools and Workflows
+
+|Repository|Description|
+|----|-----------|
+|[Workflows repository](https://github.com/common-workflow-language/workflows)|Git repository of user contributed tools and workflows.|
+|[Dockstore tool registry](http://dockstore.org)|An open platform for sharing Docker-based tools described with the Common Workflow Language used by the GA4GH.|
+
+## Software for working with CWL
+
+|Software|Description|
+|--------|-----------|
+|[cwltest](https://github.com/common-workflow-language/cwltest)|CWL testing framework,  automated testing of tools and workflows written with CWL|
+|[cwl-upgrader](https://github.com/common-workflow-language/cwl-upgrader)|Upgrade CWL documents from draft-3 to v1.0|
+|[argparse2cwl](https://github.com/common-workflow-language/gxargparse#argparse2cwl)|Generate CWL CommandLineTool wrapper from Python programs that use argparse.  Also supports [click](http://click.pocoo.org/5/) argument parser.|
+|[cwl2argparse](https://github.com/common-workflow-language/cwl2argparse)|Generate Python argparse code from CWL CommandLineTool description.|
+|[pypi2cwl](https://github.com/common-workflow-language/pypi2cwl)|Automatically run argparse2cwl on any package in PyPi|
+|[cwlavro](https://github.com/common-workflow-language/cwlavro)|Java classes for loading CWL documents|
+
+## Support, Community and Contributing
+
+The recommended place to ask a question about all things CWL is on
 [Biostars](https://www.biostars.org/t/cwl/).
 
-[![Biostars
-CWL](https://www.biostars.org/static/biostar2.logo.png)](https://www.biostars.org/t/cwl/)
-</a>
-
-## Development and testing
-
-[The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
-
-[![GitHub
-stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers)
-
-There is a Jenkins server maintained by Curoverse that runs tests for the
-reference implementation, builds and uploads packages, and builds and uploads
-Docker images:
-
-https://ci.curoverse.com/job/common-workflow-language/
-
-Current build status: [![Build Status](https://ci.curoverse.com/buildStatus/icon?job=common-workflow-language)](https://ci.curoverse.com/job/common-workflow-language/)
-
-## Community and Contributing
+[![Biostars CWL](https://www.biostars.org/static/biostar2.logo.png)](https://www.biostars.org/t/cwl/)
 
 If you are interested in learning more or contributing ideas or code,
 [come chat with us on Gitter](https://gitter.im/common-workflow-language/common-workflow-language),
@@ -97,8 +83,6 @@ join the [mailing list common-workflow-language on Google Groups](https://groups
 [fork the repository](https://github.com/common-workflow-language/common-workflow-language)
 and send a pull request!
 
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-
 Your CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
 with updates at http://mrc.commonwl.org.
 

cwltool/schemas/doc/add-new-draft.md

@@ -1,17 +1,18 @@
 How to make a new draft version of the CWL spec
 
-1. Duplicate prior draft-n directory to draft-n+1 in a branch
+1. Duplicate prior directory to "vNewVersion-dev1" or "vExistingVersion-devN+1'
+   in a branch
 2. Update references to the new draft name.
 3. Pull in the latest metaschema, where `schema_salad_repo` is the remote
    repository for the schema salad tool.
 
      git fetch --all
-     git subtree add -P draft-4/salad schema_salad_repo/master
+     git subtree add -P v1.1.0-dev1/salad schema_salad_repo/master
 
 4. In the reference implementation (cwltool): make a new branch, and update the
    subtree checkout of the spec:
-   
-     git subtree merge -P cwltool/schemas/ cwl_repo/draft-4
+  
+     git subtree merge --squash -P cwltool/schemas/ cwl_repo/v1.1.0-dev1
 
    Where `cwl_repo` is the remote repository for the CWL specifications.
 4. In the reference implementation, teach it about the new draft version:

cwltool/schemas/run_test.sh

@@ -13,9 +13,11 @@ EOF
 
 DRAFT=v1.0
 TEST_N=""
+JUNIT_XML=""
 RUNNER=cwl-runner
 PLATFORM=$(uname -s)
 COVERAGE="python"
+EXTRA=""
 
 while [[ -n "$1" ]]
 do
@@ -35,6 +37,9 @@ do
         --only-tools)
             ONLY_TOOLS=--only-tools
             ;;
+        --junit-xml=*)
+            JUNIT_XML=$arg
+            ;;
         *=*)
             eval $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
             ;;
@@ -64,9 +69,9 @@ runtest() {
 
     runs=$((runs+1))
     (cd $DRAFT
-     ${COVERAGE} -m cwltool.cwltest --tool "$1" \
+     cwltest --tool "$1" \
 	     --test=conformance_test_${DRAFT}.yaml ${TEST_N} \
-	     ${TEST_L} ${ONLY_TOOLS} --basedir ${DRAFT}
+	     ${TEST_L} ${ONLY_TOOLS} ${JUNIT_XML} --basedir ${DRAFT} -- ${EXTRA}
     )
     checkexit
 }

cwltool/schemas/site/cwlsite-job.json

@@ -0,0 +1,20 @@
+#!/usr/bin/env cwl-runner
+brand: CWL
+brandimg: {class: File, path: CWL-Logo-Header.png}
+cwl:tool: cwlsite.cwl
+render:
+- brandimg: '<img src=''CWL-Logo-Header.png'' style=''height: 61px; margin-top: -20px;
+    margin-left: -20px''></img>'
+  brandlink: index.html
+  primtype: null
+  redirect: []
+  renderlist: []
+  source: {class: File, path: ../README.md}
+  target: index.html
+- {$import: cwlsite-draft3-job.json}
+- {$import: cwlsite-v1.0.0-job.json}
+- {$import: cwlsite-v1.1.0-dev1-job.yaml}
+schemas:
+- {$import: cwlsite-draft3-schemas.json}
+- {$import: cwlsite-v1.0.0-schemas.json}
+- {$import: cwlsite-v1.1.0-dev1-schemas.yaml}

cwltool/schemas/site/cwlsite-job.yaml

@@ -14,7 +14,7 @@
             "source": {
                 "class": "File",
                 "path": "../v1.0/CommandLineTool-standalone.yml",
-                "secondaryFiles": {"$import": "draft4-deps.json"}
+                "secondaryFiles": {"$import": "v1.0.0-deps.json"}
             },
             "target": "v1.0/CommandLineTool.html",
             "renderlist": [
@@ -30,7 +30,7 @@
             "source": {
                 "class": "File",
                 "path": "../v1.0/CommonWorkflowLanguage.yml",
-                "secondaryFiles": {"$import": "draft4-deps.json"}
+                "secondaryFiles": {"$import": "v1.0.0-deps.json"}
             },
             "target": "v1.0/Workflow.html",
             "renderlist": [
@@ -50,7 +50,7 @@
             "primtype": "#CWLType"
         },
         {
-            "source": {$import: draft4-metaschema.json},
+            "source": {$import: v1.0.0-metaschema.json},
             "target": "v1.0/SchemaSalad.html",
             "renderlist": [
                 "https://w3id.org/cwl/salad#Semantic_Annotations_for_Linked_Avro_Data",

cwltool/schemas/site/cwlsite-draft4-job.json renamed to cwltool/schemas/site/cwlsite-v1.0.0-job.json

@@ -2,12 +2,12 @@
         "schema_in": {
             "class": "File",
             "path": "../v1.0/CommonWorkflowLanguage.yml",
-            "secondaryFiles": {"$import": "draft4-deps.json"}
+            "secondaryFiles": {"$import": "v1.0.0-deps.json"}
         },
         "context_target": "v1.0/cwl-context.json",
         "rdfs_target": "v1.0/cwl.ttl"
     }, {
-        "schema_in": {$import: draft4-metaschema.json},
+        "schema_in": {$import: v1.0.0-metaschema.json},
         "context_target": "v1.0/salad-context.json",
         "rdfs_target": "v1.0/salad.ttl"
     }