CWL document resolver via GA4GH tool API (#201)

* Support Dockstore lookups for tools.
* Update README.rst to add reference to discovery mechanism
* Add --enable/disable-ga4gh-tool-registry and --add-ga4gh-tool-registry

Author: denis-yuen

README.rst

@@ -78,6 +78,26 @@ is referenced and that document isn't found in the current directory then the
 following locations will be searched:
 http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem
 
+
+Use with GA4GH Tool Registry API
+--------------------------------
+
+Cwltool can launch tools directly from `GA4GH Tool Registry API`_ endpoints.
+
+By default, cwltool searches https://dockstore.org/ .  Use --add-tool-registry to add other registries to the search path.
+
+For example ::
+
+  cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats:master test.json
+
+and (defaults to latest when a version is not specified) ::
+
+  cwltool --non-strict quay.io/collaboratory/dockstore-tool-bamstats test.json
+
+For this example, grab the test.json (and input file) from https://github.com/CancerCollaboratory/dockstore-tool-bamstats
+
+.. _`GA4GH Tool Registry API`: https://github.com/ga4gh/tool-registry-schemas
+
 Import as a module
 ------------------
 
@@ -101,6 +121,8 @@ The easiest way to use cwltool to run a tool or workflow from Python is to use a
 Cwltool control flow
 --------------------
 
+Technical outline of how cwltool works internally, for maintainers.
+
 #. Use CWL `load_tool()` to load document.
 
    #. Fetches the document from file or URL

cwltool/main.py

@@ -27,7 +27,7 @@
 from .pack import pack
 from .process import (shortname, Process, relocateOutputs, cleanIntermediate,
                       scandeps, normalizeFilesDirs, use_custom_schema, use_standard_schema)
-from .resolver import tool_resolver
+from .resolver import tool_resolver, ga4gh_tool_registries
 from .stdfsaccess import StdFsAccess
 
 _logger = logging.getLogger("cwltool")
@@ -166,7 +166,16 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         help="Will be passed to `docker run` as the '--net' "
                              "parameter. Implies '--enable-net'.")
 
-    parser.add_argument("--on-error", type=str,
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",
+                        dest="enable_ga4gh_tool_registry", default=True)
+    exgroup.add_argument("--disable-ga4gh-tool-registry", action="store_false", help="Disable resolution using GA4GH tool registry API",
+                        dest="enable_ga4gh_tool_registry", default=True)
+
+    parser.add_argument("--add-ga4gh-tool-registry", action="append", help="Add a GA4GH tool registry endpoint to use for resolution, default %s" % ga4gh_tool_registries,
+                        dest="ga4gh_tool_registries", default=[])
+
+    parser.add_argument("--on-error",
                         help="Desired workflow behavior when a step fails.  One of 'stop' or 'continue'. "
                              "Default is 'stop'.", default="stop", choices=("stop", "continue"))
 
@@ -642,6 +651,11 @@ def main(argsl=None,  # type: List[str]
         if args.relax_path_checks:
             draft2tool.ACCEPTLIST_RE = draft2tool.ACCEPTLIST_EN_RELAXED_RE
 
+        if args.ga4gh_tool_registries:
+            ga4gh_tool_registries[:] = args.ga4gh_tool_registries
+        if not args.enable_ga4gh_tool_registry:
+            del ga4gh_tool_registries[:]
+
         if args.enable_ext:
             res = pkg_resources.resource_stream(__name__, 'extensions.yml')
             use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read())

cwltool/resolver.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import urllib
 
 from schema_salad.ref_resolver import file_uri
 
@@ -25,8 +26,25 @@ def resolve_local(document_loader, uri):
 
 
 def tool_resolver(document_loader, uri):
-    for r in [resolve_local]:
+    for r in [resolve_local, resolve_ga4gh_tool]:
         ret = r(document_loader, uri)
         if ret is not None:
             return ret
     return file_uri(os.path.abspath(uri), split_frag=True)
+
+
+ga4gh_tool_registries = ["https://dockstore.org:8443"]
+
+def resolve_ga4gh_tool(document_loader, uri):
+    path, version = uri.partition(":")[::2]
+    if not version:
+        version = "latest"
+    for reg in ga4gh_tool_registries:
+        ds = "{0}/api/ga4gh/v1/tools/{1}/versions/{2}/plain-CWL/descriptor".format(reg, urllib.quote(path, ""), urllib.quote(version, ""))
+        try:
+            resp = document_loader.session.head(ds)
+            resp.raise_for_status()
+            return ds
+        except Exception:
+            pass
+    return None