Merge branch 'master' into fix-648

Author: mr-c

cwltool/argparser.py

@@ -0,0 +1,373 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import logging
+import os
+
+from typing import (Any, AnyStr, Dict, List, Sequence, Text, Union, cast)
+
+from schema_salad.ref_resolver import file_uri
+from .process import (Process, shortname)
+from .resolver import ga4gh_tool_registries
+from .software_requirements import (SOFTWARE_REQUIREMENTS_ENABLED)
+
+_logger = logging.getLogger("cwltool")
+
+defaultStreamHandler = logging.StreamHandler()
+_logger.addHandler(defaultStreamHandler)
+_logger.setLevel(logging.INFO)
+
+
+def arg_parser():  # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(description='Reference executor for Common Workflow Language')
+    parser.add_argument("--basedir", type=Text)
+    parser.add_argument("--outdir", type=Text, default=os.path.abspath('.'),
+                        help="Output directory, default current directory")
+
+    parser.add_argument("--no-container", action="store_false", default=True,
+                        help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool",
+                        dest="use_container")
+
+    parser.add_argument("--preserve-environment", type=Text, action="append",
+                        help="Preserve specific environment variable when running CommandLineTools.  May be provided multiple times.",
+                        metavar="ENVVAR",
+                        default=["PATH"],
+                        dest="preserve_environment")
+
+    parser.add_argument("--preserve-entire-environment", action="store_true",
+                        help="Preserve entire parent environment when running CommandLineTools.",
+                        default=False,
+                        dest="preserve_entire_environment")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--rm-container", action="store_true", default=True,
+                         help="Delete Docker container used by jobs after they exit (default)",
+                         dest="rm_container")
+
+    exgroup.add_argument("--leave-container", action="store_false",
+                         default=True, help="Do not delete Docker container used by jobs after they exit",
+                         dest="rm_container")
+
+    parser.add_argument("--tmpdir-prefix", type=Text,
+                        help="Path prefix for temporary directories",
+                        default="tmp")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--tmp-outdir-prefix", type=Text,
+                         help="Path prefix for intermediate output directories",
+                         default="tmp")
+
+    exgroup.add_argument("--cachedir", type=Text, default="",
+                         help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--rm-tmpdir", action="store_true", default=True,
+                         help="Delete intermediate temporary directories (default)",
+                         dest="rm_tmpdir")
+
+    exgroup.add_argument("--leave-tmpdir", action="store_false",
+                         default=True, help="Do not delete intermediate temporary directories",
+                         dest="rm_tmpdir")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--move-outputs", action="store_const", const="move", default="move",
+                         help="Move output files to the workflow output directory and delete intermediate output directories (default).",
+                         dest="move_outputs")
+
+    exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move",
+                         help="Leave output files in intermediate output directories.",
+                         dest="move_outputs")
+
+    exgroup.add_argument("--copy-outputs", action="store_const", const="copy", default="move",
+                         help="Copy output files to the workflow output directory, don't delete intermediate output directories.",
+                         dest="move_outputs")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--enable-pull", default=True, action="store_true",
+                         help="Try to pull Docker images", dest="enable_pull")
+
+    exgroup.add_argument("--disable-pull", default=True, action="store_false",
+                         help="Do not try to pull Docker images", dest="enable_pull")
+
+    parser.add_argument("--rdf-serializer",
+                        help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
+                        default="turtle")
+
+    parser.add_argument("--eval-timeout",
+                        help="Time to wait for a Javascript expression to evaluate before giving an error, default 20s.",
+                        type=float,
+                        default=20)
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--print-rdf", action="store_true",
+                         help="Print corresponding RDF graph for workflow and exit")
+    exgroup.add_argument("--print-dot", action="store_true",
+                         help="Print workflow visualization in graphviz format and exit")
+    exgroup.add_argument("--print-pre", action="store_true", help="Print CWL document after preprocessing.")
+    exgroup.add_argument("--print-deps", action="store_true", help="Print CWL document dependencies.")
+    exgroup.add_argument("--print-input-deps", action="store_true", help="Print input object document dependencies.")
+    exgroup.add_argument("--pack", action="store_true", help="Combine components into single document and print.")
+    exgroup.add_argument("--version", action="store_true", help="Print version and exit")
+    exgroup.add_argument("--validate", action="store_true", help="Validate CWL document only.")
+    exgroup.add_argument("--print-supported-versions", action="store_true", help="Print supported CWL specs.")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--strict", action="store_true",
+                         help="Strict validation (unrecognized or out of place fields are error)",
+                         default=True, dest="strict")
+    exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
+                         default=True, dest="strict")
+
+    parser.add_argument("--skip-schemas", action="store_true",
+            help="Skip loading of schemas", default=True, dest="skip_schemas")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--verbose", action="store_true", help="Default logging")
+    exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
+    exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
+
+    parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
+    parser.add_argument("--user-space-docker-cmd",
+                        help="(Linux/OS X only) Specify a user space docker "
+                        "command (like udocker or dx-docker) that will be "
+                        "used to call 'pull' and 'run'")
+
+    dependency_resolvers_configuration_help = argparse.SUPPRESS
+    dependencies_directory_help = argparse.SUPPRESS
+    use_biocontainers_help = argparse.SUPPRESS
+    conda_dependencies = argparse.SUPPRESS
+
+    if SOFTWARE_REQUIREMENTS_ENABLED:
+        dependency_resolvers_configuration_help = "Dependency resolver configuration file describing how to adapt 'SoftwareRequirement' packages to current system."
+        dependencies_directory_help = "Defaut root directory used by dependency resolvers configuration."
+        use_biocontainers_help = "Use biocontainers for tools without an explicitly annotated Docker container."
+        conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages."
+
+    parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=dependency_resolvers_configuration_help)
+    parser.add_argument("--beta-dependencies-directory", default=None, help=dependencies_directory_help)
+    parser.add_argument("--beta-use-biocontainers", default=None, help=use_biocontainers_help, action="store_true")
+    parser.add_argument("--beta-conda-dependencies", default=None, help=conda_dependencies, action="store_true")
+
+    parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")
+
+    parser.add_argument("--relative-deps", choices=['primary', 'cwd'],
+                        default="primary", help="When using --print-deps, print paths "
+                                                "relative to primary file or current working directory.")
+
+    parser.add_argument("--enable-dev", action="store_true",
+                        help="Enable loading and running development versions "
+                             "of CWL spec.", default=False)
+
+    parser.add_argument("--enable-ext", action="store_true",
+                        help="Enable loading and running cwltool extensions "
+                             "to CWL spec.", default=False)
+
+    parser.add_argument("--default-container",
+                        help="Specify a default docker container that will be used if the workflow fails to specify one.")
+    parser.add_argument("--no-match-user", action="store_true",
+                        help="Disable passing the current uid to 'docker run --user`")
+    parser.add_argument("--disable-net", action="store_true",
+                        help="Use docker's default networking for containers;"
+                             " the default is to enable networking.")
+    parser.add_argument("--custom-net", type=Text,
+                        help="Will be passed to `docker run` as the '--net' "
+                             "parameter. Implies '--enable-net'.")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",
+                        dest="enable_ga4gh_tool_registry", default=True)
+    exgroup.add_argument("--disable-ga4gh-tool-registry", action="store_false", help="Disable resolution using GA4GH tool registry API",
+                        dest="enable_ga4gh_tool_registry", default=True)
+
+    parser.add_argument("--add-ga4gh-tool-registry", action="append", help="Add a GA4GH tool registry endpoint to use for resolution, default %s" % ga4gh_tool_registries,
+                        dest="ga4gh_tool_registries", default=[])
+
+    parser.add_argument("--on-error",
+                        help="Desired workflow behavior when a step fails.  One of 'stop' or 'continue'. "
+                             "Default is 'stop'.", default="stop", choices=("stop", "continue"))
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--compute-checksum", action="store_true", default=True,
+                         help="Compute checksum of contents while collecting outputs",
+                         dest="compute_checksum")
+    exgroup.add_argument("--no-compute-checksum", action="store_false",
+                         help="Do not compute checksum of contents while collecting outputs",
+                         dest="compute_checksum")
+
+    parser.add_argument("--relax-path-checks", action="store_true",
+                        default=False, help="Relax requirements on path names to permit "
+                        "spaces and hash characters.", dest="relax_path_checks")
+    exgroup.add_argument("--make-template", action="store_true",
+                         help="Generate a template input object")
+
+    parser.add_argument("--force-docker-pull", action="store_true",
+                        default=False, help="Pull latest docker image even if"
+                                            " it is locally present", dest="force_docker_pull")
+    parser.add_argument("--no-read-only", action="store_true",
+                        default=False, help="Do not set root directory in the"
+                                            " container as read-only", dest="no_read_only")
+
+    parser.add_argument("--overrides", type=str,
+                        default=None, help="Read process requirement overrides from file.")
+
+    parser.add_argument("workflow", type=Text, nargs="?", default=None)
+    parser.add_argument("job_order", nargs=argparse.REMAINDER)
+
+    return parser
+
+
+def get_default_args():
+    # type: () -> Dict[str, Any]
+    """
+    Get default values of cwltool's command line options
+    """
+    ap = arg_parser()
+    args = ap.parse_args()
+    return vars(args)
+
+
+class FSAction(argparse.Action):
+    objclass = None  # type: Text
+
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        # type: (List[Text], Text, Any, **Any) -> None
+        if nargs is not None:
+            raise ValueError("nargs not allowed")
+        super(FSAction, self).__init__(option_strings, dest, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
+        setattr(namespace,
+                self.dest,  # type: ignore
+                {"class": self.objclass,
+                 "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
+
+
+class FSAppendAction(argparse.Action):
+    objclass = None  # type: Text
+
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        # type: (List[Text], Text, Any, **Any) -> None
+        if nargs is not None:
+            raise ValueError("nargs not allowed")
+        super(FSAppendAction, self).__init__(option_strings, dest, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
+        g = getattr(namespace,
+                    self.dest  # type: ignore
+                    )
+        if not g:
+            g = []
+            setattr(namespace,
+                    self.dest,  # type: ignore
+                    g)
+        g.append(
+            {"class": self.objclass,
+             "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
+
+
+class FileAction(FSAction):
+    objclass = "File"
+
+
+class DirectoryAction(FSAction):
+    objclass = "Directory"
+
+
+class FileAppendAction(FSAppendAction):
+    objclass = "File"
+
+
+class DirectoryAppendAction(FSAppendAction):
+    objclass = "Directory"
+
+
+def add_argument(toolparser, name, inptype, records, description="",
+                 default=None):
+    # type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any) -> None
+    if len(name) == 1:
+        flag = "-"
+    else:
+        flag = "--"
+
+    required = True
+    if isinstance(inptype, list):
+        if inptype[0] == "null":
+            required = False
+            if len(inptype) == 2:
+                inptype = inptype[1]
+            else:
+                _logger.debug(u"Can't make command line argument from %s", inptype)
+                return None
+
+    ahelp = description.replace("%", "%%")
+    action = None  # type: Union[argparse.Action, Text]
+    atype = None  # type: Any
+
+    if inptype == "File":
+        action = cast(argparse.Action, FileAction)
+    elif inptype == "Directory":
+        action = cast(argparse.Action, DirectoryAction)
+    elif isinstance(inptype, dict) and inptype["type"] == "array":
+        if inptype["items"] == "File":
+            action = cast(argparse.Action, FileAppendAction)
+        elif inptype["items"] == "Directory":
+            action = cast(argparse.Action, DirectoryAppendAction)
+        else:
+            action = "append"
+    elif isinstance(inptype, dict) and inptype["type"] == "enum":
+        atype = Text
+    elif isinstance(inptype, dict) and inptype["type"] == "record":
+        records.append(name)
+        for field in inptype['fields']:
+            fieldname = name + "." + shortname(field['name'])
+            fieldtype = field['type']
+            fielddescription = field.get("doc", "")
+            add_argument(
+                toolparser, fieldname, fieldtype, records,
+                fielddescription)
+        return
+    if inptype == "string":
+        atype = Text
+    elif inptype == "int":
+        atype = int
+    elif inptype == "double":
+        atype = float
+    elif inptype == "float":
+        atype = float
+    elif inptype == "boolean":
+        action = "store_true"
+
+    if default:
+        required = False
+
+    if not atype and not action:
+        _logger.debug(u"Can't make command line argument from %s", inptype)
+        return None
+
+    if inptype != "boolean":
+        typekw = {'type': atype}
+    else:
+        typekw = {}
+
+    toolparser.add_argument(  # type: ignore
+        flag + name, required=required, help=ahelp, action=action,
+        default=default, **typekw)
+
+
+def generate_parser(toolparser, tool, namemap, records):
+    # type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text]) -> argparse.ArgumentParser
+    toolparser.add_argument("job_order", nargs="?", help="Job input json file")
+    namemap["job_order"] = "job_order"
+
+    for inp in tool.tool["inputs"]:
+        name = shortname(inp["id"])
+        namemap[name.replace("-", "_")] = name
+        inptype = inp["type"]
+        description = inp.get("doc", "")
+        default = inp.get("default", None)
+        add_argument(toolparser, name, inptype, records, description, default)
+
+    return toolparser

cwltool/factory.py

@@ -4,6 +4,7 @@
 from typing import Any, Dict, Text, Tuple, Union
 
 from . import load_tool, main, workflow
+from .argparser import get_default_args
 from .process import Process
 
 
@@ -41,7 +42,13 @@ def __init__(self,
         # type: (...) -> None
         self.makeTool = makeTool
         self.executor = executor
-        self.execkwargs = execkwargs
+
+        kwargs = get_default_args()
+        kwargs.pop("job_order")
+        kwargs.pop("workflow")
+        kwargs.pop("outdir")
+        kwargs.update(execkwargs)
+        self.execkwargs = kwargs
 
     def make(self, cwl):
         """Instantiate a CWL object from a CWl document."""