Merge branch 'main' into patch-1

Author: mr-c

.travis.yml

@@ -36,6 +36,7 @@ jobs:
       script: ${TRAVIS_BUILD_DIR}/travis.bash
     - python: "3.7"
       script: RELEASE_SKIP=head ${TRAVIS_BUILD_DIR}/release-test.sh
+      name: 'release test'
     - python: "3.8"
       name: "CWL v1.2 conformance tests"
       env:
@@ -47,6 +48,6 @@ jobs:
 script: tox
 branches:
   only:
-    - master
+    - main
 notifications:
   email: false

CWLProv.rst

@@ -230,7 +230,7 @@ These set which vocabularies ("namespaces") are used by the CWLProv statements::
     prefix wf <arcp://uuid,0e6cb79e-fe70-4807-888c-3a61b9bf232a/workflow/packed.cwl#>
     prefix foaf <http://xmlns.com/foaf/0.1/>
 
-Note that the `arcp <https://tools.ietf.org/id/draft-soilandreyes-arcp-03.html>`__  base URI will correspond to the UUID of each master workflow run.
+Note that the `arcp <https://tools.ietf.org/id/draft-soilandreyes-arcp-03.html>`__  base URI will correspond to the UUID of each main workflow run.
 
 Account who launched cwltool
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^

MANIFEST.in

@@ -33,6 +33,10 @@ include cwltool/schemas/v1.2.0-dev3/*.yml
 include cwltool/schemas/v1.2.0-dev3/*.md
 include cwltool/schemas/v1.2.0-dev3/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.2.0-dev3/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.2.0-dev4/*.yml
+include cwltool/schemas/v1.2.0-dev4/*.md
+include cwltool/schemas/v1.2.0-dev4/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.2.0-dev4/salad/schema_salad/metaschema/*.md
 include cwltool/cwlNodeEngine.js
 include cwltool/cwlNodeEngineJSConsole.js
 include cwltool/cwlNodeEngineWithContext.js

Makefile

@@ -89,7 +89,7 @@ pydocstyle_report.txt: $(PYSOURCES)
 	pydocstyle setup.py $^ > $@ 2>&1 || true
 
 diff_pydocstyle_report: pydocstyle_report.txt
-	diff-quality --violations=pycodestyle --fail-under=100 $^
+	diff-quality --compare-branch=main --violations=pycodestyle --fail-under=100 $^
 
 ## format      : check/fix all code indentation and formatting (runs black)
 format:

README.rst

@@ -12,10 +12,10 @@ CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Cov
 .. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/master.svg?label=Linux%20builds
+.. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/main.svg?label=Linux%20builds
    :target: https://travis-ci.org/common-workflow-language/cwltool
 
-.. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/master.svg?label=Windows%20builds
+.. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/main.svg?label=Windows%20builds
    :target: https://ci.appveyor.com/project/mr-c/cwltool
 
 .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/common-workflow-language/cwltool.svg
@@ -24,49 +24,49 @@ CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Cov
 .. |Downloads| image:: https://pepy.tech/badge/cwltool/month
    :target: https://pepy.tech/project/cwltool
 
-.. |CommandLineTool Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/command_line_tool.json?icon=commonwl
+.. |CommandLineTool Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/command_line_tool.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |DockerRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/docker.json?icon=commonwl
+.. |DockerRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/docker.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |EnvVarRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/env_var.json?icon=commonwl
+.. |EnvVarRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/env_var.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |ExpressionTool Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/expression_tool.json?icon=commonwl
+.. |ExpressionTool Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/expression_tool.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |InitialWorkDirRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/initial_work_dir.json?icon=commonwl
+.. |InitialWorkDirRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/initial_work_dir.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |InlineJavascriptRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/inline_javascript.json?icon=commonwl
+.. |InlineJavascriptRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/inline_javascript.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |MultipleInputRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/multiple_input.json?icon=commonwl
+.. |MultipleInputRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/multiple_input.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |Core Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/required.json?icon=commonwl
+.. |Core Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/required.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |ResourceRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/resource.json?icon=commonwl
+.. |ResourceRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/resource.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |ScatterRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/scatter.json?icon=commonwl
+.. |ScatterRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/scatter.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |SchemaDefRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/schema_def.json?icon=commonwl
+.. |SchemaDefRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/schema_def.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |ShellCommandequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/shell_command.json?icon=commonwl
+.. |ShellCommandequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/shell_command.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |StepInputRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/step_input.json?icon=commonwl
+.. |StepInputRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/step_input.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |SubWorkflowRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/subworkflow.json?icon=commonwl
+.. |SubWorkflowRequirement Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/subworkflow.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
-.. |Workflow Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/workflow.json?icon=commonwl
+.. |Workflow Support| image:: https://badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/main/cwltool/cwl_v1.0/cwltool_latest/workflow.json?icon=commonwl
    :target: https://ci.commonwl.org/job/cwltool-conformance/
 
 
@@ -177,21 +177,21 @@ with the ``--user-space-docker-cmd`` option.
 One such "user space" friendly docker replacement is ``udocker`` https://github.com/indigo-dc/udocker and another
 is ``dx-docker`` https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images
 
-udocker installation: https://github.com/indigo-dc/udocker/blob/master/doc/installation_manual.md#22-install-from-indigo-datacloud-repositories
+udocker installation: https://github.com/indigo-dc/udocker/blob/main/doc/installation_manual.md#22-install-from-indigo-datacloud-repositories
 
 dx-docker installation: start with the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
 
 Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
 
 .. code:: bash
 
-  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
+  cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json
 
 or
 
 .. code:: bash
 
-  cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
+  cwltool --user-space-docker-cmd=dx-docker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json
 
 ``cwltool`` can use `Singularity <http://singularity.lbl.gov/>`_ version 2.6.1
 or later as a Docker container runtime.
@@ -205,7 +205,7 @@ those involving Docker container ENTRYPOINTs.
 
 .. code:: bash
 
-  cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/cat-job.json
+  cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/cat-job.json
 
 Running a tool or workflow from remote or local locations
 ---------------------------------------------------------
@@ -652,7 +652,7 @@ To run the basic tests after installing `cwltool` execute the following:
   pip install -rtest-requirements.txt
   py.test --ignore cwltool/schemas/ --pyarg cwltool
 
-To run various tests in all supported Python environments we use `tox <https://github.com/common-workflow-language/cwltool/tree/master/tox.ini>`_. To run the test suite in all supported Python environments
+To run various tests in all supported Python environments we use `tox <https://github.com/common-workflow-language/cwltool/tree/main/tox.ini>`_. To run the test suite in all supported Python environments
 first downloading the complete code repository (see the ``git clone`` instructions above) and then run
 the following in the terminal:
 ``pip install tox; tox``
@@ -670,7 +670,7 @@ The GitHub repository for the CWL specifications contains a script that tests a
 implementation against a wide array of valid CWL files using the `cwltest <https://github.com/common-workflow-language/cwltest>`_
 program
 
-Instructions for running these tests can be found in the Common Workflow Language Specification repository at https://github.com/common-workflow-language/common-workflow-language/blob/master/CONFORMANCE_TESTS.md
+Instructions for running these tests can be found in the Common Workflow Language Specification repository at https://github.com/common-workflow-language/common-workflow-language/blob/main/CONFORMANCE_TESTS.md
 
 Import as a module
 ------------------

appveyor.yml

@@ -50,4 +50,4 @@ on_finish:
 
 branches:
   only:
-    - master
+    - main

cwltool/builder.py

@@ -173,6 +173,7 @@ def __init__(
         outdir: str,
         tmpdir: str,
         stagedir: str,
+        cwlVersion: str,
     ) -> None:
         """Initialize this Builder."""
         self.job = job
@@ -204,6 +205,8 @@ def __init__(
         self.tmpdir = tmpdir
         self.stagedir = stagedir
 
+        self.cwlVersion = cwlVersion
+
         self.pathmapper = None  # type: Optional[PathMapper]
         self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.find_default_container = None  # type: Optional[Callable[[], str]]
@@ -630,4 +633,5 @@ def do_eval(
             js_console=self.js_console,
             force_docker_pull=self.force_docker_pull,
             strip_whitespace=strip_whitespace,
+            cwlVersion=self.cwlVersion,
         )

cwltool/command_line_tool.py

@@ -368,7 +368,10 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
             if runtimeContext.find_default_container is not None:
                 default_container = runtimeContext.find_default_container(self)
                 if default_container is not None:
-                    dockerReq = {"class": "DockerRequirement", "dockerPull": default_container}
+                    dockerReq = {
+                        "class": "DockerRequirement",
+                        "dockerPull": default_container,
+                    }
                     if mpiRequired:
                         self.hints.insert(0, dockerReq)
                         dockerRequired = False
@@ -389,29 +392,33 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
 
         if dockerReq is not None and runtimeContext.use_container:
             if mpiReq is not None:
-                _logger.warning(
-                    "MPIRequirement with containers is a beta feature"
-                )
+                _logger.warning("MPIRequirement with containers is a beta feature")
             if runtimeContext.singularity:
                 return SingularityCommandLineJob
             elif runtimeContext.user_space_docker_cmd:
                 return UDockerCommandLineJob
             if mpiReq is not None:
                 if mpiRequired:
                     if dockerRequired:
-                        raise UnsupportedRequirement("No support for Docker and MPIRequirement both being required")
+                        raise UnsupportedRequirement(
+                            "No support for Docker and MPIRequirement both being required"
+                        )
                     else:
                         _logger.warning(
                             "MPI has been required while Docker is hinted, discarding Docker hint(s)"
                         )
-                        self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"]
+                        self.hints = [
+                            h for h in self.hints if h["class"] != "DockerRequirement"
+                        ]
                         return CommandLineJob
                 else:
                     if dockerRequired:
                         _logger.warning(
                             "Docker has been required while MPI is hinted, discarding MPI hint(s)"
                         )
-                        self.hints = [h for h in self.hints if h["class"] != MPIRequirementName]
+                        self.hints = [
+                            h for h in self.hints if h["class"] != MPIRequirementName
+                        ]
                     else:
                         raise UnsupportedRequirement(
                             "Both Docker and MPI have been hinted - don't know what to do"
@@ -873,12 +880,16 @@ def register_reader(f: CWLObjectType) -> None:
         if mpi is not None:
             np = cast(  # From the schema for MPIRequirement.processes
                 Union[int, str],
-                mpi.get('processes', runtimeContext.mpi_config.default_nproc)
+                mpi.get("processes", runtimeContext.mpi_config.default_nproc),
             )
             if isinstance(np, str):
                 tmp = builder.do_eval(np)
                 if not isinstance(tmp, int):
-                    raise TypeError("{} needs 'processes' to evaluate to an int, got {}".format(MPIRequirementName, type(np)))
+                    raise TypeError(
+                        "{} needs 'processes' to evaluate to an int, got {}".format(
+                            MPIRequirementName, type(np)
+                        )
+                    )
                 np = tmp
             j.mpi_procs = np
         yield j