Merge branch 'main' into pa/cwlutil-validator

Author: tetron

.github/workflows/ci-tests.yml

@@ -184,3 +184,43 @@ jobs:
       - uses: actions/checkout@v3
       - name: build & test cwltool_module container
         run: ./build-cwltool-docker.sh
+
+  macos:
+    name: CI test on macos-latest
+    runs-on: macos-latest
+    env:
+      TOXENV: py310-unit
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            tox.ini
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install tox tox-gh-actions
+      # # docker for mac install is not currently stable
+      # - name: 'SETUP MacOS: load Homebrew cache'
+      #   uses: actions/cache@v3
+      #   if: runner.os == 'macOS'
+      #   with:
+      #      path: |
+      #        ~/Library/Caches/Homebrew/downloads/*--Docker.dmg
+      #      key: brew-actions-setup-docker-1.0.11
+      #      restore-keys: brew-actions-setup-docker-
+      # - name: setup docker on macos (default stable version)
+      #   uses: docker-practice/actions-setup-docker@master
+      - name: Test with tox
+        run: tox
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.step == 'unit' }}
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true

CONTRIBUTING.md

@@ -31,6 +31,7 @@ For the more traditional workflow:
 - To begin using the virtual environment, it needs to be activated: `source venv/bin/activate`
 - To check if you have the virtual environment set up: `which python`
   and it should point to python executable in your virtualenv
+- Install the latest versions of `pip` and `wheel` with `pip install --upgrade pip wheel`
 - Install cwltool in development mode, along with development tools:
   `make install-dep dev`
 - Check the `cwltool` version which might be different from the version installed in

Makefile

@@ -157,11 +157,11 @@ diff-cover: coverage.xml
 diff-cover.html: coverage.xml
 	diff-cover --compare-branch=main $^ --html-report $@
 
-## test                   : run the ${MODULE} test suite
+## test                   : run the cwltool test suite
 test: check-python3 $(PYSOURCES)
 	python -m pytest -rs ${PYTEST_EXTRA}
 
-## testcov                : run the ${MODULE} test suite and collect coverage
+## testcov                : run the cwltool test suite and collect coverage
 testcov: check-python3 $(PYSOURCES)
 	python -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
 

build-cwltool-docker.sh

@@ -7,4 +7,4 @@ docker run -t -v /var/run/docker.sock:/var/run/docker.sock \
 	-v /tmp:/tmp \
 	-v "$PWD":/tmp/cwltool \
 	quay.io/commonwl/cwltool_module /bin/sh -c \
-	"apk add gcc bash && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset)' --ignore-glob '*test_udocker.py' -n auto -v -rs --pyargs cwltool"
+	"apk add gcc bash && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool"

cwltool/argparser.py

@@ -5,7 +5,6 @@
 import urllib
 from typing import (
     Any,
-    AnyStr,
     Callable,
     Dict,
     List,
@@ -18,8 +17,6 @@
     cast,
 )
 
-from schema_salad.ref_resolver import file_uri
-
 from .loghandler import _logger
 from .process import Process, shortname
 from .resolver import ga4gh_tool_registries

cwltool/command_line_tool.py

@@ -15,7 +15,6 @@
 from functools import cmp_to_key, partial
 from typing import (
     Any,
-    Callable,
     Dict,
     Generator,
     List,

cwltool/cuda.py

@@ -1,6 +1,6 @@
 import subprocess  # nosec
 import xml.dom.minidom  # nosec
-from typing import Tuple, cast
+from typing import Tuple
 
 from .loghandler import _logger
 from .utils import CWLObjectType

cwltool/docker.py

@@ -16,7 +16,6 @@
 
 from .builder import Builder
 from .context import RuntimeContext
-from .cuda import cuda_check
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob

cwltool/provenance.py

@@ -298,6 +298,7 @@ def __init__(
         self.cwltool_version = "cwltool %s" % versionstring().split()[-1]
         ##
         self.relativised_input_object = {}  # type: CWLObjectType
+        self.has_manifest = False
 
         self._initialize()
         _logger.debug("[provenance] Temporary research object: %s", self.folder)
@@ -355,6 +356,8 @@ def open_log_file_for_activity(
     def _finalize(self) -> None:
         self._write_ro_manifest()
         self._write_bag_info()
+        if not self.has_manifest:
+            (Path(self.folder) / "manifest-sha1.txt").touch()
 
     def user_provenance(self, document: ProvDocument) -> None:
         """Add the user provenance."""
@@ -853,6 +856,7 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None:
         if os.path.commonprefix(["data/", rel_path]) == "data/":
             # payload file, go to manifest
             manifest = "manifest"
+            self.has_manifest = True
         else:
             # metadata file, go to tag manifest
             manifest = "tagmanifest"

cwltool/provenance_profile.py

@@ -20,15 +20,7 @@
 )
 
 from prov.identifier import Identifier, QualifiedName
-from prov.model import (
-    PROV,
-    PROV_LABEL,
-    PROV_TYPE,
-    PROV_VALUE,
-    ProvDocument,
-    ProvEntity,
-    ProvRecord,
-)
+from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity
 from schema_salad.sourceline import SourceLine
 from typing_extensions import TYPE_CHECKING
 

setup.py

@@ -124,7 +124,7 @@
         "cwl-utils @ https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip",
     ],
     extras_require={
-        "deps": ["galaxy-tool-util >= 21.1.0"],
+        "deps": ["galaxy-tool-util >= 22.1.2, <23"],
     },
     python_requires=">=3.6, <4",
     setup_requires=PYTEST_RUNNER,

test-requirements.txt

@@ -7,4 +7,4 @@ pytest-cov
 arcp >= 0.2.0
 rdflib-jsonld>=0.4.0, <= 0.6.1;python_version<='3.6'
 -rrequirements.txt
-galaxy-tool-util >= 21.1.0
+galaxy-tool-util >= 22.1.2, < 23

tests/checker_wf/wc.cwl

@@ -1,6 +1,9 @@
 #!/usr/bin/env cwl-runner
 cwlVersion: v1.1
 class: CommandLineTool
+hints:
+  DockerRequirement:
+   dockerPull: docker.io/debian:stable-slim
 inputs:
   intxt:
     type: File

tests/subgraph/count-lines17-wf.cwl.json

@@ -113,6 +113,11 @@
                       "-l"
                     ],
                     "class": "CommandLineTool",
+                    "hints": {
+                      "DockerRequirement": {
+                        "dockerPull": "docker.io/debian:stable-slim"
+                      }
+                    },
                     "id": "count-lines17-wf.cwl@step_step1@run@step_stepX@run@step_stepY@run",
                     "inputs": [
                       {

tests/subgraph/wc-tool.cwl

@@ -3,6 +3,10 @@
 class: CommandLineTool
 cwlVersion: v1.0
 
+hints:
+  DockerRequirement:
+   dockerPull: docker.io/debian:stable-slim
+
 inputs:
   file1: File
 

tests/test_cuda.py

@@ -11,10 +11,10 @@
 from cwltool.job import CommandLineJob
 from cwltool.load_tool import load_tool
 from cwltool.main import main
-from cwltool.pathmapper import MapperEnt, PathMapper
-from cwltool.process import use_custom_schema, use_standard_schema
+from cwltool.pathmapper import PathMapper
+from cwltool.process import use_custom_schema
 from cwltool.stdfsaccess import StdFsAccess
-from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION
+from cwltool.update import INTERNAL_VERSION
 from cwltool.utils import CWLObjectType
 
 from .util import get_data, needs_docker, needs_singularity_3_or_newer

tests/test_dependencies.py

@@ -7,6 +7,10 @@
 
 import pytest
 
+from cwltool.context import LoadingContext
+from cwltool.load_tool import load_tool
+from cwltool.software_requirements import get_container_from_software_requirements
+
 from .util import get_data, get_main_output, get_tool_env, needs_docker
 
 deps = None  # type: Optional[ModuleType]
@@ -28,6 +32,17 @@ def test_biocontainers(tmp_path: Path) -> None:
     assert error_code == 0
 
 
+@needs_docker
+@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
+def test_biocontainers_resolution(tmp_path: Path) -> None:
+    """Confirm expected container name for --beta-use-biocontainers."""
+    tool = load_tool(get_data("tests/seqtk_seq.cwl"), LoadingContext())
+    assert (
+        get_container_from_software_requirements(True, tool)
+        == "quay.io/biocontainers/seqtk:r93--0"
+    )
+
+
 @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
 def test_bioconda(tmp_path: Path) -> None:
     wflow = get_data("tests/seqtk_seq.cwl")

tests/test_examples.py

@@ -910,6 +910,7 @@ def test_static_checker() -> None:
         factory.make(get_data("tests/checker_wf/broken-wf3.cwl"))
 
 
+@needs_docker
 def test_circular_dependency_checker() -> None:
     # check that the circular dependency checker raises exception when there is
     # circular dependency in the workflow.
@@ -1119,8 +1120,9 @@ def test_cid_file_dir(tmp_path: Path, factor: str) -> None:
         stderr = re.sub(r"\s\s+", " ", stderr)
         assert "completed success" in stderr
         assert error_code == 0
-        cidfiles_count = sum(1 for _ in tmp_path.glob("**/*"))
-        assert cidfiles_count == 2
+        cidfiles = list(tmp_path.glob("**/*.cid"))
+        cidfiles_count = len(cidfiles)
+        assert cidfiles_count == 2, f"Should be 2 cidfiles, but got {cidfiles}"
 
 
 @needs_docker
@@ -1449,6 +1451,7 @@ def test_no_compute_chcksum(tmp_path: Path, factor: str) -> None:
     assert "checksum" not in stdout
 
 
+@needs_docker
 @pytest.mark.parametrize("factor", test_factors)
 def test_bad_userspace_runtime(factor: str) -> None:
     test_file = "tests/wf/wc-tool.cwl"

tests/test_iwdr.py

@@ -131,7 +131,9 @@ def test_iwdr_permutations(tmp_path_factory: Any) -> None:
     )
     assert err_code == 0
     log = json.loads(stdout)["log"]
-    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log
+    with open(log["path"]) as log_h:
+        log_text = log_h.read()
+    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log_text
 
 
 def test_iwdr_permutations_readonly(tmp_path_factory: Any) -> None:
@@ -240,7 +242,9 @@ def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None:
     )
     assert err_code == 0
     log = json.loads(stdout)["log"]
-    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log
+    with open(log["path"]) as log_h:
+        log_text = log_h.read()
+    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log_text
 
 
 @needs_singularity
@@ -298,7 +302,9 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None:
     )
     assert err_code == 0
     log = json.loads(stdout)["log"]
-    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log
+    with open(log["path"]) as log_h:
+        log_text = log_h.read()
+    assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log_text
 
 
 @needs_singularity

tests/test_misc_cli.py

@@ -2,7 +2,7 @@
 
 from cwltool.utils import versionstring
 
-from .util import get_data, get_main_output
+from .util import get_data, get_main_output, needs_docker
 
 
 def test_version() -> None:
@@ -44,6 +44,7 @@ def test_basic_pack() -> None:
     assert "$graph" in stdout
 
 
+@needs_docker
 def test_basic_print_subgraph() -> None:
     """Basic test of --print-subgraph. See test_subgraph.py for detailed testing."""
     return_code, stdout, stderr = get_main_output(

tests/test_parallel.py

@@ -5,9 +5,10 @@
 from cwltool.executors import MultithreadedJobExecutor
 from cwltool.factory import Factory
 
-from .util import get_data
+from .util import get_data, needs_docker
 
 
+@needs_docker
 def test_sequential_workflow(tmp_path: Path) -> None:
     test_file = "tests/wf/count-lines1-wf.cwl"
     executor = MultithreadedJobExecutor()
@@ -20,6 +21,7 @@ def test_sequential_workflow(tmp_path: Path) -> None:
     assert echo(file1=file_contents) == {"count_output": 16}
 
 
+@needs_docker
 def test_scattered_workflow() -> None:
     test_file = "tests/wf/scatter-wf4.cwl"
     job_file = "tests/wf/scatter-job2.json"

tests/test_provenance.py

@@ -192,6 +192,15 @@ def test_directory_workflow(tmp_path: Path) -> None:
         assert p.is_file(), f"Could not find {l} as {p}"
 
 
+@needs_docker
+def test_no_data_files(tmp_path: Path) -> None:
+    folder = cwltool(
+        tmp_path,
+        get_data("tests/wf/conditional_step_no_inputs.cwl"),
+    )
+    check_bagit(folder)
+
+
 def check_output_object(base_path: Path) -> None:
     output_obj = base_path / "workflow" / "primary-output.json"
     compare_checksum = "sha1$b9214658cc453331b62c2282b772a5c063dbd284"

tests/test_stdout_stderr_log_dir.py

@@ -1,10 +1,7 @@
 import json
-import os
 from pathlib import Path
 
-from cwltool.main import main
-
-from .util import get_data, get_main_output, needs_docker
+from .util import get_data, get_main_output
 
 
 def test_log_dir_echo_output(tmp_path: Path) -> None:

tests/test_subgraph.py

@@ -240,6 +240,7 @@ def test_single_process_packed_subwf_step(tmp_path: Path) -> None:
     )
 
 
+@needs_docker
 def test_single_process_subwf_subwf_inline_step() -> None:
     """Test --single-process on an inline sub-sub-workflow step."""
     err_code, stdout, stderr = get_main_output(
@@ -350,6 +351,7 @@ def test_print_targets_embedded_reqsinherit() -> None:
     assert err_code == 0
 
 
+@needs_docker
 def test_print_targets_embedded_sub_subwfs() -> None:
     """Confirm that --print-targets works with inline sub-sub-workflows."""
     err_code, stdout, stderr = get_main_output(

tests/test_toolargparse.py

@@ -1,7 +1,7 @@
 import argparse
 from io import StringIO
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Callable, List
 
 import pytest