normalize script names: no .py suffix, always cwl- (#154)

Improve README and docs

Author: mr-c

README.rst

@@ -11,8 +11,7 @@
 cwl-utils
 ---------
 
-A collection of scripts to demonstrate the use of the new Python classes
-for loading and parsing `CWL
+Python Utilities and Autogenerated Classes for loading and parsing `CWL
 v1.0 <https://github.com/common-workflow-language/cwl-utils/blob/main/cwl_utils/parser/v1_0.py>`__,
 `CWL
 v1.1 <https://github.com/common-workflow-language/cwl-utils/blob/main/cwl_utils/parser/v1_1.py>`__,
@@ -41,7 +40,7 @@ Usage
 Pull the all referenced software container images
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``docker_extract.py`` is useful to cache or pre-pull all software
+``cwl-docker-extract`` is useful to cache or pre-pull all software
 container images referenced in a CWL CommandLineTool or CWL Workflow
 (including all referenced CommandLineTools and sub-Workflows and so on).
 
@@ -50,15 +49,71 @@ the software container images in Docker format.
 
 .. code:: bash
 
-   docker_extract.py DIRECTORY path_to_my_workflow.cwl
+   cwl-docker-extract DIRECTORY path_to_my_workflow.cwl
 
 Or you can use the Singularity software container engine to download and
 save the software container images and convert them to the Singularity
 format at the same time.
 
 .. code:: bash
 
-   docker_extract.py --singularity DIRECTORY path_to_my_workflow.cwl
+   cwl-docker-extract --singularity DIRECTORY path_to_my_workflow.cwl
+
+
+Print all referenced software packages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cwl-cite-extract`` prints all software packages found (recursively) in the
+specified CWL document.
+
+Currently the package name and any listed specs and version field are printed
+for all ``SoftwareRequirement`` s found.
+
+.. code:: bash
+
+   cwl-cite-extract path_to_my_workflow.cwl
+
+
+Replace CWL Expressions with concrete steps
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cwl-expression-refactor`` refactors CWL documents so that any CWL Expression
+evaluations are separate steps (either CWL ExpressionTools or CWL CommandLineTools.)
+This allows execution by CWL engines that do not want to support inline expression
+evaluation outside of concrete steps, or do not want to directly support CWL's
+optional ``InlineJavascriptRequirement`` at all.
+
+
+.. code:: bash
+
+   cwl-expression-refactor directory/path/to/save/outputs path_to_my_workflow.cwl [more_workflows.cwl]
+
+Split a packed CWL document
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cwl-graph-split`` splits a packed CWL document file into multiple files.
+
+Packed CWL documents use the $graph construct to contain multiple CWL Process
+objects (Workflow, CommandLineTool, ExpressionTool, Operation). Typically
+packed CWL documents contain a CWL Workflow under the name "main" and the
+workflow steps (including any sub-workflows).
+
+.. code:: bash
+
+   cwl-graph-split --outdir optional/directory/path/to/save/outputs path_to_my_workflow.cwl
+
+Normalize a CWL document
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cwl-normalizer`` normalizes one or more CWL document so that for each document,
+a JSON format CWL document is produces with it and all of its dependencies packed
+together, upgrading to CWL v1.2, as needed. Can optionally refactor CWL
+Expressions into separate steps in the manner of cwl-expression-refactor.
+
+.. code:: bash
+
+   cwl-normalizer directory/path/to/save/outputs path_to_my_workflow.cwl [more_workflows.cwl]
+
 
 Using the CWL Parsers
 ~~~~~~~~~~~~~~~~~~~~~

cwl_utils/cite_extract.py

@@ -1,20 +1,43 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-3.0-only
+import argparse
 import sys
-from typing import Iterator, Union, cast
+from typing import Iterator, List, Union, cast
 
 import cwl_utils.parser.cwl_v1_0 as cwl
 
 ProcessType = Union[cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool]
 
 
-def main() -> int:
-    """Load the first argument and extract the software requirements."""
-    top = cwl.load_document(sys.argv[1])
+def arg_parser() -> argparse.ArgumentParser:
+    """Argument parser."""
+    parser = argparse.ArgumentParser(
+        description="Print information about software used in a CWL document (Workflow or CommandLineTool). "
+        "For CWL Workflows, all steps will also be searched (recursively)."
+    )
+    parser.add_argument(
+        "input", help="Input CWL document (CWL Workflow or CWL CommandLineTool)"
+    )
+    return parser
+
+
+def parse_args(args: List[str]) -> argparse.Namespace:
+    """Parse the command line arguments."""
+    return arg_parser().parse_args(args)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Extract the software requirements."""
+    top = cwl.load_document(args.input)
     traverse(top)
     return 0
 
 
+def main() -> None:
+    """Console entry point."""
+    sys.exit(run(parse_args(sys.argv[1:])))
+
+
 def extract_software_packages(process: ProcessType) -> None:
     """Print software packages found in the given process."""
     for req in extract_software_reqs(process):
@@ -73,4 +96,4 @@ def traverse_workflow(workflow: cwl.Workflow) -> None:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()

cwl_utils/docker_extract.py

@@ -4,7 +4,7 @@
 import os
 import sys
 from pathlib import Path
-from typing import Iterator, Union, cast
+from typing import Iterator, List, Union, cast
 
 import cwl_utils.parser.cwl_v1_0 as cwl
 from cwl_utils.image_puller import (
@@ -19,10 +19,13 @@
 def arg_parser() -> argparse.ArgumentParser:
     """Argument parser."""
     parser = argparse.ArgumentParser(
-        description="Tool to save docker images from a cwl workflow."
+        description="Save container images specified in a CWL document (Workflow or CommandLineTool). "
+        "For CWL Workflows, all steps will also be searched (recursively)."
     )
     parser.add_argument("dir", help="Directory in which to save images")
-    parser.add_argument("input", help="Input CWL workflow")
+    parser.add_argument(
+        "input", help="Input CWL document (CWL Workflow or CWL CommandLineTool)"
+    )
     parser.add_argument(
         "-s",
         "--singularity",
@@ -32,12 +35,12 @@ def arg_parser() -> argparse.ArgumentParser:
     return parser
 
 
-def parse_args() -> argparse.Namespace:
+def parse_args(args: List[str]) -> argparse.Namespace:
     """Parse the command line arguments."""
-    return arg_parser().parse_args()
+    return arg_parser().parse_args(args)
 
 
-def main(args: argparse.Namespace) -> None:
+def run(args: argparse.Namespace) -> int:
     """Extract the docker reqs and download them using Singularity or docker."""
     os.makedirs(args.dir, exist_ok=True)
 
@@ -52,6 +55,7 @@ def main(args: argparse.Namespace) -> None:
         else:
             image_puller = DockerImagePuller(req.dockerPull, args.dir)
         image_puller.save_docker_image()
+    return 0
 
 
 def extract_docker_requirements(
@@ -102,6 +106,10 @@ def traverse_workflow(workflow: cwl.Workflow) -> Iterator[cwl.DockerRequirement]
         yield from traverse(get_process_from_step(step))
 
 
+def main() -> None:
+    """Command line entry point."""
+    sys.exit(run(parse_args(sys.argv[1:])))
+
+
 if __name__ == "__main__":
-    main(parse_args())
-    sys.exit(0)
+    main()

cwl_utils/cwl_expression_refactor.py renamed to cwl_utils/expression_refactor.py

@@ -96,15 +96,19 @@ def parse_args(args: List[str]) -> argparse.Namespace:
     return arg_parser().parse_args(args)
 
 
-def main(args: Optional[List[str]] = None) -> int:
+def main() -> None:
+    """Console entry point."""
+    sys.exit(run(sys.argv[1:]))
+
+
+def run(args: List[str]) -> int:
     """Collect the arguments and run."""
-    if not args:
-        args = sys.argv[1:]
-    return run(parse_args(args))
+    return refactor(parse_args(args))
 
 
-def run(args: argparse.Namespace) -> int:
+def refactor(args: argparse.Namespace) -> int:
     """Primary processing loop."""
+
     return_code = 0
     yaml = YAML(typ="rt")
     yaml.preserve_quotes = True  # type: ignore[assignment]
@@ -175,4 +179,3 @@ def run(args: argparse.Namespace) -> int:
 
 if __name__ == "__main__":
     main()
-    sys.exit(0)

cwl_utils/graph_split.py

@@ -11,7 +11,8 @@
 import argparse
 import json
 import os
-from typing import IO, Any, MutableMapping, Set, Union, cast
+import sys
+from typing import IO, Any, List, MutableMapping, Set, Union, cast
 
 from cwlformat.formatter import stringify_dict
 from ruamel.yaml.dumper import RoundTripDumper
@@ -22,7 +23,7 @@
 
 def arg_parser() -> argparse.ArgumentParser:
     """Build the argument parser."""
-    parser = argparse.ArgumentParser(description="Split the packed CWL.")
+    parser = argparse.ArgumentParser(description="Split a packed CWL document.")
     parser.add_argument("cwlfile")
     parser.add_argument(
         "-m",
@@ -57,20 +58,26 @@ def arg_parser() -> argparse.ArgumentParser:
 
 
 def main() -> None:
+    """Console entry point."""
+    sys.exit(run(sys.argv[1:]))
+
+
+def run(args: List[str]) -> int:
     """Split the packed CWL at the path of the first argument."""
-    options = arg_parser().parse_args()
+    options = arg_parser().parse_args(args)
 
     with open(options.cwlfile) as source_handle:
-        run(
+        graph_split(
             source_handle,
             options.outdir,
             options.output_format,
             options.mainfile,
             options.pretty,
         )
+    return 0
 
 
-def run(
+def graph_split(
     sourceIO: IO[str], output_dir: str, output_format: str, mainfile: str, pretty: bool
 ) -> None:
     """Loop over the provided packed CWL document and split it up."""

cwl_utils/cwl_normalizer.py renamed to cwl_utils/normalizer.py

@@ -7,7 +7,7 @@
 import sys
 import tempfile
 from pathlib import Path
-from typing import List, MutableSequence, Optional, Set
+from typing import List, MutableSequence, Set
 
 from cwlupgrader import main as cwlupgrader
 from ruamel import yaml
@@ -29,7 +29,7 @@
 def arg_parser() -> argparse.ArgumentParser:
     """Build the argument parser."""
     parser = argparse.ArgumentParser(
-        description="Tool to normalize CWL documents. Will upgrade to CWL v1.2, "
+        description="Normalizes CWL documents. Will upgrade to CWL v1.2, "
         "and pack the result. Can optionally refactor out CWL expressions."
     )
     parser.add_argument(
@@ -66,11 +66,9 @@ def parse_args(args: List[str]) -> argparse.Namespace:
     return arg_parser().parse_args(args)
 
 
-def main(args: Optional[List[str]] = None) -> int:
-    """Collect the arguments and run."""
-    if not args:
-        args = sys.argv[1:]
-    return run(parse_args(args))
+def main() -> None:
+    """Console entry point."""
+    sys.exit(run(parse_args(sys.argv[1:])))
 
 
 def run(args: argparse.Namespace) -> int:
@@ -128,4 +126,3 @@ def run(args: argparse.Namespace) -> int:
 
 if __name__ == "__main__":
     main()
-    sys.exit(0)

docs/index.rst

@@ -14,20 +14,23 @@ Modules
    :caption: Contents:
 
 
-Example scripts
-===============
+Included Utility Programs
+=========================
+
+.. autoprogram:: cwl_utils.cite_extract:arg_parser()
+   :prog: cwl-cite-extract
 
 .. autoprogram:: cwl_utils.docker_extract:arg_parser()
-   :prog: docker_extract.py
+   :prog: cwl-docker-extract
 
-.. autoprogram:: cwl_utils.cwl_expression_refactor:arg_parser()
-   :prog: cwl_expression_refactor.py
+.. autoprogram:: cwl_utils.expression_refactor:arg_parser()
+   :prog: cwl-expression-refactor
 
 .. autoprogram:: cwl_utils.graph_split:arg_parser()
-   :prog: graph_split.py
+   :prog: cwl-graph-split
 
-.. autoprogram:: cwl_utils.cwl_normalizer:arg_parser()
-   :prog: cwl_normalizer.py
+.. autoprogram:: cwl_utils.normalizer:arg_parser()
+   :prog: cwl-normalizer
 
 
 Indices and tables

setup.py

@@ -39,14 +39,16 @@
     .splitlines(),
     tests_require=["pytest<8"],
     test_suite="tests",
-    scripts=[
-        "cwl_utils/docker_extract.py",
-        "cwl_utils/cwl_expression_refactor.py",
-        "cwl_utils/cite_extract.py",
-        "cwl_utils/graph_split.py",
-        "cwl_utils/cwl_normalizer.py",
-    ],
     extras_require={"pretty": ["cwlformat"]},
+    entry_points={
+        "console_scripts": [
+            "cwl-cite-extract=cwl_utils.cite_extract:main",
+            "cwl-docker-extract=cwl_utils.docker_extract:main",
+            "cwl-expression-refactor=cwl_utils.expression_refactor:main",
+            "cwl-graph-split=cwl_utils.graph_split:main",
+            "cwl-normalizer=cwl_utils.normalizer:main",
+        ]
+    },
     classifiers=[
         "Environment :: Console",
         "Intended Audience :: Science/Research",

test-requirements.txt

@@ -1,3 +1,4 @@
 pytest < 8
 pytest-cov
 pytest-xdist
+cwlformat

tests/test_etools_to_clt.py

@@ -13,11 +13,11 @@
 import cwl_utils.parser.cwl_v1_0 as parser
 import cwl_utils.parser.cwl_v1_1 as parser1
 import cwl_utils.parser.cwl_v1_2 as parser2
-from cwl_utils.cwl_expression_refactor import main as expression_refactor
 from cwl_utils.cwl_v1_0_expression_refactor import traverse as traverse0
 from cwl_utils.cwl_v1_1_expression_refactor import traverse as traverse1
 from cwl_utils.cwl_v1_2_expression_refactor import traverse as traverse2
 from cwl_utils.errors import WorkflowException
+from cwl_utils.expression_refactor import run as expression_refactor
 
 HERE = Path(__file__).resolve().parent