Skip to content

Commit abe61c8

Browse files
tetronmr-c
andauthored
Codegen changes to support cwltool --fast-parse (#156)
* Update parsers based on latest codegen, with mypy fixes * adjust expression refactor logic for the fixed refscope * adjust cite-extract and docker-extract for getting concrete classes from `hints` Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <[email protected]> Co-authored-by: Michael R. Crusoe <[email protected]>
1 parent 53a129b commit abe61c8

File tree

10 files changed

+2827
-1270
lines changed

10 files changed

+2827
-1270
lines changed

README.rst

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,13 @@ Regenerate parsers
148148
To regenerate install the ``schema_salad`` package and run:
149149

150150
``cwl_utils/parser/cwl_v1_0.py`` was created via
151-
``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_0"``
151+
``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/codegen/v1.0/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_0" > cwl_utils/parser/cwl_v1_0.py``
152152

153153
``cwl_utils/parser/cwl_v1_1.py`` was created via
154-
``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/main/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_1"``
154+
``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/codegen/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_1" > cwl_utils/parser/cwl_v1_1.py``
155155

156156
``cwl_utils/parser/cwl_v1_2.py`` was created via
157-
``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_2"``
157+
``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_2" > cwl_utils/parser/cwl_v1_2.py``
158158

159159
Release
160160
~~~~~~~

cwl_utils/cite_extract.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,10 @@ def extract_software_reqs(
5555
yield req
5656
if process.hints:
5757
for req in process.hints:
58-
if req["class"] == "SoftwareRequirement":
58+
if isinstance(req, cwl.ProcessRequirement):
59+
if isinstance(req, cwl.SoftwareRequirement):
60+
yield req
61+
elif req["class"] == "SoftwareRequirement":
5962
yield cwl.load_field(
6063
req,
6164
cwl.SoftwareRequirementLoader,

cwl_utils/cwl_v1_2_expression_refactor.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -702,14 +702,9 @@ def process_workflow_inputs_and_outputs(
702702
target_type.name = None
703703
target = cwl.WorkflowInputParameter(id=None, type=target_type)
704704
if not isinstance(param2.outputSource, list):
705-
sources: Union[List[str], str] = "/".join(
706-
param2.outputSource.split("#")[-1].split("/")[1:]
707-
)
705+
sources: Union[List[str], str] = param2.outputSource.split("#")[-1]
708706
else:
709-
sources = [
710-
"/".join(s.split("#")[-1].split("/")[1:])
711-
for s in param2.outputSource
712-
]
707+
sources = [s.split("#")[-1] for s in param2.outputSource]
713708
source_type_items = utils.type_for_source(workflow, sources)
714709
if "null" not in source_type_items:
715710
if isinstance(source_type_items, list):

cwl_utils/docker_extract.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,10 @@ def extract_docker_reqs(process: ProcessType) -> Iterator[cwl.DockerRequirement]
7676
yield req
7777
if process.hints:
7878
for req in process.hints:
79-
if req["class"] == "DockerRequirement":
79+
if isinstance(req, cwl.ProcessRequirement):
80+
if isinstance(req, cwl.DockerRequirement):
81+
yield req
82+
elif req["class"] == "DockerRequirement":
8083
yield cwl.load_field(
8184
req,
8285
cwl.DockerRequirementLoader,

cwl_utils/expression_refactor.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
Callable,
1414
Dict,
1515
List,
16+
MutableMapping,
1617
MutableSequence,
1718
Optional,
1819
Tuple,
@@ -43,15 +44,17 @@
4344
)
4445
from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2
4546

46-
save_type = Union[Dict[str, str], List[Union[Dict[str, str], List[Any], None]], None]
47+
save_type = Optional[
48+
Union[MutableMapping[str, Any], MutableSequence[Any], int, float, bool, str]
49+
]
4750

4851

4952
class saveCWL(Protocol):
5053
"""Shortcut type for CWL v1.x parse.save()."""
5154

5255
def __call__(
5356
self,
54-
val: Optional[Union[Any, MutableSequence[Any]]],
57+
val: Any,
5558
top: bool = True,
5659
base_url: str = "",
5760
relative_uris: bool = True,

cwl_utils/parser/__init__.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
LoadingOptions = Union[
1515
cwl_v1_0.LoadingOptions, cwl_v1_1.LoadingOptions, cwl_v1_2.LoadingOptions
1616
]
17-
Savable = Union[cwl_v1_0.Savable, cwl_v1_1.Savable, cwl_v1_2.Savable]
17+
Saveable = Union[cwl_v1_0.Saveable, cwl_v1_1.Saveable, cwl_v1_2.Saveable]
1818
Workflow = Union[cwl_v1_0.Workflow, cwl_v1_1.Workflow, cwl_v1_2.Workflow]
1919
WorkflowTypes = (cwl_v1_0.Workflow, cwl_v1_1.Workflow, cwl_v1_2.Workflow)
2020
WorkflowStep = Union[
@@ -157,16 +157,16 @@ def load_document_by_yaml(
157157

158158

159159
def save(
160-
val: Optional[Union[Savable, MutableSequence[Savable]]],
160+
val: Optional[Union[Saveable, MutableSequence[Saveable]]],
161161
top: bool = True,
162162
base_url: str = "",
163163
relative_uris: bool = True,
164164
) -> Any:
165165
"""Convert a given CWL object into a built-in typed object."""
166166
if (
167-
isinstance(val, cwl_v1_0.Savable)
168-
or isinstance(val, cwl_v1_1.Savable)
169-
or isinstance(val, cwl_v1_2.Savable)
167+
isinstance(val, cwl_v1_0.Saveable)
168+
or isinstance(val, cwl_v1_1.Saveable)
169+
or isinstance(val, cwl_v1_2.Saveable)
170170
):
171171
return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
172172
if isinstance(val, MutableSequence):

0 commit comments

Comments
 (0)