Fix string interpolation and backslash escaping to match 1.2 spec. (#1309)

* Fix string interpolation and backslash escaping to match 1.2 spec.
* Adjust escaping behavior based on version.
* Add tests for each escaping behavior of interpolate.
* refresh cwl v1.2.0-dev4 schema

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: tetron

MANIFEST.in

@@ -33,6 +33,10 @@ include cwltool/schemas/v1.2.0-dev3/*.yml
 include cwltool/schemas/v1.2.0-dev3/*.md
 include cwltool/schemas/v1.2.0-dev3/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.2.0-dev3/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.2.0-dev4/*.yml
+include cwltool/schemas/v1.2.0-dev4/*.md
+include cwltool/schemas/v1.2.0-dev4/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.2.0-dev4/salad/schema_salad/metaschema/*.md
 include cwltool/cwlNodeEngine.js
 include cwltool/cwlNodeEngineJSConsole.js
 include cwltool/cwlNodeEngineWithContext.js

cwltool/builder.py

@@ -173,6 +173,7 @@ def __init__(
         outdir: str,
         tmpdir: str,
         stagedir: str,
+        cwlVersion: str,
     ) -> None:
         """Initialize this Builder."""
         self.job = job
@@ -204,6 +205,8 @@ def __init__(
         self.tmpdir = tmpdir
         self.stagedir = stagedir
 
+        self.cwlVersion = cwlVersion
+
         self.pathmapper = None  # type: Optional[PathMapper]
         self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.find_default_container = None  # type: Optional[Callable[[], str]]
@@ -630,4 +633,5 @@ def do_eval(
             js_console=self.js_console,
             force_docker_pull=self.force_docker_pull,
             strip_whitespace=strip_whitespace,
+            cwlVersion=self.cwlVersion,
         )

cwltool/command_line_tool.py

@@ -368,7 +368,10 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
             if runtimeContext.find_default_container is not None:
                 default_container = runtimeContext.find_default_container(self)
                 if default_container is not None:
-                    dockerReq = {"class": "DockerRequirement", "dockerPull": default_container}
+                    dockerReq = {
+                        "class": "DockerRequirement",
+                        "dockerPull": default_container,
+                    }
                     if mpiRequired:
                         self.hints.insert(0, dockerReq)
                         dockerRequired = False
@@ -389,29 +392,33 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
 
         if dockerReq is not None and runtimeContext.use_container:
             if mpiReq is not None:
-                _logger.warning(
-                    "MPIRequirement with containers is a beta feature"
-                )
+                _logger.warning("MPIRequirement with containers is a beta feature")
             if runtimeContext.singularity:
                 return SingularityCommandLineJob
             elif runtimeContext.user_space_docker_cmd:
                 return UDockerCommandLineJob
             if mpiReq is not None:
                 if mpiRequired:
                     if dockerRequired:
-                        raise UnsupportedRequirement("No support for Docker and MPIRequirement both being required")
+                        raise UnsupportedRequirement(
+                            "No support for Docker and MPIRequirement both being required"
+                        )
                     else:
                         _logger.warning(
                             "MPI has been required while Docker is hinted, discarding Docker hint(s)"
                         )
-                        self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"]
+                        self.hints = [
+                            h for h in self.hints if h["class"] != "DockerRequirement"
+                        ]
                         return CommandLineJob
                 else:
                     if dockerRequired:
                         _logger.warning(
                             "Docker has been required while MPI is hinted, discarding MPI hint(s)"
                         )
-                        self.hints = [h for h in self.hints if h["class"] != MPIRequirementName]
+                        self.hints = [
+                            h for h in self.hints if h["class"] != MPIRequirementName
+                        ]
                     else:
                         raise UnsupportedRequirement(
                             "Both Docker and MPI have been hinted - don't know what to do"
@@ -873,12 +880,16 @@ def register_reader(f: CWLObjectType) -> None:
         if mpi is not None:
             np = cast(  # From the schema for MPIRequirement.processes
                 Union[int, str],
-                mpi.get('processes', runtimeContext.mpi_config.default_nproc)
+                mpi.get("processes", runtimeContext.mpi_config.default_nproc),
             )
             if isinstance(np, str):
                 tmp = builder.do_eval(np)
                 if not isinstance(tmp, int):
-                    raise TypeError("{} needs 'processes' to evaluate to an int, got {}".format(MPIRequirementName, type(np)))
+                    raise TypeError(
+                        "{} needs 'processes' to evaluate to an int, got {}".format(
+                            MPIRequirementName, type(np)
+                        )
+                    )
                 np = tmp
             j.mpi_procs = np
         yield j

cwltool/expression.py

@@ -15,6 +15,7 @@
     cast,
 )
 
+import json
 from schema_salad.utils import json_dumps
 
 from .errors import WorkflowException
@@ -90,6 +91,7 @@ def scanner(scan: str) -> Optional[Tuple[int, int]]:
                 stack.append(BRACE)
             else:
                 stack.pop()
+                i -= 1
         elif state == PAREN:
             if c == "(":
                 stack.append(PAREN)
@@ -124,14 +126,13 @@ def scanner(scan: str) -> Optional[Tuple[int, int]]:
                 stack.append(BACKSLASH)
         i += 1
 
-    if len(stack) > 1:
+    if len(stack) > 1 and not (len(stack) == 2 and stack[1] in (BACKSLASH, DOLLAR)):
         raise SubstitutionError(
-            "Substitution error, unfinished block starting at position {}: {}".format(
-                start, scan[start:]
+            "Substitution error, unfinished block starting at position {}: '{}' stack was {}".format(
+                start, scan[start:], stack
             )
         )
-    else:
-        return None
+    return None
 
 
 def next_seg(
@@ -279,6 +280,7 @@ def interpolate(
     debug: bool = False,
     js_console: bool = False,
     strip_whitespace: bool = True,
+    escaping_behavior: int = 2,
 ) -> Optional[CWLOutputType]:
     if strip_whitespace:
         scan = scan.strip()
@@ -302,11 +304,30 @@ def interpolate(
                 return e
             leaf = json_dumps(e, sort_keys=True)
             if leaf[0] == '"':
-                leaf = leaf[1:-1]
+                leaf = json.loads(leaf)
             parts.append(leaf)
         elif scan[w[0]] == "\\":
-            e = scan[w[1] - 1]
-            parts.append(e)
+            if escaping_behavior == 1:
+                # Old behavior.  Just skip the next character.
+                e = scan[w[1] - 1]
+                parts.append(e)
+            elif escaping_behavior == 2:
+                # Backslash quoting requires a three character lookahead.
+                e = scan[w[0] : w[1] + 1]
+                if e in ("\\$(", "\\${"):
+                    # Suppress start of a parameter reference, drop the
+                    # backslash.
+                    parts.append(e[1:])
+                    w = (w[0], w[1] + 1)
+                elif e[1] == "\\":
+                    # Double backslash, becomes a single backslash
+                    parts.append("\\")
+                else:
+                    # Some other text, add it as-is (including the
+                    # backslash) and resume scanning.
+                    parts.append(e[:2])
+            else:
+                raise Exception("Unknown escaping behavior %s" % escaping_behavior)
 
         scan = scan[w[1] :]
         w = scanner(scan)
@@ -331,6 +352,7 @@ def do_eval(
     debug: bool = False,
     js_console: bool = False,
     strip_whitespace: bool = True,
+    cwlVersion: str = "",
 ) -> Optional[CWLOutputType]:
 
     runtime = cast(MutableMapping[str, Union[int, str, None]], copy.deepcopy(resources))
@@ -362,6 +384,17 @@ def do_eval(
                 debug=debug,
                 js_console=js_console,
                 strip_whitespace=strip_whitespace,
+                escaping_behavior=1
+                if cwlVersion
+                in (
+                    "v1.0",
+                    "v1.1.0-dev1",
+                    "v1.1",
+                    "v1.2.0-dev1",
+                    "v1.2.0-dev2",
+                    "v1.2.0-dev3",
+                )
+                else 2,
             )
 
         except Exception as e:

cwltool/process.py

@@ -864,6 +864,9 @@ def inc(d):  # type: (List[int]) -> None
                     runtime_context.stagedir or tempfile.mkdtemp()
                 )
 
+        cwl_version = cast(str, self.metadata.get(
+            "http://commonwl.org/cwltool#original_cwlVersion", None
+        ))
         builder = Builder(
             job,
             files,
@@ -886,6 +889,7 @@ def inc(d):  # type: (List[int]) -> None
             outdir,
             tmpdir,
             stagedir,
+            cwl_version,
         )
 
         bindings.extend(

cwltool/schemas/v1.2.0-dev4/CITATION

@@ -0,0 +1,28 @@
+To cite the Common Workflow Language standard in a publication, please use:
+
+Amstutz, Peter; Crusoe, Michael R; Tijanić, Nebojša; Chapman, Brad;
+Chilton, John; Heuer, Michael; Kartashov, Andrey; Kern, John; Leehr, Dan;
+Ménager, Hervé; Nedeljkovich, Maya; Scales, Matt; Soiland-Reyes, Stian;
+Stojanovic, Luka (2016): Common Workflow Language, v1.0. Specification,
+Common Workflow Language working group. https://w3id.org/cwl/v1.0/
+https://doi.org/10.6084/m9.figshare.3115156.v2
+
+@data{cwl,
+  doi = {10.6084/m9.figshare.3115156.v2},
+  url = {https://doi.org/10.6084/m9.figshare.3115156.v2},
+  author = {Peter Amstutz; Michael R. Crusoe; Nebojša Tijanić; Brad Chapman;
+John Chilton; Michael Heuer; Andrey Kartashov; John Kern; Dan Leehr;
+Hervé Ménager; Maya Nedeljkovich; Matt Scales; Stian Soiland-Reyes;
+Luka Stojanovic
+ },
+  publisher = {Figshare},
+  institution = {Common Workflow Language working group},
+  title = {Common Workflow Language, v1.0},
+  year = {2016}
+}
+
+# Are you editing this file?
+# Synchronize any changes made with
+# README.md
+# and
+# https://github.com/common-workflow-language/user_guide/blob/gh-pages/CITATION

cwltool/schemas/v1.2.0-dev4/CODE_OF_CONDUCT.md

@@ -0,0 +1,108 @@
+CWL Code of Conduct
+===================
+
+The CWL Project is dedicated to providing a harassment-free experience for
+everyone. We do not tolerate harassment of participants in any form.
+
+This code of conduct applies to all CWL Project spaces both online and off: the
+Google Group, the Gitter chat room, the Google Hangouts chats, and any other
+CWL spaces. Anyone who violates this code of conduct may be sanctioned or
+expelled from these spaces at the discretion of the CWL Leadership Team.
+
+Some CWL Project spaces may have additional rules in place, which will be
+made clearly available to participants. Participants are responsible for
+knowing and abiding by these rules.
+
+Harassment includes, but is not limited to:
+
+ - Offensive comments related to gender, gender identity and expression, sexual
+orientation, disability, mental illness, neuro(a)typicality, physical
+appearance, body size, age, race, or religion.
+ - Unwelcome comments regarding a person’s lifestyle choices and practices,
+including those related to food, health, parenting, drugs, and employment.
+ - Deliberate misgendering or use of [dead](https://www.quora.com/What-is-deadnaming/answer/Nancy-C-Walker)
+or rejected names.
+ - Gratuitous or off-topic sexual images or behaviour  in spaces where they’re not
+appropriate.
+ - Physical contact and simulated physical contact (eg, textual descriptions like
+“\*hug\*” or “\*backrub\*”) without consent or after a request to stop.
+ - Threats of violence.
+ - Incitement of violence towards any individual, including encouraging a person
+to commit suicide or to engage in self-harm.
+ - Deliberate intimidation.
+ - Stalking or following.
+ - Harassing photography or recording, including logging online activity for
+harassment purposes.
+ - Sustained disruption of discussion.
+ - Unwelcome sexual attention.
+ - Pattern of inappropriate social contact, such as requesting/assuming
+inappropriate levels of intimacy with others
+ - Continued one-on-one communication after requests to cease.
+ - Deliberate “outing” of any aspect of a person’s identity without their consent
+except as necessary to protect vulnerable people from intentional abuse.
+ - Publication of non-harassing private communication.
+
+The CWL Project prioritizes marginalized people’s safety over privileged
+people’s comfort. The CWL Leadeship Team will not act on complaints regarding:
+
+ - ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’
+ - Reasonable communication of boundaries, such as “leave me alone,” “go away,” or
+“I’m not discussing this with you.”
+ - Communicating in a [tone](http://geekfeminism.wikia.com/wiki/Tone_argument)
+you don’t find congenial
+
+Reporting
+---------
+
+If you are being harassed by a member of the CWL Project, notice that someone
+else is being harassed, or have any other concerns, please contact the CWL
+Leadership Team at [email protected]. If person who is harassing
+you is on the team, they will recuse themselves from handling your incident. We
+will respond as promptly as we can.
+
+This code of conduct applies to CWL Project spaces, but if you are being
+harassed by a member of CWL Project outside our spaces, we still want to
+know about it. We will take all good-faith reports of harassment by CWL Project
+members, especially the CWL Leadership Team, seriously. This includes harassment
+outside our spaces and harassment that took place at any point in time. The
+abuse team reserves the right to exclude people from the CWL Project based on
+their past behavior, including behavior outside CWL Project spaces and
+behavior towards people who are not in the CWL Project.
+
+In order to protect volunteers from abuse and burnout, we reserve the right to
+reject any report we believe to have been made in bad faith. Reports intended
+to silence legitimate criticism may be deleted without response.
+
+We will respect confidentiality requests for the purpose of protecting victims
+of abuse. At our discretion, we may publicly name a person about whom we’ve
+received harassment complaints, or privately warn third parties about them, if
+we believe that doing so will increase the safety of CWL Project members or
+the general public. We will not name harassment victims without their
+affirmative consent.
+
+Consequences
+------------
+
+Participants asked to stop any harassing behavior are expected to comply
+immediately.
+
+If a participant engages in harassing behavior, the CWL Leadership Team may
+take any action they deem appropriate, up to and including expulsion from all
+CWL Project spaces and identification of the participant as a harasser to other
+CWL Project members or the general public.
+
+This anti-harassment policy is based on the [example policy from the Geek
+Feminism wiki](http://geekfeminism.wikia.com/wiki/Community_anti-harassment/Policy),
+created by the Geek Feminism community.
+
+CWL Leadership Team
+-------------------
+
+As a stop gap measure until a more formal governance structure is adopted, the
+following individuals make up the leadership of the CWL Project: Peter Amstutz,
+John Chilton, Michael R. Crusoe, and Nebojša Tijanić.
+
+To report an issue with anyone on the team you can escalate to Ward Vandewege
+ (Curoverse) [email protected], Anton Nekrutenko (Galaxy)
+anton AT bx DOT psu DOT edu, C. Titus Brown (UC Davis) [email protected], or
+Brandi Davis-Dusenbery (Seven Bridges Genomics) [email protected].