Merge commit 'b0701278de2bab6c830df1e07ea481dd3335b18e' into inline-expressions

Author: Peter Amstutz

cwltool/schemas/README.md

@@ -78,6 +78,9 @@ and send a pull request!
 
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+Your CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
+with weekly updates at http://mrc.commonwl.org.
+
 ### Code of Conduct
 
 The CWL Project is dedicated to providing a harassment-free experience for

cwltool/schemas/draft-3/conformance_test_draft-3.yaml

@@ -290,3 +290,153 @@
   tool: draft-3/schemadef-wf.cwl
   doc: |
     Test SchemaDefRequirement definition used in workflow parameter
+
+- job: draft-3/empty.json
+  output: {
+    "t1": {
+        "bar": {
+            "b az": 2,
+            "b\"az": null,
+            "b'az": true,
+            "baz": "zab1",
+            "buz": [
+                "a",
+                "b",
+                "c"
+            ]
+        }
+    },
+    "t10": true,
+    "t11": true,
+    "t12": null,
+    "t13": "-zab1",
+    "t14": "-zab1",
+    "t15": "-zab1",
+    "t16": "-zab1",
+    "t17": "zab1 zab1",
+    "t18": "zab1 zab1",
+    "t19": "zab1 zab1",
+    "t2": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t20": "zab1 zab1",
+    "t21": "2 2",
+    "t22": "true true",
+    "t23": "true true",
+    "t24": "null null",
+    "t25": "b",
+    "t26": "b b",
+    "t3": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t4": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t5": "zab1",
+    "t6": "zab1",
+    "t7": "zab1",
+    "t8": "zab1",
+    "t9": 2
+    }
+  tool: draft-3/params.cwl
+  doc: |
+    Test parameter evaluation, no support for JS expressions
+
+- job: draft-3/empty.json
+  output: {
+    "t1": {
+        "bar": {
+            "b az": 2,
+            "b\"az": null,
+            "b'az": true,
+            "baz": "zab1",
+            "buz": [
+                "a",
+                "b",
+                "c"
+            ]
+        }
+    },
+    "t10": true,
+    "t11": true,
+    "t12": null,
+    "t13": "-zab1",
+    "t14": "-zab1",
+    "t15": "-zab1",
+    "t16": "-zab1",
+    "t17": "zab1 zab1",
+    "t18": "zab1 zab1",
+    "t19": "zab1 zab1",
+    "t2": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t20": "zab1 zab1",
+    "t21": "2 2",
+    "t22": "true true",
+    "t23": "true true",
+    "t24": "null null",
+    "t25": "b",
+    "t26": "b b",
+    "t3": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t4": {
+        "b az": 2,
+        "b\"az": null,
+        "b'az": true,
+        "baz": "zab1",
+        "buz": [
+            "a",
+            "b",
+            "c"
+        ]
+    },
+    "t5": "zab1",
+    "t6": "zab1",
+    "t7": "zab1",
+    "t8": "zab1",
+    "t9": 2
+    }
+  tool: draft-3/params2.cwl
+  doc: |
+    Test parameter evaluation, with support for JS expressions

cwltool/schemas/draft-3/cwl-avro.yml

@@ -323,6 +323,68 @@
     Process requirements are the primary mechanism for specifying extensions to
     the CWL core specification.
 
+    ## Parameter references
+
+    Parameter references are denoted by the syntax `$(...)` and may be used in
+    any field permitting the pseudo-type `Expression`, as specified by this
+    document.  Conforming implementations must support parameter
+    references.  Parameter references use the following subset of
+    [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) syntax:
+
+    symbol::    {Unicode alphanumeric}+
+    singleq::   '[' ''' ( {character} - ''' | '\' ''' )* ''' ']'
+    doubleq::   '[' '"' ( {character} - '"' | '\' '"' )* '"' ']'
+    index::     '[' {decimal digit}+ ']'
+    segment::   '.' {symbol} | {singleq} | {doubleq} | {index}
+    parameter:: '$' '(' {symbol} {segment}* ')'
+
+    Use the following algorithm to resolve a parameter reference:
+
+      1. Match the leading symbol as key
+      2. Look up the key in the parameter context (described below) to get the current value.
+         It is an error if the key is not found in the parameter context.
+      3. If there are no subsequent segments, terminate and return current value
+      4. Else, match the next segment
+      5. Extract the symbol, string, or index from the segment as key
+      6. Look up the key in current value and assign as new current value.  If
+         the key is a symbol or string, the current value must be an object.
+         If the key is an index, the current value must be an array or string.
+         It is an error if the key does not match the required type, or the key is not found or out
+         of range.
+      7. Repeat steps 3-6
+
+    The root namspace is the parameter context.  The following parameters must
+    be provided:
+
+      * `inputs`: The input object to the current Process.
+      * `self`: A context-specific value.  The contextual values for 'self' are
+        documented for specific fields elsewhere in this specification.  If
+        a contextual value of 'self' is not documented for a field, it
+        must be 'null'.
+      * `runtime`: An object containing configuration specific to the Process
+        type.  Values include `tmpdir`, `outdir`, and values associated with
+        [ResourceRequirement](#resourcerequirement).
+        Other fields under `runtime` are listed elsewhere in this document.
+
+    If the value of a field has no leading or trailing non-whitespace
+    characters around a parameter reference, the effective value of the field
+    becomes the value of the referenced parameter, preserving the return type.
+
+    If the value of a field has non-whitespace leading or trailing characters
+    around an parameter reference, it is subject to string interpolation.  The
+    effective value of the field is a string containing the leading characters;
+    followed by the string value of the parameter reference; followed by the
+    trailing characters.  The string value of the parameter reference is its
+    textual JSON representation with the following rules:
+
+      * Leading and trailing quotes are stripped from strings
+      * Objects entries are sorted by key
+
+    Multiple parameter references may appear in a single field.  This case is
+    must be treated as a string interpolation.  After interpolating the first
+    parameter reference, interpolation must be recursively applied to the
+    trailing characters to yield the final string value.
+
     ## Expressions
 
     An expression is a fragment of [Javascript/ECMAScript
@@ -334,62 +396,48 @@
     processes in that they are intended to modify the behavior of the workflow
     itself rather than perform the primary work of the workflow.
 
-    Expressions may be used in any field that allows expressions, as specified
-    by this document.
+    To declare the use of expressions, the document must include the process
+    requirement `InlineJavascriptRequirement`.  Expressions may be used in any
+    field permitting the pseudo-type `Expression`, as specified by this
+    document.
 
     Expressions are denoted by the syntax `$(...)` or `${...}`.  A code
-    fragment wrapped in the `$(...)` syntax must be evaluated as a [ECMAScript
-    expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11).  A
+    fragment wrapped in the `$(...)` syntax must be evaluated as a
+    [ECMAScript expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11).  A
     code fragment wrapped in the `${...}` syntax must be evaluated as a
-    [EMACScript function
-    body](http://www.ecma-international.org/ecma-262/5.1/#sec-13) for an
-    anonymous, zero-argument function.  Expressions must return a valid JSON
+    [EMACScript function body](http://www.ecma-international.org/ecma-262/5.1/#sec-13)
+    for an anonymous, zero-argument function.  Expressions must return a valid JSON
     data type: one of null, string, number, boolean, array, object.
     Implementations must permit any syntatically valid Javascript and account
-    for nesting of parenthesis or braces and the strings that may contain
+    for nesting of parenthesis or braces and that strings that may contain
     parenthesis or braces when scanning for expressions.
 
-    If a field has no leading or trailing non-whitespace characters around an
-    expression, the effective value of the field is the value returned by
-    evaluating the expression, preserving the return type.
-
-    If a field has non-whitespace leading or trailing characters around an
-    expression, it is subject to string interpolation.  The effective value of
-    the field is a string containing the leading characters followed by the
-    value returned by evaluating the expression, followed by the trailing
-    characters.  The value returned by the expression is coerced to a string
-    subject to the following rules:
-
-      * strings are included directly with no transformation
-      * numbers are converted to decimal representation string
-      * null is the string `(null)`
-      * boolean "true" or "false" yield the strings "true" or "false"
-      * arrays are serialized to JSON
-      * objects are serialized to JSON
-
-    Multiple expressions may appear in a single field.  This case is must be treated
-    as a string interpolation.
-
-    Before executing the expression, the runtime must initialize the global
-    object `inputs` containing a copy of the validated contents of the job
-    object.  The runtime must also initialize a global object `runtime`
-    containing the values for `tmpdir`, `outdir`, and values associated with
-    [ResourceRequirement](#resourcerequirement).
-
     The runtime must include any code defined in the ["expressionLib" field of
     InlineJavascriptRequirement](#inlinejavascriptrequirement) prior to
     executing the actual expression.
 
+    Before executing the expression, the runtime must initialize as global
+    variables the fields of the parameter context described above.
+
+    The effective value of the field after expression evaluation follows the
+    same rules as parameter references discussed above.  Multiple expressions
+    may appear in a single field.
+
     Expressions must be evaluated in an isolated context (a "sandbox") which
     permits no side effects to leak outside the context.  Expressions also must
-    be evaluated in [Javascript strict
-    mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2).
+    be evaluated in [Javascript strict mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2).
+
+    The order in which expressions are evaluated is undefined except where
+    otherwise noted in this document.
 
-    The order in which expressions are evaluated is undefined.
+    An implementation may choose to implement parameter references by
+    evalutating as a Javascript expression.  The results of evaluating
+    parameter references must be identical whether implemented by Javascript
+    evaluation or some other means.
 
     Implementations may apply other limits, such as process isolation, timeouts,
     and operating system containers/jails to minimize the security risks associated
-    with running untrusted code embedded in a tool description document.
+    with running untrusted code embedded in a CWL document.
 
     ## Workflow graph
 
@@ -719,15 +767,6 @@
         object.
 
 
-- name: StandardExpressionEngine
-  type: enum
-  doc: "Symbols for default CWL expression engines that may be used in the `engine` field."
-  docParent: "#Expression"
-  symbols:
-    - "cwl:JsonPointer"
-    - "cwl:JavascriptEngine"
-
-
 - type: enum
   name: Expression
   docAfter: "#ExpressionTool"
@@ -736,6 +775,7 @@
   symbols:
     - cwl:ExpressionPlaceholder
 
+
 - name: Binding
   type: record
   docParent: "#Parameter"

cwltool/schemas/draft-3/draft-3/bwa-mem-job.json

@@ -23,7 +23,7 @@
     ],
     "minimum_seed_length": 3,
     "allocatedResources": {
-        "cpu": 4,
+        "cores": 4,
         "mem": 5000
     }
 }

cwltool/schemas/draft-3/draft-3/cat1-tool.cwl

@@ -5,7 +5,7 @@ description: "Print the contents of a file to stdout using 'cat' running in a do
 hints:
   - class: DockerRequirement
     dockerPull: debian:wheezy
-inputs: [
+inputs:
   - id: file1
     type: File
     inputBinding: {position: 1}

cwltool/schemas/draft-3/draft-3/count-lines2-wf.cwl

@@ -1,6 +1,8 @@
 #!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: "cwl:draft-3.dev2"
+requirements:
+  - class: InlineJavascriptRequirement
 
 inputs:
     - { id: file1, type: File }
@@ -31,4 +33,9 @@ steps:
       - { id: parseInt_output }
     run:
       class: ExpressionTool
-      expression: ${return {'parseInt_output': parseInt($job.parseInt_file1.contents)};}
+      inputs:
+        - { id: parseInt_file1, type: File, inputBinding: { loadContents: true } }
+      outputs:
+        - { id: parseInt_output, type: int }
+      expression: >
+        ${return {'parseInt_output': parseInt(inputs.parseInt_file1.contents)};}

cwltool/schemas/draft-3/draft-3/params.cwl

@@ -0,0 +1,16 @@
+class: CommandLineTool
+cwlVersion: cwl:draft-3.dev2
+inputs:
+  - id: bar
+    type: Any
+    default: {
+          "baz": "zab1",
+          "b az": 2,
+          "b'az": true,
+          'b"az': null,
+          "buz": ['a', 'b', 'c']
+        }
+
+outputs: {"@import": params_inc.yml}
+
+baseCommand: "true"