switch to ruamel.yaml, use C loader if available, only load safely (#63)

Author: mr-c

cwltool/cwltest.py

@@ -7,8 +7,12 @@
 import sys
 import shutil
 import tempfile
-import yaml
-import yaml.scanner
+import ruamel.yaml as yaml
+try:
+        from ruamel.yaml import CSafeLoader as SafeLoader
+except ImportError:
+        from ruamel.yaml import SafeLoader
+
 import pipes
 import logging
 import schema_salad.ref_resolver
@@ -86,7 +90,7 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
                             t["job"]]
 
             outstr = subprocess.check_output(test_command)
-            out = yaml.load(outstr)
+            out = yaml.load(outstr, Loader=SafeLoader)
             if not isinstance(out, dict):
                 raise ValueError("Non-dict value parsed from output string.")
     except ValueError as v:
@@ -155,7 +159,7 @@ def main():  # type: () -> int
         return 1
 
     with open(args.test) as f:
-        tests = yaml.load(f)
+        tests = yaml.load(f, Loader=SafeLoader)
 
     failures = 0
     unsupported = 0

cwltool/draft2tool.py

@@ -6,7 +6,11 @@
 import os
 from .pathmapper import PathMapper, DockerPathMapper
 from .job import CommandLineJob
-import yaml
+import ruamel.yaml as yaml
+try:
+        from ruamel.yaml import CSafeLoader as SafeLoader
+except ImportError:
+        from ruamel.yaml import SafeLoader
 import glob
 import logging
 import hashlib
@@ -298,7 +302,7 @@ def collect_output_ports(self, ports, builder, outdir):
             custom_output = os.path.join(outdir, "cwl.output.json")
             if builder.fs_access.exists(custom_output):
                 with builder.fs_access.open(custom_output, "r") as f:
-                    ret = yaml.load(f)
+                    ret = yaml.load(f, Loader=SafeLoader)
                 _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
                 adjustFileObjs(ret, remove_hostfs)
                 adjustFileObjs(ret,

cwltool/expression.py

@@ -5,7 +5,6 @@
 import logging
 import os
 from .errors import WorkflowException
-import yaml
 import schema_salad.validate as validate
 import schema_salad.ref_resolver
 from . import sandboxjs

cwltool/job.py

@@ -4,7 +4,6 @@
 import tempfile
 import glob
 import json
-import yaml
 import logging
 import sys
 import requests

cwltool/main.py

@@ -13,7 +13,11 @@
 import tempfile
 import schema_salad.jsonld_context
 import schema_salad.makedoc
-import yaml
+import ruamel.yaml as yaml
+try:
+        from ruamel.yaml import CSafeLoader as SafeLoader
+except ImportError:
+        from ruamel.yaml import SafeLoader
 import urlparse
 from . import process
 from . import job
@@ -419,7 +423,7 @@ def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps
     if len(args.job_order) == 1 and args.job_order[0][0] != "-":
         job_order_file = args.job_order[0]
     elif len(args.job_order) == 1 and args.job_order[0] == "-":
-        job_order_object = yaml.load(stdin)
+        job_order_object = yaml.load(stdin, Loader=SafeLoader)
         job_order_object, _ = loader.resolve_all(job_order_object, "")
     else:
         job_order_file = None

cwltool/process.py

@@ -4,7 +4,6 @@
 import json
 import schema_salad.validate as validate
 import copy
-import yaml
 import copy
 import logging
 import pprint

setup.py

@@ -32,7 +32,7 @@
                                 'schemas/draft-3/salad/schema_salad/metaschema/*.md']},
       install_requires=[
           'requests',
-          'PyYAML',
+          'ruamel.yaml',
           'rdflib >= 4.1.0',
           'rdflib-jsonld >= 0.3.0',
           'shellescape',