Merge branch 'master' of github.com:common-workflow-language/schema_salad

Conflicts:
	schema_salad/ref_resolver.py
	schema_salad/schema.py

Author: Peter Amstutz

.travis.yml

@@ -5,11 +5,11 @@ os:
 env:
   - TOX_ENV=py27-lint
   - TOX_ENV=py27-unit
-    #  - TOX_ENV=py34-lint
-    #  - TOX_ENV=py33-lint
-    #  - TOX_ENV=py27-lint
-    #  - TOX_ENV=py34-unit
-    #  - TOX_ENV=py33-unit
+  - TOX_ENV=py34-mypy
+#  - TOX_ENV=py34-lint
+#  - TOX_ENV=py33-lint
+#  - TOX_ENV=py34-unit
+#  - TOX_ENV=py33-unit
 
 install:
   - pip install tox

Makefile

@@ -102,51 +102,50 @@ diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
 .coverage: $(PYSOURCES)
-	python-coverage run --branch --source=${MODULE} tests/test_examples.py
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --branch --source=${MODULE} setup.py test
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-jsonld-context schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-rdfs schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-avro schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-rdf schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-pre schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-index schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.main \
 		--print-metadata schema_salad/metaschema/metaschema.yml \
 		> /dev/null
-	python-coverage run --append --branch --source=${MODULE} \
+	coverage run --append --branch --source=${MODULE} \
 		-m schema_salad.makedoc schema_salad/metaschema/metaschema.yml \
 		> /dev/null
 
-
 coverage.xml: .coverage
-	python-coverage xml
+	coverage xml
 
 coverage.html: htmlcov/index.html
 
 htmlcov/index.html: .coverage
-	python-coverage html
+	coverage html
 	@echo Test coverage of the Python code is now in htmlcov/index.html
 
 coverage-report: .coverage
-	python-coverage report
+	coverage report
 
 diff-cover: coverage-gcovr.xml coverage.xml
 	diff-cover coverage-gcovr.xml coverage.xml
@@ -170,4 +169,14 @@ list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
+mypy: ${PYSOURCES}
+	MYPYPATH=typeshed/2.7 mypy --py2 schema_salad
+	#MYPYPATH=typeshed/2.7 mypy --py2  --disallow-untyped-calls schema_salad
+
+jenkins:
+	if ! test -d env ; then virtualenv env ; fi
+	. env/bin/activate ; \
+	${MAKE} install-dep coverage.html coverage.xml mypy pep257_report.txt \
+		sloccount.sc pep8_report.txt pylint_report.txt
+
 FORCE:

schema_salad/__init__.py

@@ -1,7 +1,6 @@
 import logging
 import sys
-if sys.version_info >= (2,7):
-    import typing
+import typing
 
 __author__ = '[email protected]'
 

schema_salad/__main__.py

@@ -1,6 +1,5 @@
 from . import main
 import sys
-if sys.version_info >= (2,7):
-    import typing
+import typing
 
 sys.exit(main.main())

schema_salad/add_dictlist.py

@@ -1,8 +1,7 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, Dict
 
-def add_dictlist(di, key, val):
+def add_dictlist(di, key, val):  # type: (Dict, Any, Any) -> None
     if key not in di:
         di[key] = []
     di[key].append(val)

schema_salad/aslist.py

@@ -1,8 +1,7 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, List
 
-def aslist(l):
+def aslist(l):  # type: (Any) -> List
     """Convenience function to wrap single items and lists, and return lists unchanged."""
 
     if isinstance(l, list):

schema_salad/flatten.py

@@ -1,9 +1,9 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, Tuple
 
 # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
 def flatten(l, ltypes=(list, tuple)):
+    # type: (Any, Any) -> Any
     if l is None:
         return []
     if not isinstance(l, ltypes):
@@ -21,4 +21,4 @@ def flatten(l, ltypes=(list, tuple)):
             else:
                 l[i:i + 1] = l[i]
         i += 1
-    return ltype(l)
+    return ltype(l)  # type: ignore

schema_salad/jsonld_context.py

@@ -1,10 +1,11 @@
+import collections
 import shutil
 import json
 import ruamel.yaml as yaml
 try:
-        from ruamel.yaml import CSafeLoader as SafeLoader
+    from ruamel.yaml import CSafeLoader as SafeLoader
 except ImportError:
-        from ruamel.yaml import SafeLoader
+    from ruamel.yaml import SafeLoader  # type: ignore
 import os
 import subprocess
 import copy
@@ -15,62 +16,74 @@
 from rdflib import Graph, URIRef
 import rdflib.namespace
 from rdflib.namespace import RDF, RDFS
-try:
-    import urlparse
-except ImportError:
-    import urllib.parse as urlparse
-    unicode=str
-    basestring=str
+import urlparse
 import logging
 from .aslist import aslist
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, cast, Dict, Iterable, Tuple, Union
+from .ref_resolver import Loader
 
 _logger = logging.getLogger("salad")
 
+
 def pred(datatype, field, name, context, defaultBase, namespaces):
+    # type: (Dict[str, Union[Dict, str]], Dict, str, Loader.ContextType, str, Dict[str, rdflib.namespace.Namespace]) -> Union[Dict, str]
     split = urlparse.urlsplit(name)
 
-    v = None
+    vee = None  # type: Union[str, unicode]
 
     if split.scheme:
-        v = name
-        (ns, ln) = rdflib.namespace.split_uri(unicode(v))
+        vee = name
+        (ns, ln) = rdflib.namespace.split_uri(unicode(vee))
         name = ln
         if ns[0:-1] in namespaces:
-            v = unicode(namespaces[ns[0:-1]][ln])
-        _logger.debug("name, v %s %s", name, v)
+            vee = unicode(namespaces[ns[0:-1]][ln])
+        _logger.debug("name, v %s %s", name, vee)
+
+    v = None  # type: Any
 
     if field and "jsonldPredicate" in field:
         if isinstance(field["jsonldPredicate"], dict):
             v = {}
             for k, val in field["jsonldPredicate"].items():
-                v[("@"+k[1:] if k.startswith("_") else k)] = val
+                v[("@" + k[1:] if k.startswith("_") else k)] = val
         else:
             v = field["jsonldPredicate"]
     elif "jsonldPredicate" in datatype:
-        for d in datatype["jsonldPredicate"]:
-            if d["symbol"] == name:
-                v = d["predicate"]
+        if isinstance(datatype["jsonldPredicate"], collections.Iterable):
+            for d in datatype["jsonldPredicate"]:
+                if isinstance(d, dict):
+                    if d["symbol"] == name:
+                        v = d["predicate"]
+                else:
+                    raise Exception(
+                        "entries in the jsonldPredicate List must be "
+                        "Dictionaries")
+        else:
+            raise Exception("jsonldPredicate must be a List of Dictionaries.")
     # if not v:
     #     if field and "jsonldPrefix" in field:
     #         defaultBase = field["jsonldPrefix"]
     #     elif "jsonldPrefix" in datatype:
     #         defaultBase = datatype["jsonldPrefix"]
 
-    if not v:
-        v = defaultBase + name
+    ret = v or vee
+
+    if not ret:
+        ret = defaultBase + name
 
     if name in context:
-        if context[name] != v:
-            raise Exception("Predicate collision on %s, '%s' != '%s'" % (name, context[name], v))
+        if context[name] != ret:
+            raise Exception("Predicate collision on %s, '%s' != '%s'" %
+                            (name, context[name], ret))
     else:
-        _logger.debug("Adding to context '%s' %s (%s)", name, v, type(v))
-        context[name] = v
+        _logger.debug("Adding to context '%s' %s (%s)", name, ret, type(ret))
+        context[name] = ret
+
+    return ret
 
-    return v
 
 def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
+    # type: (Dict[str, Any], Graph, Loader.ContextType, str, Dict[str, rdflib.namespace.Namespace], str) -> None
     if t["type"] == "record":
         recordname = t["name"]
 
@@ -90,12 +103,14 @@ def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
             predicate = "%s:%s" % (defaultPrefix, recordname)
 
         if context.get(recordname, predicate) != predicate:
-            raise Exception("Predicate collision on '%s', '%s' != '%s'" % (recordname, context[recordname], predicate))
+            raise Exception("Predicate collision on '%s', '%s' != '%s'" % (
+                recordname, context[recordname], predicate))
 
         if not recordname:
             raise Exception()
 
-        _logger.debug("Adding to context '%s' %s (%s)", recordname, predicate, type(predicate))
+        _logger.debug("Adding to context '%s' %s (%s)",
+                      recordname, predicate, type(predicate))
         context[recordname] = predicate
 
         for i in t.get("fields", []):
@@ -123,7 +138,8 @@ def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
                 # TODO generate range from datatype.
 
             if isinstance(i["type"], dict) and "name" in i["type"]:
-                process_type(i["type"], g, context, defaultBase, namespaces, defaultPrefix)
+                process_type(i["type"], g, context, defaultBase,
+                             namespaces, defaultPrefix)
 
         if "extends" in t:
             for e in aslist(t["extends"]):
@@ -136,31 +152,26 @@ def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
 
 
 def salad_to_jsonld_context(j, schema_ctx):
-    context = {}
+    # type: (Iterable, Dict[str, Any]) -> Tuple[Loader.ContextType, Graph]
+    context = {}  # type: Loader.ContextType
     namespaces = {}
     g = Graph()
     defaultPrefix = ""
 
-    for k,v in schema_ctx.items():
+    for k, v in schema_ctx.items():
         context[k] = v
         namespaces[k] = rdflib.namespace.Namespace(v)
 
     if "@base" in context:
-        defaultBase = context["@base"]
+        defaultBase = cast(str, context["@base"])
         del context["@base"]
     else:
         defaultBase = ""
 
-    for k,v in namespaces.items():
+    for k, v in namespaces.items():
         g.bind(k, v)
 
     for t in j:
         process_type(t, g, context, defaultBase, namespaces, defaultPrefix)
 
     return (context, g)
-
-if __name__ == "__main__":
-    with open(sys.argv[1]) as f:
-        j = yaml.load(f, Loader=SafeLoader)
-        (ctx, g) = salad_to_jsonld_context(j)
-        print(json.dumps(ctx, indent=4, sort_keys=True))