Squashed 'draft-4/salad/' changes from 9c8c36f..c509a07

c509a07 Merge pull request #34 from common-workflow-language/unscoped_ids
f857e9d type fixups
0407370 Allow idmaps for record fields.
5d9a08e Use makerdf for --print-rdf
ce00252 Add test for non-scoped identifiers.
ef27301 restore a dropped cast
6ac559e version bump
2cfde6b Merge pull request #32 from common-workflow-language/type_dsl
b8edddc re-enable test_avro_regression
bedb736 Merge branch 'type_dsl' of github.com:common-workflow-language/schema_salad into type_dsl
e93b038 Add missing file
9ca6cc9 remove more str types, possible fix for edam failed test
d65fb83 more unicode
4350b30 type clarification
96f08e8 Merge remote-tracking branch 'origin/master' into type_dsl
57f7e83 borrow the updated re.pyi from typeshed
c03eb1e type fixes
6e71f35 Merge remote-tracking branch 'origin/master' into type_dsl
e554b2b Merge pull request #33 from common-workflow-language/fix-resolve_all_refactor
ad34521 fix bug introduced by refactor
5393edc fixes types, removes a few more Anys
e298bdc Detect when an abstract class is used but doesn't have a concrete subtype.
851b3c1 Split minimal base types used in CWL into metaschema_base.yml
8823aaa Adjust ref scopes
3431e18 Use scopedRef and typeDSL in metaschema
1e6dab6 Using typeDSL in metaschema.
3055f70 Update bootstrap context.
31db28e Merge branch 'master' into type_dsl.  Add test for typeDSL.
116d9e4 Merge branch 'resolve_all_refactor'
8072bf5 "type" DSL wip
33ef215 Refactor resolve_all into smaller methods.
ad8a111 Merge pull request #28 from common-workflow-language/scoped_ref
5aadf27 fix types
4ef52be Improve documentation.  Add feature of skipping some inner scopes.
c6d4a7b Reworking refScope to be fixed instead of search based.
ae0ef4e Remove explicit calls to validate_links since it is folded into resolve_all().
2fa6bfb update types
e5d4730 New test for scoped ref that better reflects actual desired behavior.
efbc9c4 Add scopedRef to metaschema.  Tweak test.
9fcd67d Fold link validation into resolve_all() and resolve scoped identifiers.
da10eec always be updating (mypy-lang)
e1e17ea upgrade Jenkins setuptools
563e5ef fix jenkins build
f0ac285 Merge branch 'master' of github.com:common-workflow-language/schema_salad
84738f1 better RDF schema error handling (#26)
3b8dc09 mypy 0.4 is out (#25)
8c5bda0 Fix mapSubject fields when $import or $include is provided.
5b969ca fix diff-cover
d798a7e split out mypy
f43fa45 Fix tests
2d0398e Bump version.
aeab7c3 Merge branch 'master' of github.com:common-workflow-language/schema_salad
e381f64 Return metaschema loader from load_schema.  Don't crash expanding identifier fields when value is not a string.
15dae9f enhance CI
fd2fcb3 more type checking (#21)
c51c723 enable rdfa schema format (#22)
ff192e8 turn off py3 support for now (#19)
0c5dba5 measure code coverage (#12)
2514a23 switch to ruamel.yaml, use C loader if available, only load safely (#20)
37c129d Merge branch 'master' of github.com:common-workflow-language/schema_salad
0b09a19 Makedoc detects any ``` for toggling looking for # for section headings.
d361b6c Bump version #
2da084e Add mapSubject and mapPredicate to metaschema.
2b00752 Turn idMap into mapSubject and mapPredicate.
af7a419 Support id maps in preprocessing.
984ab9c Fix makedoc section header bug.
cc0f52b Add margin to <pre> sections in generated documentation.

git-subtree-dir: draft-4/salad
git-subtree-split: c509a07

Author: Peter Amstutz

.travis.yml

@@ -3,14 +3,13 @@ python: 2.7
 os:
   - linux
 env:
-  - TOX_ENV=py34-lint
-  - TOX_ENV=py33-lint
   - TOX_ENV=py27-lint
-  - TOX_ENV=py26-lint
-  - TOX_ENV=py34-unit
-  - TOX_ENV=py33-unit
   - TOX_ENV=py27-unit
-  - TOX_ENV=py26-unit
+  - TOX_ENV=py34-mypy
+#  - TOX_ENV=py34-lint
+#  - TOX_ENV=py33-lint
+#  - TOX_ENV=py34-unit
+#  - TOX_ENV=py33-unit
 
 install:
   - pip install tox

Makefile

@@ -0,0 +1,185 @@
+# This file is part of schema-salad,
+# https://github.com/common-workflow-language/schema-salad/, and is
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Contact: [email protected]
+
+# make pep8 to check for basic Python code compliance
+# make autopep8 to fix most pep8 errors
+# make pylint to check Python code for enhanced compliance including naming
+#  and documentation
+# make coverage-report to check coverage of the python scripts by the tests
+
+MODULE=schema_salad
+
+# `SHELL=bash` Will break Titus's laptop, so don't use BASH-isms like
+# `[[` conditional expressions.
+PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257
+
+VERSION=$(shell git describe --tags --dirty | sed s/v//)
+
+## all         : default task
+all: ./setup.py develop
+
+## help        : print this help message and exit
+help: Makefile
+	@sed -n 's/^##//p' $<
+
+## install-dep : install most of the development dependencies via pip
+install-dep: install-dependencies
+
+install-dependencies:
+	pip install --upgrade $(DEVPKGS)
+	pip install -r requirements.txt
+
+## install     : install the ${MODULE} module and schema-salad-tool
+install: FORCE
+	./setup.py build install
+
+## dist        : create a module package for distribution
+dist: dist/${MODULE}-$(VERSION).tar.gz
+
+dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
+	./setup.py sdist
+
+## clean       : clean up all temporary / machine-generated files
+clean: FORCE
+	rm -f ${MODILE}/*.pyc tests/*.pyc
+	./setup.py clean --all || true
+	rm -Rf .coverage
+	rm -f diff-cover.html
+
+## pep8        : check Python code style
+pep8: $(PYSOURCES)
+	pep8 --exclude=_version.py  --show-source --show-pep8 $^ || true
+
+pep8_report.txt: $(PYSOURCES)
+	pep8 --exclude=_version.py $^ > $@ || true
+
+diff_pep8_report: pep8_report.txt
+	diff-quality --violations=pep8 pep8_report.txt
+
+## pep257      : check Python code style
+pep257: $(PYSOURCES)
+	pep257 --ignore=D100,D101,D102,D103 $^ || true
+
+pep257_report.txt: $(PYSOURCES)
+	pep257 setup.py $^ > $@ 2>&1 || true
+
+diff_pep257_report: pep257_report.txt
+	diff-quality --violations=pep8 pep257_report.txt
+
+## autopep8    : fix most Python code indentation and formatting
+autopep8: $(PYSOURCES)
+	autopep8 --recursive --in-place --ignore E309 $^
+
+# A command to automatically run astyle and autopep8 on appropriate files
+## format      : check/fix all code indentation and formatting (runs autopep8)
+format: autopep8
+	# Do nothing
+
+## pylint      : run static code analysis on Python code
+pylint: $(PYSOURCES)
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+                $^ || true
+
+pylint_report.txt: ${PYSOURCES}
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+		$^ > $@ || true
+
+diff_pylint_report: pylint_report.txt
+	diff-quality --violations=pylint pylint_report.txt
+
+.coverage: $(PYSOURCES)
+	coverage run --branch --source=${MODULE} setup.py test
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-jsonld-context schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-rdfs schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-avro schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-rdf schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-pre schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-index schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.main \
+		--print-metadata schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+	coverage run --append --branch --source=${MODULE} \
+		-m schema_salad.makedoc schema_salad/metaschema/metaschema.yml \
+		> /dev/null
+
+coverage.xml: .coverage
+	coverage xml
+
+coverage.html: htmlcov/index.html
+
+htmlcov/index.html: .coverage
+	coverage html
+	@echo Test coverage of the Python code is now in htmlcov/index.html
+
+coverage-report: .coverage
+	coverage report
+
+diff-cover: coverage.xml
+	diff-cover $^
+
+diff-cover.html: coverage.xml
+	diff-cover $^ --html-report $@
+
+## test        : run the ${MODULE} test suite
+test: FORCE
+	python setup.py test
+
+sloccount.sc: ${PYSOURCES} Makefile
+	sloccount --duplicates --wide --details $^ > $@
+
+## sloccount   : count lines of code
+sloccount: ${PYSOURCES} Makefile
+	sloccount $^
+
+list-author-emails:
+	@echo 'name, E-Mail Address'
+	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
+
+mypy: ${PYSOURCES}
+	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls schema_salad
+
+jenkins:
+	if ! test -d env ; then virtualenv env ; fi
+	. env/bin/activate ; \
+	pip install -U setuptools pip wheel ; \
+	${MAKE} install-dep coverage.html coverage.xml pep257_report.txt \
+		sloccount.sc pep8_report.txt pylint_report.txt
+	if ! test -d env3 ; then virtualenv -p python3 env3 ; fi
+	. env3/bin/activate ; \
+	pip install -U mypy-lang; ${MAKE} mypy
+
+FORCE:

README.rst

@@ -70,6 +70,6 @@ provides for robust support of inline documentation.
 .. _JSON-LD: http://json-ld.org
 .. _Avro: http://avro.apache.org
 .. _metaschema: https://github.com/common-workflow-language/schema_salad/blob/master/schema_salad/metaschema/metaschema.yml
-.. _specification: https://common-workflow-language.github.io/draft-3/SchemaSalad.html
+.. _specification: http://www.commonwl.org/draft-3/SchemaSalad.html
 .. _Language: https://github.com/common-workflow-language/common-workflow-language/blob/master/draft-3/CommandLineTool.yml
 .. _RDF: https://www.w3.org/RDF/

requirements.txt

@@ -1,5 +1,5 @@
 requests
-PyYAML
+ruamel.yaml
 rdflib >= 4.1.
 rdflib-jsonld >= 0.3.0
 mistune

schema_salad/__init__.py

@@ -1,7 +1,6 @@
 import logging
 import sys
-if sys.version_info >= (2,7):
-    import typing
+import typing
 
 __author__ = '[email protected]'
 

schema_salad/__main__.py

@@ -1,6 +1,5 @@
 from . import main
 import sys
-if sys.version_info >= (2,7):
-    import typing
+import typing
 
 sys.exit(main.main())

schema_salad/add_dictlist.py

@@ -1,8 +1,7 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, Dict
 
-def add_dictlist(di, key, val):
+def add_dictlist(di, key, val):  # type: (Dict, Any, Any) -> None
     if key not in di:
         di[key] = []
     di[key].append(val)

schema_salad/aslist.py

@@ -1,8 +1,7 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, List
 
-def aslist(l):
+def aslist(l):  # type: (Any) -> List
     """Convenience function to wrap single items and lists, and return lists unchanged."""
 
     if isinstance(l, list):

schema_salad/flatten.py

@@ -1,9 +1,9 @@
 import sys
-if sys.version_info >= (2,7):
-    import typing
+from typing import Any, Tuple
 
 # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
 def flatten(l, ltypes=(list, tuple)):
+    # type: (Any, Any) -> Any
     if l is None:
         return []
     if not isinstance(l, ltypes):
@@ -21,4 +21,4 @@ def flatten(l, ltypes=(list, tuple)):
             else:
                 l[i:i + 1] = l[i]
         i += 1
-    return ltype(l)
+    return ltype(l)  # type: ignore