Merge pull request #38 from common-workflow-language/mixin

Implement $mixin

Author: mr-c

schema_salad/metaschema/import_include.md

@@ -110,3 +110,67 @@ This becomes:
   }
 }
 ```
+
+
+## Mixin
+
+During preprocessing traversal, an implementation must resolve `$mixin`
+directives.  An `$mixin` directive is an object consisting of the field
+`$mixin` specifying resource by URI string.  If there are additional fields in
+the `$mixin` object, these fields override fields in the object which is loaded
+from the `$mixin` URI.
+
+The URI string must be resolved to an absolute URI using the link resolution
+rules described previously.  Implementations must support loading from `file`,
+`http` and `https` resources.  The URI referenced by `$mixin` must be loaded
+and recursively preprocessed as a Salad document.  The external imported
+document must inherit the context of the importing document, however the file
+URI for processing the imported document must be the URI used to retrieve the
+imported document.  The `$mixin` URI must not include a document fragment.
+
+Once loaded and processed, the `$mixin` node is replaced in the document
+structure by the object or array yielded from the import operation.
+
+URIs may reference document fragments which refer to specific an object in
+the target document.  This indicates that the `$mixin` node must be
+replaced by only the object with the appropriate fragment identifier.
+
+It is a fatal error if an import directive refers to an external resource
+or resource fragment which does not exist or is not accessible.
+
+### Mixin example
+
+mixin.yml:
+```
+{
+  "hello": "world",
+  "carrot": "orange"
+}
+
+```
+
+parent.yml:
+```
+{
+  "form": {
+    "bar": {
+      "$mixin": "mixin.yml"
+      "carrot": "cake"
+      }
+  }
+}
+
+```
+
+This becomes:
+
+```
+{
+  "form": {
+    "bar": {
+      "hello": "world",
+      "carrot": "cake"
+    }
+  }
+}
+```

schema_salad/ref_resolver.py

@@ -7,6 +7,7 @@
 import requests
 import urlparse
 import re
+import copy
 import ruamel.yaml as yaml
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
@@ -259,25 +260,30 @@ def resolve_ref(self, ref, base_url=None, checklinks=True):
 
         obj = None  # type: Dict[unicode, Any]
         inc = False
+        mixin = None
 
         # If `ref` is a dict, look for special directives.
         if isinstance(ref, dict):
             obj = ref
-            if u"$import" in ref:
+            if u"$import" in obj:
                 if len(obj) == 1:
                     ref = obj[u"$import"]
                     obj = None
                 else:
                     raise ValueError(
-                        "'$import' must be the only field in %s" % (str(obj)))
+                        u"'$import' must be the only field in %s" % (str(obj)))
             elif u"$include" in obj:
                 if len(obj) == 1:
                     ref = obj[u"$include"]
                     inc = True
                     obj = None
                 else:
                     raise ValueError(
-                        "'$include' must be the only field in %s" % (str(obj)))
+                        u"'$include' must be the only field in %s" % (str(obj)))
+            elif u"$mixin" in obj:
+                ref = obj[u"$mixin"]
+                mixin = obj
+                obj = None
             else:
                 ref = None
                 for identifier in self.identifiers:
@@ -286,15 +292,15 @@ def resolve_ref(self, ref, base_url=None, checklinks=True):
                         break
                 if not ref:
                     raise ValueError(
-                        "Object `%s` does not have identifier field in %s" % (obj, self.identifiers))
+                        u"Object `%s` does not have identifier field in %s" % (obj, self.identifiers))
 
         if not isinstance(ref, (str, unicode)):
-            raise ValueError("Must be string: `%s`" % str(ref))
+            raise ValueError(u"Must be string: `%s`" % str(ref))
 
         url = self.expand_url(ref, base_url, scoped_id=(obj is not None))
 
         # Has this reference been loaded already?
-        if url in self.idx:
+        if url in self.idx and (not mixin):
             return self.idx[url], {}
 
         # "$include" directive means load raw text
@@ -309,14 +315,25 @@ def resolve_ref(self, ref, base_url=None, checklinks=True):
         else:
             # Load structured document
             doc_url, frg = urlparse.urldefrag(url)
-            if doc_url in self.idx:
+            if doc_url in self.idx and (not mixin):
+                # If the base document is in the index, it was already loaded,
+                # so if we didn't find the reference earlier then it must not
+                # exist.
                 raise validate.ValidationException(
-                    "Reference `#%s` not found in file `%s`." % (frg, doc_url))
-            doc = self.fetch(doc_url)
+                    u"Reference `#%s` not found in file `%s`." % (frg, doc_url))
+            doc = self.fetch(doc_url, inject_ids=(not mixin))
 
         # Recursively expand urls and resolve directives
-        resolved_obj, metadata = self.resolve_all(
-            doc if doc else obj, doc_url, checklinks=checklinks)
+        if mixin:
+            doc = copy.deepcopy(doc)
+            doc.update(mixin)
+            del doc["$mixin"]
+            url = None
+            resolved_obj, metadata = self.resolve_all(
+                doc, base_url, file_base=doc_url, checklinks=checklinks)
+        else:
+            resolved_obj, metadata = self.resolve_all(
+                doc if doc else obj, doc_url, checklinks=checklinks)
 
         # Requested reference should be in the index now, otherwise it's a bad
         # reference
@@ -477,6 +494,8 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True):
             # Handle $import and $include
             if (u'$import' in document or u'$include' in document):
                 return self.resolve_ref(document, base_url=file_base, checklinks=checklinks)
+            elif u'$mixin' in document:
+                return self.resolve_ref(document, base_url=base_url, checklinks=checklinks)
         elif isinstance(document, list):
             pass
         else:
@@ -534,7 +553,7 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True):
                     document[key], _ = loader.resolve_all(
                         val, base_url, file_base=file_base, checklinks=False)
             except validate.ValidationException as v:
-                _logger.debug("loader is %s", id(loader))
+                _logger.warn("loader is %s", id(loader), exc_info=v)
                 raise validate.ValidationException("(%s) (%s) Validation error in field %s:\n%s" % (
                     id(loader), file_base, key, validate.indent(str(v))))
 
@@ -543,7 +562,7 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True):
             try:
                 while i < len(document):
                     val = document[i]
-                    if isinstance(val, dict) and u"$import" in val:
+                    if isinstance(val, dict) and (u"$import" in val or u"$mixin" in val):
                         l, _ = loader.resolve_ref(val, base_url=file_base, checklinks=False)
                         if isinstance(l, list):  # never true?
                             del document[i]
@@ -558,6 +577,7 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True):
                             val, base_url, file_base=file_base, checklinks=False)
                         i += 1
             except validate.ValidationException as v:
+                _logger.warn("failed", exc_info=v)
                 raise validate.ValidationException("(%s) (%s) Validation error in position %i:\n%s" % (
                     id(loader), file_base, i, validate.indent(str(v))))
 
@@ -601,7 +621,7 @@ def fetch_text(self, url):
         else:
             raise ValueError('Unsupported scheme in url: %s' % url)
 
-    def fetch(self, url):  # type: (unicode) -> Any
+    def fetch(self, url, inject_ids=True):  # type: (unicode, bool) -> Any
         if url in self.idx:
             return self.idx[url]
         try:
@@ -614,7 +634,7 @@ def fetch(self, url):  # type: (unicode) -> Any
             result = yaml.load(textIO, Loader=SafeLoader)
         except yaml.parser.ParserError as e:  # type: ignore
             raise validate.ValidationException("Syntax error %s" % (e))
-        if isinstance(result, dict) and self.identifiers:
+        if isinstance(result, dict) and inject_ids and self.identifiers:
             for identifier in self.identifiers:
                 if identifier not in result:
                     result[identifier] = url

tests/mixin.yml

@@ -0,0 +1,2 @@
+id: four
+one: two

tests/test_examples.py

@@ -6,6 +6,7 @@
 import rdflib
 import ruamel.yaml as yaml
 import json
+import os
 
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
@@ -322,5 +323,33 @@ def test_scoped_id(self):
         print(g.serialize(format="n3"))
 
 
+    def test_mixin(self):
+        ldr = schema_salad.ref_resolver.Loader({})
+        ra = ldr.resolve_ref({"$mixin": "mixin.yml", "one": "five"},
+                             base_url="file://"+os.getcwd()+"/tests/")
+        self.assertEqual({'id': 'four', 'one': 'five'}, ra[0])
+
+        ldr = schema_salad.ref_resolver.Loader({"id": "@id"})
+        base_url="file://"+os.getcwd()+"/tests/"
+        ra = ldr.resolve_all([{
+            "id": "a",
+            "m": {"$mixin": "mixin.yml"}
+        }, {
+            "id": "b",
+            "m": {"$mixin": "mixin.yml"}
+        }], base_url=base_url)
+        self.assertEqual([{
+            'id': base_url+'#a',
+            'm': {
+                'id': base_url+u'#a/four',
+                'one': 'two'
+            },
+        }, {
+            'id': base_url+'#b',
+            'm': {
+                'id': base_url+u'#b/four',
+                'one': 'two'}
+        }], ra[0])
+
 if __name__ == '__main__':
     unittest.main()