Add map and typedsl resolution documentation. (#96)

* Add map and typedsl resolution documentation.
* Improve schema layout in spec.
* Update Salad spec to mark as v1.0 instead of draft-1.

Author: tetron

schema_salad/makedoc.py

@@ -158,8 +158,8 @@ def fix_doc(doc):  # type: (Union[List[str], str]) -> str
 
 class RenderType(object):
 
-    def __init__(self, toc, j, renderlist, redirects):
-        # type: (ToC, List[Dict], str, Dict) -> None
+    def __init__(self, toc, j, renderlist, redirects, primitiveType):
+        # type: (ToC, List[Dict], str, Dict, str) -> None
         self.typedoc = StringIO()
         self.toc = toc
         self.subs = {}  # type: Dict[str, str]
@@ -168,6 +168,7 @@ def __init__(self, toc, j, renderlist, redirects):
         self.rendered = set()  # type: Set[str]
         self.redirects = redirects
         self.title = None  # type: Optional[str]
+        self.primitiveType = primitiveType
 
         for t in j:
             if "extends" in t:
@@ -227,7 +228,6 @@ def typefmt(self,
                 jsonldPredicate=None    # type: Optional[Dict[str, str]]
                 ):
         # type: (...) -> Union[str, unicode]
-        global primitiveType
         if isinstance(tp, list):
             if nbsp and len(tp) <= 3:
                 return "&nbsp;|&nbsp;".join([self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate) for n in tp])
@@ -264,7 +264,7 @@ def typefmt(self,
             if str(tp) in redirects:
                 return """<a href="%s">%s</a>""" % (redirects[tp], redirects[tp])
             elif str(tp) in basicTypes:
-                return """<a href="%s">%s</a>""" % (primitiveType, schema.avro_name(str(tp)))
+                return """<a href="%s">%s</a>""" % (self.primitiveType, schema.avro_name(str(tp)))
             else:
                 _, frg = urlparse.urldefrag(tp)
                 if frg is not '':
@@ -277,6 +277,9 @@ def render_type(self, f, depth):  # type: (Dict[str, Any], int) -> None
             return
         self.rendered.add(f["name"])
 
+        if f.get("abstract"):
+            return
+
         if "doc" not in f:
             f["doc"] = ""
 
@@ -324,7 +327,7 @@ def extendsfrom(item, ex):
 
             _, frg = urlparse.urldefrag(f["name"])
             num = self.toc.add_entry(depth, frg)
-            doc = "## %s %s\n" % (num, frg)
+            doc = "%s %s %s\n" % (("#" * depth), num, frg)
         else:
             doc = ""
 
@@ -413,12 +416,12 @@ def extendsfrom(item, ex):
             self.render_type(self.typemap[s], depth)
 
 
-def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
-    # type: (List[Dict[unicode, Any]], IO[Any], str, Dict, str, str) -> None
+def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink, primtype):
+    # type: (List[Dict[unicode, Any]], IO[Any], str, Dict, str, str, str) -> None
     toc = ToC()
     toc.start_numbering = False
 
-    rt = RenderType(toc, j, renderlist, redirects)
+    rt = RenderType(toc, j, renderlist, redirects, primtype)
     content = rt.typedoc.getvalue()  # type: unicode
 
     outdoc.write("""
@@ -496,8 +499,7 @@ def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
     </html>""")
 
 
-if __name__ == "__main__":
-
+def main():  # type: () -> None
     parser = argparse.ArgumentParser()
     parser.add_argument("schema")
     parser.add_argument('--only', action='append')
@@ -526,10 +528,12 @@ def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
                 s.append(j)
             else:
                 raise ValueError("Schema must resolve to a list or a dict")
-
-    primitiveType = args.primtype
     redirect = {}
     for r in (args.redirect or []):
         redirect[r.split("=")[0]] = r.split("=")[1]
     renderlist = args.only if args.only else []
-    avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
+    avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink, args.primtype)
+
+
+if __name__ == "__main__":
+    main()

schema_salad/metaschema/map_res.yml

@@ -0,0 +1,36 @@
+- |
+  ## Identifier maps
+
+  The schema may designate certain fields as having a `mapSubject`.  If the
+  value of the field is a JSON object, it must be transformed into an array of
+  JSON objects.  Each key-value pair from the source JSON object is a list
+  item, each list item must be a JSON objects, and the value of the key is
+  assigned to the field specified by `mapSubject`.
+
+  Fields which have `mapSubject` specified may also supply a `mapPredicate`.
+  If the value of a map item is not a JSON object, the item is transformed to a
+  JSON object with the key assigned to the field specified by `mapSubject` and
+  the value assigned to the field specified by `mapPredicate`.
+
+  ### Identifier map example
+
+  Given the following schema:
+
+  ```
+- $include: map_res_schema.yml
+- |
+  ```
+
+  Process the following example:
+
+  ```
+- $include: map_res_src.yml
+- |
+  ```
+
+  This becomes:
+
+  ```
+- $include: map_res_proc.yml
+- |
+  ```

schema_salad/metaschema/map_res_proc.yml

@@ -0,0 +1,12 @@
+{
+    "mapped": [
+        {
+            "value": "daphne",
+            "key": "fred"
+        },
+        {
+            "value": "scooby",
+            "key": "shaggy"
+        }
+    ]
+}

schema_salad/metaschema/map_res_schema.yml

@@ -0,0 +1,30 @@
+{
+  "$graph": [{
+    "name": "MappedType",
+    "type": "record",
+    "documentRoot": true,
+    "fields": [{
+      "name": "mapped",
+      "type": {
+        "type": "array",
+        "items": "ExampleRecord"
+      },
+      "jsonldPredicate": {
+        "mapSubject": "key",
+        "mapPredicate": "value"
+      }
+    }],
+  },
+  {
+    "name": "ExampleRecord",
+    "type": "record",
+    "fields": [{
+      "name": "key",
+      "type": "string"
+      }, {
+      "name": "value",
+      "type": "string"
+      }
+    ]
+  }]
+}

schema_salad/metaschema/map_res_src.yml

@@ -0,0 +1,8 @@
+{
+  "mapped": {
+    "shaggy": {
+      "value": "scooby"
+    },
+    "fred": "daphne"
+  }
+}

schema_salad/metaschema/metaschema.yml

@@ -18,6 +18,8 @@ $graph:
     - $import: link_res.yml
     - $import: vocab_res.yml
     - $include: import_include.md
+    - $import: map_res.yml
+    - $import: typedsl_res.yml
 
 - name: "Link_Validation"
   type: documentation
@@ -154,6 +156,7 @@ $graph:
 - name: NamedType
   type: record
   abstract: true
+  docParent: "#Schema"
   fields:
     - name: name
       type: string
@@ -164,6 +167,7 @@ $graph:
 - name: DocType
   type: record
   abstract: true
+  docParent: "#Schema"
   fields:
     - name: doc
       type:
@@ -240,6 +244,7 @@ $graph:
 
 
 - name: SaladRecordSchema
+  docParent: "#Schema"
   type: record
   extends: [NamedType, RecordSchema, SchemaDefinedType]
   documentRoot: true
@@ -277,6 +282,7 @@ $graph:
         mapPredicate: specializeTo
 
 - name: SaladEnumSchema
+  docParent: "#Schema"
   type: record
   extends: [EnumSchema, SchemaDefinedType]
   documentRoot: true
@@ -297,6 +303,7 @@ $graph:
 
 - name: Documentation
   type: record
+  docParent: "#Schema"
   extends: [NamedType, DocType]
   documentRoot: true
   doc: |

schema_salad/metaschema/metaschema_base.yml

@@ -8,6 +8,12 @@ $namespaces:
   xsd:  "http://www.w3.org/2001/XMLSchema#"
 
 $graph:
+
+- name: "Schema"
+  type: documentation
+  doc: |
+    # Schema
+
 - name: PrimitiveType
   type: enum
   symbols:
@@ -35,6 +41,7 @@ $graph:
 - name: Any
   type: enum
   symbols: ["#Any"]
+  docAfter: "#PrimitiveType"
   doc: |
     The **Any** type validates for any non-null value.
 

schema_salad/metaschema/salad.md

@@ -26,7 +26,7 @@ Web.
 
 This document is the product of the [Common Workflow Language working
 group](https://groups.google.com/forum/#!forum/common-workflow-language).  The
-latest version of this document is available in the "schema_salad" directory at
+latest version of this document is available in the "schema_salad" repository at
 
 https://github.com/common-workflow-language/schema_salad
 
@@ -38,7 +38,7 @@ under the terms of the Apache License, version 2.0.
 # Introduction
 
 The JSON data model is an extremely popular way to represent structured
-data.  It is attractive because of it's relative simplicity and is a
+data.  It is attractive because of its relative simplicity and is a
 natural fit with the standard types of many programming languages.
 However, this simplicity means that basic JSON lacks expressive features
 useful for working with complex data structures and document formats, such
@@ -70,12 +70,17 @@ and RDF schema, and production of RDF triples by applying the JSON-LD
 context.  The schema language also provides for robust support of inline
 documentation.
 
-## Introduction to draft 1
+## Introduction to v1.0
 
-This is the first version of Schema Salad.  It is developed concurrently
-with draft 3 of the Common Workflow Language for use in specifying the
-Common Workflow Language, however Schema Salad is intended to be useful to
-a broader audience.
+This is the second version of of the Schema Salad specification.  It is
+developed concurrently with v1.0 of the Common Workflow Language for use in
+specifying the Common Workflow Language, however Schema Salad is intended to be
+useful to a broader audience.  Compared to the draft-1 schema salad
+specification, the following changes have been made:
+
+* Use of [mapSubject and mapPredicate](#Identifier_maps) to transform maps to lists of records.
+* Resolution of the [domain Specific Language for types](#Domain_Specific_Language_for_types)
+* Consolidation of the formal [schema into section 5](#Schema).
 
 ## References to Other Specifications
 

schema_salad/metaschema/typedsl_res.yml

@@ -0,0 +1,33 @@
+- |
+  ## Domain Specific Language for types
+
+  Fields may be tagged `typeDSL: true`.  If so, the field is expanded using the
+  following micro-DSL for schema salad types:
+
+  * If the type ends with a question mark `?` it is expanded to a union with `null`
+  * If the type ends with square brackets `[]` it is expanded to an array with items of the preceeding type symbol
+  * The type may end with both `[]?` to indicate it is an optional array.
+  * Identifier resolution is applied after type DSL expansion.
+
+  ### Type DSL example
+
+  Given the following schema:
+
+  ```
+- $include: typedsl_res_schema.yml
+- |
+  ```
+
+  Process the following example:
+
+  ```
+- $include: typedsl_res_src.yml
+- |
+  ```
+
+  This becomes:
+
+  ```
+- $include: typedsl_res_proc.yml
+- |
+  ```

schema_salad/metaschema/typedsl_res_proc.yml

@@ -0,0 +1,26 @@
+[
+    {
+        "extype": "string"
+    }, 
+    {
+        "extype": [
+            "null", 
+            "string"
+        ]
+    }, 
+    {
+        "extype": {
+            "type": "array", 
+            "items": "string"
+        }
+    }, 
+    {
+        "extype": [
+            "null", 
+            {
+                "type": "array", 
+                "items": "string"
+            }
+        ]
+    }
+]

schema_salad/metaschema/typedsl_res_schema.yml

@@ -0,0 +1,17 @@
+{
+  "$graph": [
+  {"$import": "metaschema_base.yml"},
+  {
+    "name": "TypeDSLExample",
+    "type": "record",
+    "documentRoot": true,
+    "fields": [{
+      "name": "extype",
+      "type": "string",
+      "jsonldPredicate": {
+        _type: "@vocab",
+        "typeDSL": true
+      }
+    }]
+  }]
+}

schema_salad/metaschema/typedsl_res_src.yml

@@ -0,0 +1,9 @@
+[{
+  "extype": "string"
+}, {
+  "extype": "string?"
+}, {
+  "extype": "string[]"
+}, {
+  "extype": "string[]?"
+}]