Update the test suite.

Author: Julian

json/.travis.yml

@@ -1,4 +1,4 @@
 language: python
 python: "2.7"
 install: pip install jsonschema
-script: bin/suite_sanity_check -v
+script: bin/jsonschema_suite check

json/README.md

@@ -56,41 +56,17 @@ Currently, draft 3 should have essentially full coverage for the core schema.
 
 The beginnings of draft 4 are underway.
 
-Optional Tests
---------------
-
-Tests for optional validation are available in the "optional" directory,
-including for bignums (large integers), and for several of the values of the
-"format" property.
-
-Values of "format" covered are:
-
-  * regex
-  * date-time
-  * date
-  * time
-  * uri
-  * email
-  * ip-address
-  * ipv6
-  * host-name
-  * color
-
-Those also mentioned in draft 3, but currently omitted in the tests are:
-
-  * utc-millisec
-  * style
-  * phone
-
 Who Uses the Test Suite
 -----------------------
 
 This suite is being used by:
 
+  * [json-schema-validator (Java)](https://github.com/fge/json-schema-validator)
   * [jsonschema (python)](https://github.com/Julian/jsonschema)
   * [aeson-schema (haskell)](https://github.com/timjb/aeson-schema)
   * [direct-schema (javascript)](https://github.com/IreneKnapp/direct-schema)
   * [jsonschema (javascript)](https://github.com/tdegrunt/jsonschema)
+  * [JaySchema (javascript)](https://github.com/natesilva/jayschema)
 
 If you use it as well, please fork and send a pull request adding yourself to
 the list :).

json/bin/suite_sanity_check renamed to json/bin/jsonschema_suite

@@ -1,9 +1,12 @@
 #! /usr/bin/env python
-from unittest import TestCase, skipIf
+import argparse
 import fnmatch
 import json
 import logging
 import os
+import random
+import sys
+import unittest
 
 try:
     import jsonschema
@@ -37,7 +40,7 @@ TESTSUITE_SCHEMA = {
             }
         },
         "additionalProperties": False,
-    "minItems": 1
+        "minItems": 1
     }
 }
 
@@ -67,7 +70,7 @@ def collect(root_dir):
             yield os.path.join(root, filename)
 
 
-class SanityTests(TestCase):
+class SanityTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         logging.info("Looking for tests in %s", SUITE_ROOT_DIR)
@@ -101,16 +104,22 @@ class SanityTests(TestCase):
                 "%r contains a duplicate description" % (group,)
             )
 
-    @skipIf(jsonschema is None, "Validation library not present! Skipping.")
+    @unittest.skipIf(jsonschema is None, "Validation library not present!")
     def test_all_schemas_are_valid(self):
-        for case in cases(self.test_files):
-            # XXX: Add the other versions for the other drafts
-            try:
-                jsonschema.Draft3Validator.check_schema(case["schema"])
-            except jsonschema.SchemaError as error:
-                self.fail("%s contains an invalid schema (%s)" % (case, error))
-
-    @skipIf(jsonschema is None, "Validation library not present! Skipping.")
+        for schema in os.listdir(SUITE_ROOT_DIR):
+            schema_validator = jsonschema.validators.get(schema)
+            if schema_validator:
+                test_files = collect(os.path.join(SUITE_ROOT_DIR, schema))
+                for case in cases(test_files):
+                    try:
+                        schema_validator.check_schema(case["schema"])
+                    except jsonschema.SchemaError as error:
+                        self.fail("%s contains an invalid schema (%s)" %
+                                  (case, error))
+            else:
+                logging.warning("No schema validator for %s" % schema)
+
+    @unittest.skipIf(jsonschema is None, "Validation library not present!")
     def test_suites_are_valid(self):
         validator = jsonschema.Draft3Validator(TESTSUITE_SCHEMA)
         for tests in files(self.test_files):
@@ -120,6 +129,44 @@ class SanityTests(TestCase):
                 self.fail(str(error))
 
 
+def main(arguments):
+    if arguments.command == "check":
+        suite = unittest.TestLoader().loadTestsFromTestCase(SanityTests)
+        result = unittest.TextTestRunner(verbosity=2).run(suite)
+        sys.exit(not result.wasSuccessful())
+    elif arguments.command == "flatten":
+        selected_cases = [case for case in cases(collect(arguments.version))]
+
+        if arguments.randomize:
+            random.shuffle(selected_cases)
+
+        json.dump(selected_cases, arguments.out, indent=4)
+
+
+parser = argparse.ArgumentParser(
+    description="JSON Schema Test Suite utilities",
+)
+subparsers = parser.add_subparsers(help="utility commands", dest="command")
+
+check = subparsers.add_parser("check", help="Sanity check the test suite.")
+
+flatten = subparsers.add_parser(
+    "flatten",
+    help="Output a flattened file containing the test cases and any "
+            "additionally specified optional test cases."
+)
+flatten.add_argument(
+    "--randomize",
+    action="store_true",
+    help="randomize the order of the outputted cases",
+)
+flatten.add_argument(
+    "version", help="the directory containing the version to output",
+)
+flatten.add_argument(
+    "out", help="the output file to write to", type=argparse.FileType("w"),
+)
+
+
 if __name__ == "__main__":
-    from unittest import main
-    main()
+    main(parser.parse_args())

json/tests/draft3/optional/bignum.json

@@ -31,5 +31,19 @@
                 "valid": false
             }
         ]
+    },
+    {
+        "description": "float comparison with high precision",
+        "schema": {
+            "maximum": 972783798187987123879878123.18878137,
+            "exclusiveMaximum": true
+        },
+        "tests": [
+            {
+                "description": "comparison works for high numbers",
+                "data": 972783798187987123879878123.188781371,
+                "valid": false
+            }
+        ]
     }
 ]

json/tests/draft3/optional/format.json

@@ -21,7 +21,7 @@
         "tests": [
             {
                 "description": "a valid date-time string",
-                "data": "1963-06-19T08:30:06.283185",
+                "data": "1963-06-19T08:30:06.283185Z",
                 "valid": true
             },
             {
@@ -204,4 +204,4 @@
             }
         ]
     }
-]
+]

json/tests/draft3/optional/jsregex.json

@@ -0,0 +1,18 @@
+[
+    {
+        "description": "ECMA 262 regex dialect recognition",
+        "schema": { "format": "pattern" },
+        "tests": [
+            {
+                "description": "[^] is a valid regex",
+                "data": "[^]",
+                "valid": true
+            },
+            {
+                "description": "ECMA 262 has no support for lookbehind",
+                "data": "(?<=foo)bar",
+                "valid": false
+            }
+        ]
+    }
+]

json/tests/draft3/optional/zeroTerminatedFloats.json

@@ -0,0 +1,15 @@
+[
+    {
+        "description": "some languages do not distinguish between different types of numeric value",
+        "schema": {
+            "type": "integer"
+        },
+        "tests": [
+            {
+                "description": "a float is not an integer even without fractional part",
+                "data": 1.0,
+                "valid": false
+            }
+        ]
+    }
+]

json/tests/draft3/patternProperties.json

@@ -75,5 +75,36 @@
                 "valid": false
             }
         ]
+    },
+    {
+        "description": "regexes are not anchored by default and are case sensitive",
+        "schema": {
+            "patternProperties": {
+                "[0-9]{2,}": { "type": "boolean" },
+                "X_": { "type": "string" }
+            }
+        },
+        "tests": [
+            {
+                "description": "non recognized members are ignored",
+                "data": { "answer 1": "42" },
+                "valid": true
+            },
+            {
+                "description": "recognized members are accounted for",
+                "data": { "a31b": null },
+                "valid": false
+            },
+            {
+                "description": "regexes are case sensitive",
+                "data": { "a_x_3": 3 },
+                "valid": true
+            },
+            {
+                "description": "regexes are case sensitive, 2",
+                "data": { "a_X_3": 3 },
+                "valid": false
+            }
+        ]
     }
 ]

json/tests/draft3/type.json

@@ -13,11 +13,6 @@
                 "data": 1.1,
                 "valid": false
             },
-            {
-                "description": "a float is not an integer even without fractional part",
-                "data": 1.0,
-                "valid": false
-            },
             {
                 "description": "a string is not an integer",
                 "data": "foo",

json/tests/draft4/additionalItems.json

@@ -0,0 +1,82 @@
+[
+    {
+        "description": "additionalItems as schema",
+        "schema": {
+            "items": [{}],
+            "additionalItems": {"type": "integer"}
+        },
+        "tests": [
+            {
+                "description": "additional items match schema",
+                "data": [ null, 2, 3, 4 ],
+                "valid": true
+            },
+            {
+                "description": "additional items do not match schema",
+                "data": [ null, 2, 3, "foo" ],
+                "valid": false
+            }
+        ]
+    },
+    {
+        "description": "items is schema, no additionalItems",
+        "schema": {
+            "items": {},
+            "additionalItems": false
+        },
+        "tests": [
+            {
+                "description": "all items match schema",
+                "data": [ 1, 2, 3, 4, 5 ],
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "array of items with no additionalItems",
+        "schema": {
+            "items": [{}, {}, {}],
+            "additionalItems": false
+        },
+        "tests": [
+            {
+                "description": "no additional items present",
+                "data": [ 1, 2, 3 ],
+                "valid": true
+            },
+            {
+                "description": "additional items are not permitted",
+                "data": [ 1, 2, 3, 4 ],
+                "valid": false
+            }
+        ]
+    },
+    {
+        "description": "additionalItems as false without items",
+        "schema": {"additionalItems": false},
+        "tests": [
+            {
+                "description":
+                    "items defaults to empty schema so everything is valid",
+                "data": [ 1, 2, 3, 4, 5 ],
+                "valid": true
+            },
+            {
+                "description": "ignores non-arrays",
+                "data": {"foo" : "bar"},
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "additionalItems are allowed by default",
+        "schema": {"items": [{"type": "integer"}]},
+        "tests": [
+            {
+                "description": "only the first item is validated",
+                "data": [1, "foo", false],
+                "valid": true
+            }
+        ]
+    }
+]