Add general http caching support. Applies to $import, $include and $schemas.

Author: Peter Amstutz

schema_salad/ref_resolver.py

@@ -4,20 +4,26 @@
 import hashlib
 import logging
 import collections
-import requests
 import urlparse
 import re
 import copy
+import pprint
+from StringIO import StringIO
+
+from . import validate
+from .aslist import aslist
+from .flatten import flatten
+
+import requests
+from cachecontrol import CacheControl
+from cachecontrol.caches import FileCache
 import ruamel.yaml as yaml
+
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
 except ImportError:
     from ruamel.yaml import SafeLoader  # type: ignore
-from . import validate
-import pprint
-from StringIO import StringIO
-from .aslist import aslist
-from .flatten import flatten
+
 import rdflib
 from rdflib.namespace import RDF, RDFS, OWL
 from rdflib.plugins.parsers.notation3 import BadSyntax
@@ -64,7 +70,7 @@ def merge_properties(a, b):
 def SubLoader(loader):  # type: (Loader) -> Loader
     return Loader(loader.ctx, schemagraph=loader.graph,
                   foreign_properties=loader.foreign_properties, idx=loader.idx,
-                  cache=loader.cache)
+                  cache=loader.cache, session=loader.session)
 
 
 class Loader(object):
@@ -73,7 +79,7 @@ class Loader(object):
     DocumentType = TypeVar('DocumentType', List, Dict[unicode, Any])
 
     def __init__(self, ctx, schemagraph=None, foreign_properties=None,
-                 idx=None, cache=None):
+                 idx=None, cache=None, session=None):
         # type: (Loader.ContextType, rdflib.Graph, Set[unicode], Dict[unicode, Union[List, Dict[unicode, Any], unicode]], Dict[unicode, Any]) -> None
         normalize = lambda url: urlparse.urlsplit(url).geturl()
         if idx is not None:
@@ -97,6 +103,12 @@ def __init__(self, ctx, schemagraph=None, foreign_properties=None,
         else:
             self.cache = {}
 
+        if session is not None:
+            self.session = session
+        else:
+            self.session = CacheControl(requests.Session(),
+                                        cache=FileCache(os.path.join(os.environ["HOME"], ".cache", "salad")))
+
         self.url_fields = None  # type: Set[unicode]
         self.scoped_ref_fields = None  # type: Dict[unicode, int]
         self.vocab_fields = None  # type: Set[unicode]
@@ -166,23 +178,22 @@ def add_namespaces(self, ns):  # type: (Dict[unicode, unicode]) -> None
     def add_schemas(self, ns, base_url):
         # type: (Union[List[unicode], unicode], unicode) -> None
         for sch in aslist(ns):
-            for fmt in ['xml', 'turtle', 'rdfa']:
-                try:
-                    fetchurl = urlparse.urljoin(base_url, sch)
-                    if fetchurl not in self.cache:
-                        _logger.info("Getting external schema %s", fetchurl)
-                        cachekey = hashlib.md5(fetchurl).hexdigest()
-                        if os.path.join(os.environ["HOME"], ".settings
-                        self.cache[fetchurl] = rdflib.graph.Graph()
-                        self.cache[fetchurl].parse(fetchurl, format=fmt)
-                    self.graph += self.cache[fetchurl]
-                    break
-                except xml.sax.SAXParseException:  # type: ignore
-                    pass
-                except TypeError:
-                    pass
-                except BadSyntax:
-                    pass
+            fetchurl = urlparse.urljoin(base_url, sch)
+            if fetchurl not in self.cache:
+                _logger.info("Getting external schema %s", fetchurl)
+                content = self.fetch_text(fetchurl)
+                self.cache[fetchurl] = rdflib.graph.Graph()
+                for fmt in ['xml', 'turtle', 'rdfa']:
+                    try:
+                        self.cache[fetchurl].parse(data=content, format=fmt)
+                        self.graph += self.cache[fetchurl]
+                        break
+                    except xml.sax.SAXParseException:  # type: ignore
+                        pass
+                    except TypeError:
+                        pass
+                    except BadSyntax:
+                        pass
 
         for s, _, _ in self.graph.triples((None, RDF.type, RDF.Property)):
             self._add_properties(s)
@@ -607,9 +618,9 @@ def fetch_text(self, url):
         split = urlparse.urlsplit(url)
         scheme, path = split.scheme, split.path
 
-        if scheme in [u'http', u'https'] and requests:
+        if scheme in [u'http', u'https'] and self.session:
             try:
-                resp = requests.get(url)
+                resp = self.session.get(url)
                 resp.raise_for_status()
             except Exception as e:
                 raise RuntimeError(url, e)

schema_salad/tests/test_examples.py

@@ -32,13 +32,13 @@ def test_schemas(self):
         l = schema_salad.ref_resolver.Loader({})
 
         ra, _ = l.resolve_all({
-            u"$schemas": [get_data("tests/EDAM.owl")],
+            u"$schemas": ["file://" + get_data("tests/EDAM.owl")],
             u"$namespaces": {u"edam": u"http://edamontology.org/"},
             u"edam:has_format": u"edam:format_1915"
         }, "")
 
         self.assertEqual({
-            u"$schemas": [get_data("tests/EDAM.owl")],
+            u"$schemas": ["file://" + get_data("tests/EDAM.owl")],
             u"$namespaces": {u"edam": u"http://edamontology.org/"},
             u'http://edamontology.org/has_format': u'http://edamontology.org/format_1915'
         }, ra)

setup.py

@@ -33,7 +33,9 @@
     'rdflib >= 4.1.0',
     'rdflib-jsonld >= 0.3.0',
     'mistune',
-    'typing']
+    'typing',
+    'CacheControl',
+    'lockfile']
 
 install_requires.append("avro")  # TODO: remove me once cwltool is
 # available in Debian Stable, Ubuntu 12.04 LTS