Merge pull request #49 from common-workflow-language/schema-cache

Cache external schemas after being loaded 1st time

Author: mr-c

schema_salad/ref_resolver.py

@@ -4,20 +4,26 @@
 import hashlib
 import logging
 import collections
-import requests
 import urlparse
 import re
 import copy
+import pprint
+from StringIO import StringIO
+
+from . import validate
+from .aslist import aslist
+from .flatten import flatten
+
+import requests
+from cachecontrol.wrapper import CacheControl
+from cachecontrol.caches import FileCache
 import ruamel.yaml as yaml
+
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
 except ImportError:
     from ruamel.yaml import SafeLoader  # type: ignore
-from . import validate
-import pprint
-from StringIO import StringIO
-from .aslist import aslist
-from .flatten import flatten
+
 import rdflib
 from rdflib.namespace import RDF, RDFS, OWL
 from rdflib.plugins.parsers.notation3 import BadSyntax
@@ -64,7 +70,7 @@ def merge_properties(a, b):
 def SubLoader(loader):  # type: (Loader) -> Loader
     return Loader(loader.ctx, schemagraph=loader.graph,
                   foreign_properties=loader.foreign_properties, idx=loader.idx,
-                  cache=loader.cache)
+                  cache=loader.cache, session=loader.session)
 
 
 class Loader(object):
@@ -73,8 +79,8 @@ class Loader(object):
     DocumentType = TypeVar('DocumentType', List, Dict[unicode, Any])
 
     def __init__(self, ctx, schemagraph=None, foreign_properties=None,
-                 idx=None, cache=None):
-        # type: (Loader.ContextType, rdflib.Graph, Set[unicode], Dict[unicode, Union[List, Dict[unicode, Any], unicode]], Dict[unicode, Any]) -> None
+                 idx=None, cache=None, session=None):
+        # type: (Loader.ContextType, rdflib.Graph, Set[unicode], Dict[unicode, Union[List, Dict[unicode, Any], unicode]], Dict[unicode, Any], requests.sessions.Session) -> None
         normalize = lambda url: urlparse.urlsplit(url).geturl()
         if idx is not None:
             self.idx = idx
@@ -97,6 +103,13 @@ def __init__(self, ctx, schemagraph=None, foreign_properties=None,
         else:
             self.cache = {}
 
+        self.session = None  # type: requests.sessions.Session
+        if session is not None:
+            self.session = session
+        else:
+            self.session = CacheControl(requests.Session(),
+                                        cache=FileCache(os.path.join(os.environ["HOME"], ".cache", "salad")))
+
         self.url_fields = None  # type: Set[unicode]
         self.scoped_ref_fields = None  # type: Dict[unicode, int]
         self.vocab_fields = None  # type: Set[unicode]
@@ -166,17 +179,22 @@ def add_namespaces(self, ns):  # type: (Dict[unicode, unicode]) -> None
     def add_schemas(self, ns, base_url):
         # type: (Union[List[unicode], unicode], unicode) -> None
         for sch in aslist(ns):
-            for fmt in ['xml', 'turtle', 'rdfa']:
-                try:
-                    self.graph.parse(urlparse.urljoin(base_url, sch),
-                                     format=fmt)
-                    break
-                except xml.sax.SAXParseException:  # type: ignore
-                    pass
-                except TypeError:
-                    pass
-                except BadSyntax:
-                    pass
+            fetchurl = urlparse.urljoin(base_url, sch)
+            if fetchurl not in self.cache:
+                _logger.info("Getting external schema %s", fetchurl)
+                content = self.fetch_text(fetchurl)
+                self.cache[fetchurl] = rdflib.graph.Graph()
+                for fmt in ['xml', 'turtle', 'rdfa']:
+                    try:
+                        self.cache[fetchurl].parse(data=content, format=fmt)
+                        self.graph += self.cache[fetchurl]
+                        break
+                    except xml.sax.SAXParseException:  # type: ignore
+                        pass
+                    except TypeError:
+                        pass
+                    except BadSyntax:
+                        pass
 
         for s, _, _ in self.graph.triples((None, RDF.type, RDF.Property)):
             self._add_properties(s)
@@ -601,9 +619,9 @@ def fetch_text(self, url):
         split = urlparse.urlsplit(url)
         scheme, path = split.scheme, split.path
 
-        if scheme in [u'http', u'https'] and requests:
+        if scheme in [u'http', u'https'] and self.session:
             try:
-                resp = requests.get(url)
+                resp = self.session.get(url)
                 resp.raise_for_status()
             except Exception as e:
                 raise RuntimeError(url, e)

schema_salad/tests/test_examples.py

@@ -32,13 +32,13 @@ def test_schemas(self):
         l = schema_salad.ref_resolver.Loader({})
 
         ra, _ = l.resolve_all({
-            u"$schemas": [get_data("tests/EDAM.owl")],
+            u"$schemas": ["file://" + get_data("tests/EDAM.owl")],
             u"$namespaces": {u"edam": u"http://edamontology.org/"},
             u"edam:has_format": u"edam:format_1915"
         }, "")
 
         self.assertEqual({
-            u"$schemas": [get_data("tests/EDAM.owl")],
+            u"$schemas": ["file://" + get_data("tests/EDAM.owl")],
             u"$namespaces": {u"edam": u"http://edamontology.org/"},
             u'http://edamontology.org/has_format': u'http://edamontology.org/format_1915'
         }, ra)

setup.py

@@ -33,7 +33,9 @@
     'rdflib >= 4.1.0',
     'rdflib-jsonld >= 0.3.0',
     'mistune',
-    'typing']
+    'typing',
+    'CacheControl',
+    'lockfile']
 
 install_requires.append("avro")  # TODO: remove me once cwltool is
 # available in Debian Stable, Ubuntu 12.04 LTS

typeshed/2.7/cachecontrol/__init__.pyi

@@ -0,0 +1,10 @@
+# Stubs for cachecontrol (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+from .wrapper import CacheControl as CacheControl
+#from .adapter import CacheControlAdapter as CacheControlAdapter
+#from .controller import CacheController as CacheController
+
+__email__ = ...  # type: Any

typeshed/2.7/cachecontrol/adapter.pyi

@@ -0,0 +1,19 @@
+# Stubs for cachecontrol.adapter (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+from requests.adapters import HTTPAdapter
+from .controller import CacheController as CacheController
+from .cache import DictCache as DictCache
+from .filewrapper import CallbackFileWrapper as CallbackFileWrapper
+
+class CacheControlAdapter(HTTPAdapter):
+    invalidating_methods = ...  # type: Any
+    cache = ...  # type: Any
+    heuristic = ...  # type: Any
+    controller = ...  # type: Any
+    def __init__(self, cache=None, cache_etags=True, controller_class=None, serializer=None, heuristic=None, *args, **kw): ...
+    def send(self, request, **kw): ...
+    def build_response(self, request, response, from_cache=False): ...
+    def close(self): ...

typeshed/2.7/cachecontrol/cache.pyi

@@ -0,0 +1,19 @@
+# Stubs for cachecontrol.cache (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+
+class BaseCache:
+    def get(self, key): ...
+    def set(self, key, value): ...
+    def delete(self, key): ...
+    def close(self): ...
+
+class DictCache(BaseCache):
+    lock = ...  # type: Any
+    data = ...  # type: Any
+    def __init__(self, init_dict=None): ...
+    def get(self, key): ...
+    def set(self, key, value): ...
+    def delete(self, key): ...

typeshed/2.7/cachecontrol/caches/__init__.pyi

@@ -0,0 +1,9 @@
+# Stubs for cachecontrol.caches (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+from .file_cache import FileCache as FileCache
+#from .redis_cache import RedisCache as RedisCache
+
+notice = ...  # type: Any

typeshed/2.7/cachecontrol/caches/file_cache.pyi

@@ -0,0 +1,22 @@
+# Stubs for cachecontrol.caches.file_cache (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any, Callable
+from ..cache import BaseCache as BaseCache
+from ..controller import CacheController as CacheController
+
+class FileCache(BaseCache):
+    directory = ...  # type: str
+    forever = ...  # type: bool
+    filemode = ...  # type: str
+    dirmode = ...  # type: str
+    lock_class = ...  # type: Callable
+    def __init__(self, directory: str, forever=False, filemode=384, dirmode=448, use_dir_lock=None, lock_class=None) -> None: ...
+    @staticmethod
+    def encode(x): ...
+    def get(self, key): ...
+    def set(self, key, value): ...
+    def delete(self, key): ...
+
+def url_to_file_path(url, filecache): ...

typeshed/2.7/cachecontrol/compat.pyi

@@ -0,0 +1,7 @@
+# Stubs for cachecontrol.compat (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+
+text_type = ...  # type: Any

typeshed/2.7/cachecontrol/controller.pyi

@@ -0,0 +1,25 @@
+# Stubs for cachecontrol.controller (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any
+#from .cache import DictCache as DictCache
+#from .serialize import Serializer as Serializer
+
+logger = ...  # type: Any
+URI = ...  # type: Any
+
+def parse_uri(uri): ...
+
+class CacheController:
+    cache = ...  # type: Any
+    cache_etags = ...  # type: Any
+    serializer = ...  # type: Any
+    def __init__(self, cache=None, cache_etags=True, serializer=None): ...
+    @classmethod
+    def cache_url(cls, uri): ...
+    def parse_cache_control(self, headers): ...
+    def cached_request(self, request): ...
+    def conditional_headers(self, request): ...
+    def cache_response(self, request, response, body=None): ...
+    def update_cached_response(self, request, response): ...

typeshed/2.7/cachecontrol/filewrapper.pyi

@@ -0,0 +1,8 @@
+# Stubs for cachecontrol.filewrapper (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+class CallbackFileWrapper:
+    def __init__(self, fp, callback): ...
+    def __getattr__(self, name): ...
+    def read(self, amt=None): ...

typeshed/2.7/cachecontrol/serialize.pyi

@@ -0,0 +1,10 @@
+# Stubs for cachecontrol.serialize (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from .compat import HTTPResponse as HTTPResponse, pickle as pickle, text_type as text_type
+
+class Serializer:
+    def dumps(self, request, response, body=None): ...
+    def loads(self, request, data): ...
+    def prepare_response(self, request, cached): ...

typeshed/2.7/cachecontrol/wrapper.pyi

@@ -0,0 +1,14 @@
+# Stubs for cachecontrol.wrapper (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Union
+from .adapter import CacheControlAdapter as CacheControlAdapter
+from .cache import DictCache, BaseCache
+import requests
+
+def CacheControl(sess: requests.sessions.Session,
+                 cache: Union[DictCache, BaseCache] = None,
+                 cache_etags: bool = True,
+                 serializer=None,
+                 heuristic=None) -> requests.sessions.Session: ...

typeshed/2.7/requests/sessions.pyi

@@ -1,6 +1,6 @@
 # Stubs for requests.sessions (Python 3)
 
-from typing import Any, Union, MutableMapping
+from typing import Any, AnyStr, Union, MutableMapping
 from . import auth
 from . import compat
 from . import cookies
@@ -76,7 +76,7 @@ class Session(SessionRedirectMixin):
                 cookies=..., files=..., auth=..., timeout=..., allow_redirects=...,
                 proxies=..., hooks=..., stream=..., verify=..., cert=...,
                 json=...) -> Response: ...
-    def get(self, url: str, **kwargs) -> Response: ...
+    def get(self, url: AnyStr, **kwargs) -> Response: ...
     def options(self, url: str, **kwargs) -> Response: ...
     def head(self, url: str, **kwargs) -> Response: ...
     def post(self, url: str, data=..., json=..., **kwargs) -> Response: ...