Merge pull request #255 from ziirish/master

add: Wildcard fields

Author: noirbizarre

doc/marshalling.rst

@@ -214,6 +214,63 @@ You can also unmarshal fields as lists ::
     >>> json.dumps(marshal(data, resource_fields))
     >>> '{"first_names": ["Emile", "Raoul"], "name": "Bougnazal"}'
 
+.. _wildcard-field:
+
+Wildcard Field
+--------------
+
+If you don't know the name(s) of the field(s) you want to unmarshall, you can
+use :class:`~fields.Wildcard` ::
+
+    >>> from flask_restplus import fields, marshal
+    >>> import json
+    >>>
+    >>> wild = fields.Wildcard(fields.String)
+    >>> wildcard_fields = {'*': wild}
+    >>> data = {'John': 12, 'bob': 42, 'Jane': '68'}
+    >>> json.dumps(marshal(data, wildcard_fields))
+    >>> '{"Jane": "68", "bob": "42", "John": "12"}'
+
+The name you give to your :class:`~fields.Wildcard` acts as a real glob as
+shown bellow ::
+
+    >>> from flask_restplus import fields, marshal
+    >>> import json
+    >>>
+    >>> wild = fields.Wildcard(fields.String)
+    >>> wildcard_fields = {'j*': wild}
+    >>> data = {'John': 12, 'bob': 42, 'Jane': '68'}
+    >>> json.dumps(marshal(data, wildcard_fields))
+    >>> '{"Jane": "68", "John": "12"}'
+
+.. note ::
+    It is important you define your :class:`~fields.Wildcard` **outside** your
+    model (ie. you **cannot** use it like this:
+    ``res_fields = {'*': fields.Wildcard(fields.String)}``) because it has to be
+    stateful to keep a track of what fields it has already treated.
+
+.. note ::
+    The glob is not a regex, it can only treat simple wildcards like '*' or '?'.
+
+In order to avoid unexpected behavior, when mixing :class:`~fields.Wildcard`
+with other fields, you may want to use an ``OrderedDict`` and use the
+:class:`~fields.Wildcard` as the last field ::
+
+    >>> from flask_restplus import fields, marshal
+    >>> from collections import OrderedDict
+    >>> import json
+    >>>
+    >>> wild = fields.Wildcard(fields.Integer)
+    >>> mod = OrderedDict()
+    >>> mod['zoro'] = fields.String
+    >>> mod['*'] = wild
+    >>> # you can use it in api.model like this:
+    >>> # some_fields = api.model('MyModel', mod)
+    >>>
+    >>> data = {'John': 12, 'bob': 42, 'Jane': '68', 'zoro': 72}
+    >>> json.dumps(marshal(data, mod))
+    >>> '{"zoro": "72", "Jane": 68, "bob": 42, "John": 12}'
+
 .. _nested-field:
 
 Nested Field

flask_restplus/fields.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import re
+import fnmatch
+import inspect
+
 from calendar import timegm
 from datetime import date, datetime
 from decimal import Decimal, ROUND_HALF_EVEN
@@ -20,7 +24,7 @@
 
 __all__ = ('Raw', 'String', 'FormattedString', 'Url', 'DateTime', 'Date',
            'Boolean', 'Integer', 'Float', 'Arbitrary', 'Fixed',
-           'Nested', 'List', 'ClassName', 'Polymorph',
+           'Nested', 'List', 'ClassName', 'Polymorph', 'Wildcard',
            'StringMixin', 'MinMaxMixin', 'NumberMixin', 'MarshallingError')
 
 
@@ -702,3 +706,102 @@ def clone(self, mask=None):
 
         data['mask'] = mask
         return Polymorph(mapping, **data)
+
+
+class Wildcard(Raw):
+    '''
+    Field for marshalling list of "unkown" fields.
+
+    :param cls_or_instance: The field type the list will contain.
+    '''
+    exclude = set()
+    # cache the flat object
+    _flat = None
+    _obj = None
+    _cache = set()
+    _last = None
+
+    def __init__(self, cls_or_instance, **kwargs):
+        super(Wildcard, self).__init__(**kwargs)
+        error_msg = 'The type of the wildcard elements must be a subclass of fields.Raw'
+        if isinstance(cls_or_instance, type):
+            if not issubclass(cls_or_instance, Raw):
+                raise MarshallingError(error_msg)
+            self.container = cls_or_instance()
+        else:
+            if not isinstance(cls_or_instance, Raw):
+                raise MarshallingError(error_msg)
+            self.container = cls_or_instance
+
+    def _flatten(self, obj):
+        if obj is None:
+            return None
+        if obj == self._obj and self._flat is not None:
+            return self._flat
+        if isinstance(obj, dict):
+            self._flat = [x for x in iteritems(obj)]
+        else:
+
+            def __match_attributes(attribute):
+                attr_name, attr_obj = attribute
+                if inspect.isroutine(attr_obj) or \
+                        (attr_name.startswith('__') and attr_name.endswith('__')):
+                    return False
+                return True
+
+            attributes = inspect.getmembers(obj)
+            self._flat = [x for x in attributes if __match_attributes(x)]
+
+        self._cache = set()
+        self._obj = obj
+        return self._flat
+
+    @property
+    def key(self):
+        return self._last
+
+    def reset(self):
+        self.exclude = set()
+        self._flat = None
+        self._obj = None
+        self._cache = set()
+        self._last = None
+
+    def output(self, key, obj, ordered=False):
+        value = None
+        reg = fnmatch.translate(key)
+
+        if self._flatten(obj):
+            while True:
+                try:
+                    # we are using pop() so that we don't
+                    # loop over the whole object every time dropping the
+                    # complexity to O(n)
+                    (objkey, val) = self._flat.pop()
+                    if objkey not in self._cache and \
+                            objkey not in self.exclude and \
+                            re.match(reg, objkey, re.IGNORECASE):
+                        value = val
+                        self._cache.add(objkey)
+                        self._last = objkey
+                        break
+                except IndexError:
+                    break
+
+        if value is None:
+            if self.default is not None:
+                return self.container.format(self.default)
+            return None
+
+        return self.container.format(value)
+
+    def schema(self):
+        schema = super(Wildcard, self).schema()
+        schema['type'] = 'object'
+        schema['additionalProperties'] = self.container.__schema__
+        return schema
+
+    def clone(self):
+        kwargs = self.__dict__.copy()
+        model = kwargs.pop('container')
+        return self.__class__(model, **kwargs)

flask_restplus/marshalling.py

@@ -11,6 +11,12 @@
 from .utils import unpack
 
 
+def make(cls):
+    if isinstance(cls, type):
+        return cls()
+    return cls
+
+
 def marshal(data, fields, envelope=None, skip_none=False, mask=None, ordered=False):
     """Takes raw data (in the form of a dict, list, object) and a dict of
     fields to output and filters the data based on those fields.
@@ -49,11 +55,103 @@ def marshal(data, fields, envelope=None, skip_none=False, mask=None, ordered=Fal
     OrderedDict([('a', 100)])
 
     """
+    out, has_wildcards = _marshal(data, fields, envelope, skip_none, mask, ordered)
+
+    if has_wildcards:
+        # ugly local import to avoid dependency loop
+        from .fields import Wildcard
+
+        items = []
+        keys = []
+        for dkey, val in fields.items():
+            key = dkey
+            if isinstance(val, dict):
+                value = marshal(data, val, skip_none=skip_none, ordered=ordered)
+            else:
+                field = make(val)
+                is_wildcard = isinstance(field, Wildcard)
+                # exclude already parsed keys from the wildcard
+                if is_wildcard:
+                    field.reset()
+                    if keys:
+                        field.exclude |= set(keys)
+                        keys = []
+                value = field.output(dkey, data)
+                if is_wildcard:
+
+                    def _append(k, v):
+                        if skip_none and (v is None or v == OrderedDict() or v == {}):
+                            return
+                        items.append((k, v))
+
+                    key = field.key or dkey
+                    _append(key, value)
+                    while True:
+                        value = field.output(dkey, data, ordered=ordered)
+                        if value is None or \
+                                value == field.container.format(field.default):
+                            break
+                        key = field.key
+                        _append(key, value)
+                    continue
+
+            keys.append(key)
+            if skip_none and (value is None or value == OrderedDict() or value == {}):
+                continue
+            items.append((key, value))
+
+        items = tuple(items)
+
+        out = OrderedDict(items) if ordered else dict(items)
+
+        if envelope:
+            out = OrderedDict([(envelope, out)]) if ordered else {envelope: out}
+
+        return out
+
+    return out
+
+
+def _marshal(data, fields, envelope=None, skip_none=False, mask=None, ordered=False):
+    """Takes raw data (in the form of a dict, list, object) and a dict of
+    fields to output and filters the data based on those fields.
+
+    :param data: the actual object(s) from which the fields are taken from
+    :param fields: a dict of whose keys will make up the final serialized
+                   response output
+    :param envelope: optional key that will be used to envelop the serialized
+                     response
+    :param bool skip_none: optional key will be used to eliminate fields
+                           which value is None or the field's key not
+                           exist in data
+    :param bool ordered: Wether or not to preserve order
+
+
+    >>> from flask_restplus import fields, marshal
+    >>> data = { 'a': 100, 'b': 'foo', 'c': None }
+    >>> mfields = { 'a': fields.Raw, 'c': fields.Raw, 'd': fields.Raw }
+
+    >>> marshal(data, mfields)
+    {'a': 100, 'c': None, 'd': None}
+
+    >>> marshal(data, mfields, envelope='data')
+    {'data': {'a': 100, 'c': None, 'd': None}}
+
+    >>> marshal(data, mfields, skip_none=True)
+    {'a': 100}
+
+    >>> marshal(data, mfields, ordered=True)
+    OrderedDict([('a', 100), ('c', None), ('d', None)])
 
-    def make(cls):
-        if isinstance(cls, type):
-            return cls()
-        return cls
+    >>> marshal(data, mfields, envelope='data', ordered=True)
+    OrderedDict([('data', OrderedDict([('a', 100), ('c', None), ('d', None)]))])
+
+    >>> marshal(data, mfields, skip_none=True, ordered=True)
+    OrderedDict([('a', 100)])
+
+    """
+    # ugly local import to avoid dependency loop
+    from .fields import Wildcard
 
     mask = mask or getattr(fields, '__mask__', None)
     fields = getattr(fields, 'resolved', fields)
@@ -64,12 +162,21 @@ def make(cls):
         out = [marshal(d, fields, skip_none=skip_none, ordered=ordered) for d in data]
         if envelope:
             out = OrderedDict([(envelope, out)]) if ordered else {envelope: out}
-        return out
+        return out, False
+
+    has_wildcards = {'present': False}
+
+    def __format_field(key, val):
+        field = make(val)
+        if isinstance(field, Wildcard):
+            has_wildcards['present'] = True
+        value = field.output(key, data, ordered=ordered)
+        return (key, value)
 
     items = (
-        (k, marshal(data, v, skip_none=skip_none, ordered=ordered)
+        (k, marshal(data, v, skip_none=skip_none, ordered=ordered))
         if isinstance(v, dict)
-        else make(v).output(k, data, ordered=ordered))
+        else __format_field(k, v)
         for k, v in iteritems(fields)
     )
 
@@ -82,7 +189,7 @@ def make(cls):
     if envelope:
         out = OrderedDict([(envelope, out)]) if ordered else {envelope: out}
 
-    return out
+    return out, has_wildcards['present']
 
 
 class marshal_with(object):