Use __pydantic_serializer__ for serializing to JSON (#557)

* Use __pydantic_serializer__ for serializing to JSON

* Make to_json(by_alias=False) work

* Use presence of __pydantic_serializer__ instead of anything else to go down model serialization path

Author: dmontagu

pydantic_core/core_schema.py

@@ -223,8 +223,8 @@ def simple_ser_schema(type: ExpectedSerializationTypes) -> SimpleSerSchema:
     'timedelta',
     'url',
     'multi_host_url',
+    'pydantic_serializable',
     'dataclass',
-    'model',
     'enum',
     'path',
 ]

src/serializers/infer.rs

@@ -13,6 +13,7 @@ use serde::ser::{Error, Serialize, SerializeMap, SerializeSeq, Serializer};
 use crate::build_tools::{py_err, safe_repr};
 use crate::serializers::errors::SERIALIZATION_ERR_MARKER;
 use crate::serializers::filter::SchemaFilter;
+use crate::serializers::shared::PydanticSerializer;
 use crate::serializers::{shared::TypeSerializer, SchemaSerializer};
 use crate::url::{PyMultiHostUrl, PyUrl};
 
@@ -98,8 +99,8 @@ pub(crate) fn infer_to_python_known(
                 return serializer.serializer.to_python(value, include, exclude, extra);
             }
         }
-        // Fallback to dict serialization if `__pydantic_serializer__` is not set.else
-        // This is currently only relevant to non-pydantic dataclasses.
+        // Fallback to dict serialization if `__pydantic_serializer__` is not set.
+        // This currently only affects non-pydantic dataclasses.
         let dict = object_to_dict(value, is_model, extra)?;
         serialize_dict(dict)
     };
@@ -173,8 +174,8 @@ pub(crate) fn infer_to_python_known(
                 let py_url: PyMultiHostUrl = value.extract()?;
                 py_url.__str__().into_py(py)
             }
+            ObType::PydanticSerializable => serialize_with_serializer(value, true)?,
             ObType::Dataclass => serialize_with_serializer(value, false)?,
-            ObType::Model => serialize_with_serializer(value, true)?,
             ObType::Enum => {
                 let v = value.getattr(intern!(py, "value"))?;
                 infer_to_python(v, include, exclude, extra)?.into_py(py)
@@ -239,8 +240,8 @@ pub(crate) fn infer_to_python_known(
                 }
                 new_dict.into_py(py)
             }
-            ObType::Dataclass => serialize_dict(object_to_dict(value, false, extra)?)?,
-            ObType::Model => serialize_dict(object_to_dict(value, true, extra)?)?,
+            ObType::PydanticSerializable => serialize_with_serializer(value, true)?,
+            ObType::Dataclass => serialize_with_serializer(value, false)?,
             ObType::Generator => {
                 let iter = super::type_serializers::generator::SerializationIterator::new(
                     value.downcast()?,
@@ -388,6 +389,22 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
         }};
     }
 
+    macro_rules! serialize_with_serializer {
+        ($py_serializable:expr, $is_model:expr) => {{
+            if let Ok(py_serializer) = value.getattr(intern!($py_serializable.py(), "__pydantic_serializer__")) {
+                if let Ok(extracted_serializer) = py_serializer.extract::<SchemaSerializer>() {
+                    let pydantic_serializer =
+                        PydanticSerializer::new(value, &extracted_serializer.serializer, include, exclude, extra);
+                    return pydantic_serializer.serialize(serializer);
+                }
+            }
+            // Fallback to dict serialization if `__pydantic_serializer__` is not set.
+            // This currently only affects non-pydantic dataclasses.
+            let dict = object_to_dict(value, $is_model, extra).map_err(py_err_se_err)?;
+            serialize_dict!(dict)
+        }};
+    }
+
     let ser_result = match ob_type {
         ObType::None => serializer.serialize_none(),
         ObType::Int | ObType::IntSubclass => serialize!(i64),
@@ -442,8 +459,8 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
             let py_url: PyMultiHostUrl = value.extract().map_err(py_err_se_err)?;
             serializer.serialize_str(&py_url.__str__())
         }
-        ObType::Dataclass => serialize_dict!(object_to_dict(value, false, extra).map_err(py_err_se_err)?),
-        ObType::Model => serialize_dict!(object_to_dict(value, true, extra).map_err(py_err_se_err)?),
+        ObType::Dataclass => serialize_with_serializer!(value, false),
+        ObType::PydanticSerializable => serialize_with_serializer!(value, true),
         ObType::Enum => {
             let v = value.getattr(intern!(value.py(), "value")).map_err(py_err_se_err)?;
             infer_serialize(v, serializer, include, exclude, extra)
@@ -565,7 +582,7 @@ pub(crate) fn infer_json_key_known<'py>(ob_type: &ObType, key: &'py PyAny, extra
         ObType::List | ObType::Set | ObType::Frozenset | ObType::Dict | ObType::Generator => {
             py_err!(PyTypeError; "`{}` not valid as object key", ob_type)
         }
-        ObType::Dataclass | ObType::Model => {
+        ObType::Dataclass | ObType::PydanticSerializable => {
             // check that the instance is hashable
             key.hash()?;
             let key = key.str()?.to_string();

src/serializers/mod.rs

@@ -170,14 +170,16 @@ impl SchemaSerializer {
 
 #[allow(clippy::too_many_arguments)]
 #[pyfunction]
-#[pyo3(signature = (value, *, indent = None, include = None, exclude = None, exclude_none = false, round_trip = false,
-    timedelta_mode = None, bytes_mode = None, serialize_unknown = false, fallback = None))]
+#[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = true,
+    exclude_none = false, round_trip = false, timedelta_mode = None, bytes_mode = None,
+    serialize_unknown = false, fallback = None))]
 pub fn to_json(
     py: Python,
     value: &PyAny,
     indent: Option<usize>,
     include: Option<&PyAny>,
     exclude: Option<&PyAny>,
+    by_alias: bool,
     exclude_none: bool,
     round_trip: bool,
     timedelta_mode: Option<&str>,
@@ -189,7 +191,7 @@ pub fn to_json(
     let extra = state.extra(
         py,
         &SerMode::Json,
-        true,
+        by_alias,
         exclude_none,
         round_trip,
         serialize_unknown,

src/serializers/ob_type.rs

@@ -126,7 +126,7 @@ impl ObTypeLookup {
             ObType::Url => self.url == ob_type,
             ObType::MultiHostUrl => self.multi_host_url == ob_type,
             ObType::Dataclass => is_dataclass(op_value),
-            ObType::Model => is_pydantic_model(op_value),
+            ObType::PydanticSerializable => is_pydantic_serializable(op_value),
             ObType::Enum => self.enum_type == ob_type,
             ObType::Generator => self.generator == ob_type,
             ObType::Path => self.path == ob_type,
@@ -208,10 +208,10 @@ impl ObTypeLookup {
             ObType::Url
         } else if ob_type == self.multi_host_url {
             ObType::MultiHostUrl
+        } else if is_pydantic_serializable(op_value) {
+            ObType::PydanticSerializable
         } else if is_dataclass(op_value) {
             ObType::Dataclass
-        } else if is_pydantic_model(op_value) {
-            ObType::Model
         } else if self.is_enum(op_value, type_ptr) {
             ObType::Enum
         } else if ob_type == self.generator || is_generator(op_value) {
@@ -256,10 +256,10 @@ fn is_dataclass(op_value: Option<&PyAny>) -> bool {
     }
 }
 
-fn is_pydantic_model(op_value: Option<&PyAny>) -> bool {
+fn is_pydantic_serializable(op_value: Option<&PyAny>) -> bool {
     if let Some(value) = op_value {
         value
-            .hasattr(intern!(value.py(), "__pydantic_validator__"))
+            .hasattr(intern!(value.py(), "__pydantic_serializer__"))
             .unwrap_or(false)
     } else {
         false
@@ -305,9 +305,10 @@ pub enum ObType {
     // types from this package
     Url,
     MultiHostUrl,
-    // dataclasses and pydantic models
+    // anything with __pydantic_serializer__, including BaseModel and pydantic dataclasses
+    PydanticSerializable,
+    // vanilla dataclasses
     Dataclass,
-    Model,
     // enum type
     Enum,
     // generator type

src/serializers/type_serializers/model.rs

@@ -83,7 +83,7 @@ impl TypeSerializer for ModelSerializer {
 
     fn json_key<'py>(&self, key: &'py PyAny, extra: &Extra) -> PyResult<Cow<'py, str>> {
         if self.allow_value(key, extra)? {
-            infer_json_key_known(&ObType::Model, key, extra)
+            infer_json_key_known(&ObType::PydanticSerializable, key, extra)
         } else {
             extra.warnings.on_fallback_py(&self.name, key, extra)?;
             infer_json_key(key, extra)

tests/serializers/test_any.py

@@ -11,9 +11,10 @@
 import pytest
 from dirty_equals import HasRepr, IsList
 
-from pydantic_core import PydanticSerializationError, SchemaSerializer, core_schema, to_json
+from pydantic_core import PydanticSerializationError, SchemaSerializer, SchemaValidator, core_schema, to_json
 
 from ..conftest import plain_repr
+from .test_dataclasses import IsStrictDict, on_pypy
 from .test_list_tuple import as_list, as_tuple
 
 
@@ -35,7 +36,7 @@ class MyDataclass:
 
 
 class MyModel:
-    __pydantic_validator__ = 42
+    __pydantic_serializer__ = 42
 
     def __init__(self, **kwargs):
         for key, value in kwargs.items():
@@ -242,7 +243,7 @@ def test_include_dict(any_serializer):
 
 
 class FieldsSetModel:
-    __pydantic_validator__ = 42
+    __pydantic_serializer__ = 42
     __slots__ = '__dict__', '__pydantic_fields_set__'
 
     def __init__(self, **kwargs):
@@ -412,3 +413,63 @@ def test_encoding(any_serializer, gen_input, kwargs, expected_json):
     assert to_json(gen_input(), **kwargs) == expected_json
     if not kwargs:
         assert any_serializer.to_python(gen_input(), mode='json') == json.loads(expected_json)
+
+
+def test_any_dataclass():
+    @dataclasses.dataclass
+    class Foo:
+        a: str
+        b: bytes
+
+    # Build a schema that does not include the field 'b', to test that it is not serialized
+    schema = core_schema.dataclass_schema(
+        Foo,
+        core_schema.dataclass_args_schema(
+            'Foo', [core_schema.dataclass_field(name='a', schema=core_schema.str_schema())]
+        ),
+    )
+    Foo.__pydantic_serializer__ = SchemaSerializer(schema)
+
+    s = SchemaSerializer(core_schema.any_schema())
+    assert s.to_python(Foo(a='hello', b=b'more')) == IsStrictDict(a='hello')
+    assert s.to_python(Foo(a='hello', b=b'more'), mode='json') == IsStrictDict(a='hello')
+    j = s.to_json(Foo(a='hello', b=b'more'))
+
+    if on_pypy:
+        assert json.loads(j) == {'a': 'hello'}
+    else:
+        assert j == b'{"a":"hello"}'
+
+    assert s.to_python(Foo(a='hello', b=b'more'), exclude={'a'}) == IsStrictDict()
+
+
+def test_any_model():
+    class Foo:
+        a: str
+        b: bytes
+
+        def __init__(self, a: str, b: bytes):
+            self.a = a
+            self.b = b
+
+    # Build a schema that does not include the field 'b', to test that it is not serialized
+    schema = core_schema.dataclass_schema(
+        Foo,
+        core_schema.dataclass_args_schema(
+            'Foo', [core_schema.dataclass_field(name='a', schema=core_schema.str_schema())]
+        ),
+    )
+    Foo.__pydantic_validator__ = SchemaValidator(schema)
+    Foo.__pydantic_serializer__ = SchemaSerializer(schema)
+
+    s = SchemaSerializer(core_schema.any_schema())
+    assert s.to_python(Foo(a='hello', b=b'more')) == IsStrictDict(a='hello')
+    assert s.to_python(Foo(a='hello', b=b'more'), mode='json') == IsStrictDict(a='hello')
+    j = s.to_json(Foo(a='hello', b=b'more'))
+
+    if on_pypy:
+        assert json.loads(j) == {'a': 'hello'}
+    else:
+        assert j == b'{"a":"hello"}'
+
+    assert s.to_python(Foo(a='hello', b=b'more'), exclude={'a'}) == IsStrictDict()

tests/test_json.py

@@ -262,13 +262,8 @@ def __init__(self, my_foo: int, my_bar: str):
     instance = Foobar(my_foo=1, my_bar='a')
     assert to_jsonable_python(instance) == {'myFoo': 1, 'myBar': 'a'}
     assert to_jsonable_python(instance, by_alias=False) == {'my_foo': 1, 'my_bar': 'a'}
-
-    # Just including this to document the behavior. Note that trying to get `to_json` to respect aliases
-    # by putting the by_alias field into the `extra` in `to_json` won't work because it always uses an
-    # AnySchema for serialization. If you want any python-introspection to happen during the json serialization
-    # process for the purpose of respecting objects' own serialization schemas, you need to call
-    # to_jsonable_python and serialize the result yourself.
-    assert to_json(instance) == b'{"my_foo":1,"my_bar":"a"}'
+    assert to_json(instance) == b'{"myFoo":1,"myBar":"a"}'
+    assert to_json(instance, by_alias=False) == b'{"my_foo":1,"my_bar":"a"}'
 
 
 def test_cycle_same():