Skip to content

Commit 68453df

Browse files
davidhewittdavidhewitt
authored
fix python GC traversal for validators and serializers (#787)
1 parent 33fea1e commit 68453df

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+629
-51
lines changed

‎src/lib.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ use pyo3::{prelude::*, sync::GILOnceCell};
1010
#[global_allocator]
1111
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
1212

13+
// parse this first to get access to the contained macro
14+
#[macro_use]
15+
mod py_gc;
16+
1317
mod argument_markers;
1418
mod build_tools;
1519
mod definitions;

‎src/py_gc.rs

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
use ahash::AHashMap;
2+
use enum_dispatch::enum_dispatch;
3+
use pyo3::{AsPyPointer, Py, PyTraverseError, PyVisit};
4+
5+
/// Trait implemented by types which can be traversed by the Python GC.
6+
#[enum_dispatch]
7+
pub trait PyGcTraverse {
8+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError>;
9+
}
10+
11+
impl<T> PyGcTraverse for Py<T>
12+
where
13+
Py<T>: AsPyPointer,
14+
{
15+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
16+
visit.call(self)
17+
}
18+
}
19+
20+
impl<T: PyGcTraverse> PyGcTraverse for Vec<T> {
21+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
22+
for item in self {
23+
item.py_gc_traverse(visit)?;
24+
}
25+
Ok(())
26+
}
27+
}
28+
29+
impl<T: PyGcTraverse> PyGcTraverse for AHashMap<String, T> {
30+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
31+
for item in self.values() {
32+
item.py_gc_traverse(visit)?;
33+
}
34+
Ok(())
35+
}
36+
}
37+
38+
impl<T: PyGcTraverse> PyGcTraverse for Box<T> {
39+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
40+
T::py_gc_traverse(self, visit)
41+
}
42+
}
43+
44+
impl<T: PyGcTraverse> PyGcTraverse for Option<T> {
45+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
46+
match self {
47+
Some(item) => T::py_gc_traverse(item, visit),
48+
None => Ok(()),
49+
}
50+
}
51+
}
52+
53+
/// A crude alternative to a "derive" macro to help with building PyGcTraverse implementations
54+
macro_rules! impl_py_gc_traverse {
55+
($name:ty { }) => {
56+
impl crate::py_gc::PyGcTraverse for $name {
57+
fn py_gc_traverse(&self, _visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> {
58+
Ok(())
59+
}
60+
}
61+
};
62+
($name:ty { $($fields:ident),* }) => {
63+
impl crate::py_gc::PyGcTraverse for $name {
64+
fn py_gc_traverse(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> {
65+
$(self.$fields.py_gc_traverse(visit)?;)*
66+
Ok(())
67+
}
68+
}
69+
};
70+
}

‎src/serializers/computed_fields.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
use pyo3::intern;
21
use pyo3::prelude::*;
32
use pyo3::types::{PyDict, PyList, PyString};
3+
use pyo3::{intern, PyTraverseError, PyVisit};
44
use serde::ser::SerializeMap;
55
use serde::Serialize;
66

77
use crate::build_tools::py_schema_error_type;
88
use crate::definitions::DefinitionsBuilder;
9+
use crate::py_gc::PyGcTraverse;
910
use crate::serializers::filter::SchemaFilter;
1011
use crate::serializers::shared::{BuildSerializer, CombinedSerializer, PydanticSerializer, TypeSerializer};
1112
use crate::tools::SchemaDict;
@@ -156,6 +157,16 @@ pub(crate) struct ComputedFieldSerializer<'py> {
156157
extra: &'py Extra<'py>,
157158
}
158159

160+
impl_py_gc_traverse!(ComputedField { serializer });
161+
162+
impl PyGcTraverse for ComputedFields {
163+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
164+
self.0.py_gc_traverse(visit)
165+
}
166+
}
167+
168+
impl_py_gc_traverse!(ComputedFieldSerializer<'_> { computed_field });
169+
159170
impl<'py> Serialize for ComputedFieldSerializer<'py> {
160171
fn serialize<S: serde::ser::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
161172
let py = self.model.py();

‎src/serializers/fields.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ pub(super) struct SerField {
2828
pub required: bool,
2929
}
3030

31+
impl_py_gc_traverse!(SerField { serializer });
32+
3133
impl SerField {
3234
pub fn new(
3335
py: Python,
@@ -142,6 +144,11 @@ macro_rules! option_length {
142144
};
143145
}
144146

147+
impl_py_gc_traverse!(GeneralFieldsSerializer {
148+
fields,
149+
computed_fields
150+
});
151+
145152
impl TypeSerializer for GeneralFieldsSerializer {
146153
fn to_python(
147154
&self,

‎src/serializers/mod.rs

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use pyo3::types::{PyBytes, PyDict};
66
use pyo3::{PyTraverseError, PyVisit};
77

88
use crate::definitions::DefinitionsBuilder;
9+
use crate::py_gc::PyGcTraverse;
910
use crate::validators::SelfValidator;
1011

1112
use config::SerializationConfig;
@@ -191,13 +192,6 @@ impl SchemaSerializer {
191192
}
192193
Ok(())
193194
}
194-
195-
fn __clear__(&mut self) {
196-
self.serializer.py_gc_clear();
197-
for slot in &mut self.definitions {
198-
slot.py_gc_clear();
199-
}
200-
}
201195
}
202196

203197
#[allow(clippy::too_many_arguments)]

‎src/serializers/shared.rs

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use serde_json::ser::PrettyFormatter;
1414
use crate::build_tools::py_schema_err;
1515
use crate::build_tools::py_schema_error_type;
1616
use crate::definitions::DefinitionsBuilder;
17+
use crate::py_gc::PyGcTraverse;
1718
use crate::tools::{py_err, SchemaDict};
1819

1920
use super::errors::se_err_py_err;
@@ -215,12 +216,50 @@ impl BuildSerializer for CombinedSerializer {
215216
}
216217
}
217218

219+
// Implemented by hand because `enum_dispatch` fails with a proc macro compile error =/
220+
impl PyGcTraverse for CombinedSerializer {
221+
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
222+
match self {
223+
CombinedSerializer::Function(inner) => inner.py_gc_traverse(visit),
224+
CombinedSerializer::FunctionWrap(inner) => inner.py_gc_traverse(visit),
225+
CombinedSerializer::Fields(inner) => inner.py_gc_traverse(visit),
226+
CombinedSerializer::None(inner) => inner.py_gc_traverse(visit),
227+
CombinedSerializer::Nullable(inner) => inner.py_gc_traverse(visit),
228+
CombinedSerializer::Int(inner) => inner.py_gc_traverse(visit),
229+
CombinedSerializer::Bool(inner) => inner.py_gc_traverse(visit),
230+
CombinedSerializer::Float(inner) => inner.py_gc_traverse(visit),
231+
CombinedSerializer::Str(inner) => inner.py_gc_traverse(visit),
232+
CombinedSerializer::Bytes(inner) => inner.py_gc_traverse(visit),
233+
CombinedSerializer::Datetime(inner) => inner.py_gc_traverse(visit),
234+
CombinedSerializer::TimeDelta(inner) => inner.py_gc_traverse(visit),
235+
CombinedSerializer::Date(inner) => inner.py_gc_traverse(visit),
236+
CombinedSerializer::Time(inner) => inner.py_gc_traverse(visit),
237+
CombinedSerializer::List(inner) => inner.py_gc_traverse(visit),
238+
CombinedSerializer::Set(inner) => inner.py_gc_traverse(visit),
239+
CombinedSerializer::FrozenSet(inner) => inner.py_gc_traverse(visit),
240+
CombinedSerializer::Generator(inner) => inner.py_gc_traverse(visit),
241+
CombinedSerializer::Dict(inner) => inner.py_gc_traverse(visit),
242+
CombinedSerializer::Model(inner) => inner.py_gc_traverse(visit),
243+
CombinedSerializer::Dataclass(inner) => inner.py_gc_traverse(visit),
244+
CombinedSerializer::Url(inner) => inner.py_gc_traverse(visit),
245+
CombinedSerializer::MultiHostUrl(inner) => inner.py_gc_traverse(visit),
246+
CombinedSerializer::Any(inner) => inner.py_gc_traverse(visit),
247+
CombinedSerializer::Format(inner) => inner.py_gc_traverse(visit),
248+
CombinedSerializer::ToString(inner) => inner.py_gc_traverse(visit),
249+
CombinedSerializer::WithDefault(inner) => inner.py_gc_traverse(visit),
250+
CombinedSerializer::Json(inner) => inner.py_gc_traverse(visit),
251+
CombinedSerializer::JsonOrPython(inner) => inner.py_gc_traverse(visit),
252+
CombinedSerializer::Union(inner) => inner.py_gc_traverse(visit),
253+
CombinedSerializer::Literal(inner) => inner.py_gc_traverse(visit),
254+
CombinedSerializer::Recursive(inner) => inner.py_gc_traverse(visit),
255+
CombinedSerializer::TuplePositional(inner) => inner.py_gc_traverse(visit),
256+
CombinedSerializer::TupleVariable(inner) => inner.py_gc_traverse(visit),
257+
}
258+
}
259+
}
260+
218261
#[enum_dispatch(CombinedSerializer)]
219262
pub(crate) trait TypeSerializer: Send + Sync + Clone + Debug {
220-
fn py_gc_traverse(&self, _visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
221-
Ok(())
222-
}
223-
fn py_gc_clear(&mut self) {}
224263
fn to_python(
225264
&self,
226265
value: &PyAny,

‎src/serializers/type_serializers/any.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ impl BuildSerializer for AnySerializer {
2626
}
2727
}
2828

29+
impl_py_gc_traverse!(AnySerializer {});
30+
2931
impl TypeSerializer for AnySerializer {
3032
fn to_python(
3133
&self,

‎src/serializers/type_serializers/bytes.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ impl BuildSerializer for BytesSerializer {
2525
}
2626
}
2727

28+
impl_py_gc_traverse!(BytesSerializer {});
29+
2830
impl TypeSerializer for BytesSerializer {
2931
fn to_python(
3032
&self,

‎src/serializers/type_serializers/dataclass.rs

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1+
use pyo3::intern;
12
use pyo3::prelude::*;
23
use pyo3::types::{PyDict, PyList, PyString, PyType};
3-
use pyo3::{intern, PyTraverseError, PyVisit};
44
use std::borrow::Cow;
55

66
use ahash::AHashMap;
@@ -121,13 +121,9 @@ impl DataclassSerializer {
121121
}
122122
}
123123

124-
impl TypeSerializer for DataclassSerializer {
125-
fn py_gc_traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
126-
visit.call(&self.class)?;
127-
self.serializer.py_gc_traverse(visit)?;
128-
Ok(())
129-
}
124+
impl_py_gc_traverse!(DataclassSerializer { class, serializer });
130125

126+
impl TypeSerializer for DataclassSerializer {
131127
fn to_python(
132128
&self,
133129
value: &PyAny,

‎src/serializers/type_serializers/datetime_etc.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ macro_rules! build_serializer {
4040
}
4141
}
4242

43+
impl_py_gc_traverse!($struct_name {});
44+
4345
impl TypeSerializer for $struct_name {
4446
fn to_python(
4547
&self,

‎src/serializers/type_serializers/definitions.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use pyo3::prelude::*;
55
use pyo3::types::{PyDict, PyList};
66

77
use crate::definitions::DefinitionsBuilder;
8+
89
use crate::tools::SchemaDict;
910

1011
use super::{py_err_se_err, BuildSerializer, CombinedSerializer, Extra, TypeSerializer};
@@ -59,6 +60,8 @@ impl BuildSerializer for DefinitionRefSerializer {
5960
}
6061
}
6162

63+
impl_py_gc_traverse!(DefinitionRefSerializer {});
64+
6265
impl TypeSerializer for DefinitionRefSerializer {
6366
fn to_python(
6467
&self,

‎src/serializers/type_serializers/dict.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ impl BuildSerializer for DictSerializer {
6565
}
6666
}
6767

68+
impl_py_gc_traverse!(DictSerializer {
69+
key_serializer,
70+
value_serializer
71+
});
72+
6873
impl TypeSerializer for DictSerializer {
6974
fn to_python(
7075
&self,

‎src/serializers/type_serializers/format.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ impl FormatSerializer {
104104
}
105105
}
106106

107+
impl_py_gc_traverse!(FormatSerializer { format_func });
108+
107109
impl TypeSerializer for FormatSerializer {
108110
fn to_python(
109111
&self,
@@ -175,6 +177,8 @@ impl BuildSerializer for ToStringSerializer {
175177
}
176178
}
177179

180+
impl_py_gc_traverse!(ToStringSerializer {});
181+
178182
impl TypeSerializer for ToStringSerializer {
179183
fn to_python(
180184
&self,

‎src/serializers/type_serializers/function.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,12 @@ macro_rules! function_type_serializer {
269269
};
270270
}
271271

272+
impl_py_gc_traverse!(FunctionPlainSerializer {
273+
func,
274+
return_serializer,
275+
fallback_serializer
276+
});
277+
272278
function_type_serializer!(FunctionPlainSerializer);
273279

274280
fn copy_outer_schema(schema: &PyDict) -> PyResult<&PyDict> {
@@ -399,6 +405,12 @@ impl FunctionWrapSerializer {
399405
}
400406
}
401407

408+
impl_py_gc_traverse!(FunctionWrapSerializer {
409+
serializer,
410+
func,
411+
return_serializer
412+
});
413+
402414
function_type_serializer!(FunctionWrapSerializer);
403415

404416
#[pyclass(module = "pydantic_core._pydantic_core")]

‎src/serializers/type_serializers/generator.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ impl BuildSerializer for GeneratorSerializer {
4242
}
4343
}
4444

45+
impl_py_gc_traverse!(GeneratorSerializer { item_serializer });
46+
4547
impl TypeSerializer for GeneratorSerializer {
4648
fn to_python(
4749
&self,

‎src/serializers/type_serializers/json.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ impl BuildSerializer for JsonSerializer {
4242
}
4343
}
4444

45+
impl_py_gc_traverse!(JsonSerializer { serializer });
46+
4547
impl TypeSerializer for JsonSerializer {
4648
fn to_python(
4749
&self,

‎src/serializers/type_serializers/json_or_python.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ impl BuildSerializer for JsonOrPythonSerializer {
4545
}
4646
}
4747

48+
impl_py_gc_traverse!(JsonOrPythonSerializer { json, python });
49+
4850
impl TypeSerializer for JsonOrPythonSerializer {
4951
fn to_python(
5052
&self,

‎src/serializers/type_serializers/list.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ impl BuildSerializer for ListSerializer {
4545
}
4646
}
4747

48+
impl_py_gc_traverse!(ListSerializer { item_serializer });
49+
4850
impl TypeSerializer for ListSerializer {
4951
fn to_python(
5052
&self,

‎src/serializers/type_serializers/literal.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ impl LiteralSerializer {
106106
}
107107
}
108108

109+
impl_py_gc_traverse!(LiteralSerializer { expected_py });
110+
109111
impl TypeSerializer for LiteralSerializer {
110112
fn to_python(
111113
&self,

0 commit comments

Comments
 (0)