eager length checks and accept arbitrary iterables for list/sets/tuples (#610)

Co-authored-by: Adrian Garcia Badaracco <[email protected]>

Author: samuelcolvin

pydantic_core/core_schema.py

@@ -1184,7 +1184,6 @@ class ListSchema(TypedDict, total=False):
     min_length: int
     max_length: int
     strict: bool
-    allow_any_iter: bool
     ref: str
     metadata: Any
     serialization: IncExSeqOrElseSerSchema
@@ -1196,7 +1195,6 @@ def list_schema(
     min_length: int | None = None,
     max_length: int | None = None,
     strict: bool | None = None,
-    allow_any_iter: bool | None = None,
     ref: str | None = None,
     metadata: Any = None,
     serialization: IncExSeqOrElseSerSchema | None = None,
@@ -1217,7 +1215,6 @@ def list_schema(
         min_length: The value must be a list with at least this many items
         max_length: The value must be a list with at most this many items
         strict: The value must be a list with exactly this many items
-        allow_any_iter: Whether the value can be any iterable
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
         serialization: Custom serialization schema
@@ -1228,7 +1225,6 @@ def list_schema(
         min_length=min_length,
         max_length=max_length,
         strict=strict,
-        allow_any_iter=allow_any_iter,
         ref=ref,
         metadata=metadata,
         serialization=serialization,
@@ -1349,7 +1345,6 @@ class SetSchema(TypedDict, total=False):
     items_schema: CoreSchema
     min_length: int
     max_length: int
-    generator_max_length: int
     strict: bool
     ref: str
     metadata: Any
@@ -1361,7 +1356,6 @@ def set_schema(
     *,
     min_length: int | None = None,
     max_length: int | None = None,
-    generator_max_length: int | None = None,
     strict: bool | None = None,
     ref: str | None = None,
     metadata: Any = None,
@@ -1384,9 +1378,6 @@ def set_schema(
         items_schema: The value must be a set with items that match this schema
         min_length: The value must be a set with at least this many items
         max_length: The value must be a set with at most this many items
-        generator_max_length: At most this many items will be read from a generator before failing validation
-            This is important because generators can be infinite, and even with a `max_length` on the set,
-            an infinite generator could run forever without producing more than `max_length` distinct items.
         strict: The value must be a set with exactly this many items
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
@@ -1397,7 +1388,6 @@ def set_schema(
         items_schema=items_schema,
         min_length=min_length,
         max_length=max_length,
-        generator_max_length=generator_max_length,
         strict=strict,
         ref=ref,
         metadata=metadata,
@@ -1410,7 +1400,6 @@ class FrozenSetSchema(TypedDict, total=False):
     items_schema: CoreSchema
     min_length: int
     max_length: int
-    generator_max_length: int
     strict: bool
     ref: str
     metadata: Any
@@ -1422,7 +1411,6 @@ def frozenset_schema(
     *,
     min_length: int | None = None,
     max_length: int | None = None,
-    generator_max_length: int | None = None,
     strict: bool | None = None,
     ref: str | None = None,
     metadata: Any = None,
@@ -1445,7 +1433,6 @@ def frozenset_schema(
         items_schema: The value must be a frozenset with items that match this schema
         min_length: The value must be a frozenset with at least this many items
         max_length: The value must be a frozenset with at most this many items
-        generator_max_length: The value must generate a frozenset with at most this many items
         strict: The value must be a frozenset with exactly this many items
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
@@ -1456,7 +1443,6 @@ def frozenset_schema(
         items_schema=items_schema,
         min_length=min_length,
         max_length=max_length,
-        generator_max_length=generator_max_length,
         strict=strict,
         ref=ref,
         metadata=metadata,

src/input/input_abstract.rs

@@ -8,7 +8,7 @@ use crate::{PyMultiHostUrl, PyUrl};
 
 use super::datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta};
 use super::return_enums::{EitherBytes, EitherString};
-use super::{GenericArguments, GenericCollection, GenericIterator, GenericMapping, JsonInput};
+use super::{GenericArguments, GenericIterable, GenericIterator, GenericMapping, JsonInput};
 
 #[derive(Debug, Clone, Copy)]
 pub enum InputType {
@@ -166,58 +166,60 @@ pub trait Input<'a>: fmt::Debug + ToPyObject {
         self.validate_dict(strict)
     }
 
-    fn validate_list(&'a self, strict: bool, allow_any_iter: bool) -> ValResult<GenericCollection<'a>> {
-        if strict && !allow_any_iter {
+    fn validate_list(&'a self, strict: bool) -> ValResult<GenericIterable<'a>> {
+        if strict {
             self.strict_list()
         } else {
-            self.lax_list(allow_any_iter)
+            self.lax_list()
         }
     }
-    fn strict_list(&'a self) -> ValResult<GenericCollection<'a>>;
+    fn strict_list(&'a self) -> ValResult<GenericIterable<'a>>;
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn lax_list(&'a self, _allow_any_iter: bool) -> ValResult<GenericCollection<'a>> {
+    fn lax_list(&'a self) -> ValResult<GenericIterable<'a>> {
         self.strict_list()
     }
 
-    fn validate_tuple(&'a self, strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_tuple(&'a self, strict: bool) -> ValResult<GenericIterable<'a>> {
         if strict {
             self.strict_tuple()
         } else {
             self.lax_tuple()
         }
     }
-    fn strict_tuple(&'a self) -> ValResult<GenericCollection<'a>>;
+    fn strict_tuple(&'a self) -> ValResult<GenericIterable<'a>>;
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn lax_tuple(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn lax_tuple(&'a self) -> ValResult<GenericIterable<'a>> {
         self.strict_tuple()
     }
 
-    fn validate_set(&'a self, strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_set(&'a self, strict: bool) -> ValResult<GenericIterable<'a>> {
         if strict {
             self.strict_set()
         } else {
             self.lax_set()
         }
     }
-    fn strict_set(&'a self) -> ValResult<GenericCollection<'a>>;
+    fn strict_set(&'a self) -> ValResult<GenericIterable<'a>>;
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn lax_set(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn lax_set(&'a self) -> ValResult<GenericIterable<'a>> {
         self.strict_set()
     }
 
-    fn validate_frozenset(&'a self, strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_frozenset(&'a self, strict: bool) -> ValResult<GenericIterable<'a>> {
         if strict {
             self.strict_frozenset()
         } else {
             self.lax_frozenset()
         }
     }
-    fn strict_frozenset(&'a self) -> ValResult<GenericCollection<'a>>;
+    fn strict_frozenset(&'a self) -> ValResult<GenericIterable<'a>>;
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn lax_frozenset(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn lax_frozenset(&'a self) -> ValResult<GenericIterable<'a>> {
         self.strict_frozenset()
     }
 
+    fn extract_generic_iterable(&'a self) -> ValResult<GenericIterable<'a>>;
+
     fn validate_iter(&self) -> ValResult<GenericIterator>;
 
     fn validate_date(&self, strict: bool) -> ValResult<EitherDate> {

src/input/input_json.rs

@@ -10,7 +10,7 @@ use super::datetime::{
 use super::parse_json::JsonArray;
 use super::shared::{float_as_int, int_as_bool, map_json_err, str_as_bool, str_as_int};
 use super::{
-    EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericCollection, GenericIterator, GenericMapping,
+    EitherBytes, EitherString, EitherTimedelta, GenericArguments, GenericIterable, GenericIterator, GenericMapping,
     Input, JsonArgs, JsonInput,
 };
 
@@ -187,53 +187,62 @@ impl<'a> Input<'a> for JsonInput {
         self.validate_dict(false)
     }
 
-    fn validate_list(&'a self, _strict: bool, _allow_any_iter: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_list(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         match self {
-            JsonInput::Array(a) => Ok(a.into()),
+            JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)),
             _ => Err(ValError::new(ErrorType::ListType, self)),
         }
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_list(&'a self) -> ValResult<GenericCollection<'a>> {
-        self.validate_list(false, false)
+    fn strict_list(&'a self) -> ValResult<GenericIterable<'a>> {
+        self.validate_list(false)
     }
 
-    fn validate_tuple(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_tuple(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         // just as in set's case, List has to be allowed
         match self {
-            JsonInput::Array(a) => Ok(a.into()),
+            JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)),
             _ => Err(ValError::new(ErrorType::TupleType, self)),
         }
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_tuple(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_tuple(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_tuple(false)
     }
 
-    fn validate_set(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_set(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         // we allow a list here since otherwise it would be impossible to create a set from JSON
         match self {
-            JsonInput::Array(a) => Ok(a.into()),
+            JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)),
             _ => Err(ValError::new(ErrorType::SetType, self)),
         }
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_set(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_set(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_set(false)
     }
 
-    fn validate_frozenset(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_frozenset(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         // we allow a list here since otherwise it would be impossible to create a frozenset from JSON
         match self {
-            JsonInput::Array(a) => Ok(a.into()),
+            JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)),
             _ => Err(ValError::new(ErrorType::FrozenSetType, self)),
         }
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_frozenset(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_frozenset(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_frozenset(false)
     }
 
+    fn extract_generic_iterable(&self) -> ValResult<GenericIterable> {
+        match self {
+            JsonInput::Array(a) => Ok(GenericIterable::JsonArray(a)),
+            JsonInput::String(s) => Ok(GenericIterable::JsonString(s)),
+            JsonInput::Object(object) => Ok(GenericIterable::JsonObject(object)),
+            _ => Err(ValError::new(ErrorType::IterableType, self)),
+        }
+    }
+
     fn validate_iter(&self) -> ValResult<GenericIterator> {
         match self {
             JsonInput::Array(a) => Ok(a.clone().into()),
@@ -405,41 +414,45 @@ impl<'a> Input<'a> for String {
     }
 
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn validate_list(&'a self, _strict: bool, _allow_any_iter: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_list(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         Err(ValError::new(ErrorType::ListType, self))
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_list(&'a self) -> ValResult<GenericCollection<'a>> {
-        self.validate_list(false, false)
+    fn strict_list(&'a self) -> ValResult<GenericIterable<'a>> {
+        self.validate_list(false)
     }
 
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn validate_tuple(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_tuple(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         Err(ValError::new(ErrorType::TupleType, self))
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_tuple(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_tuple(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_tuple(false)
     }
 
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn validate_set(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_set(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         Err(ValError::new(ErrorType::SetType, self))
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_set(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_set(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_set(false)
     }
 
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn validate_frozenset(&'a self, _strict: bool) -> ValResult<GenericCollection<'a>> {
+    fn validate_frozenset(&'a self, _strict: bool) -> ValResult<GenericIterable<'a>> {
         Err(ValError::new(ErrorType::FrozenSetType, self))
     }
     #[cfg_attr(has_no_coverage, no_coverage)]
-    fn strict_frozenset(&'a self) -> ValResult<GenericCollection<'a>> {
+    fn strict_frozenset(&'a self) -> ValResult<GenericIterable<'a>> {
         self.validate_frozenset(false)
     }
 
+    fn extract_generic_iterable(&'a self) -> ValResult<GenericIterable<'a>> {
+        Ok(GenericIterable::JsonString(self))
+    }
+
     fn validate_iter(&self) -> ValResult<GenericIterator> {
         Ok(string_to_vec(self).into())
     }