Implement validation based on ser_json_bytes to support round trip

Author: josh-newman

src/input/input_abstract.rs

@@ -6,6 +6,7 @@ use pyo3::{intern, prelude::*};
 
 use crate::errors::{ErrorTypeDefaults, InputValue, LocItem, ValError, ValResult};
 use crate::lookup_key::{LookupKey, LookupPath};
+use crate::serializers::config::BytesMode;
 use crate::tools::py_err;
 
 use super::datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta};
@@ -71,7 +72,7 @@ pub trait Input<'py>: fmt::Debug + ToPyObject {
 
     fn validate_str(&self, strict: bool, coerce_numbers_to_str: bool) -> ValMatch<EitherString<'_>>;
 
-    fn validate_bytes<'a>(&'a self, strict: bool) -> ValMatch<EitherBytes<'a, 'py>>;
+    fn validate_bytes<'a>(&'a self, strict: bool, mode: BytesMode) -> ValMatch<EitherBytes<'a, 'py>>;
 
     fn validate_bool(&self, strict: bool) -> ValMatch<bool>;
 

src/input/input_json.rs

@@ -9,6 +9,7 @@ use strum::EnumMessage;
 
 use crate::errors::{ErrorType, ErrorTypeDefaults, InputValue, LocItem, ValError, ValResult};
 use crate::lookup_key::{LookupKey, LookupPath};
+use crate::serializers::config::BytesMode;
 use crate::validators::decimal::create_decimal;
 
 use super::datetime::{
@@ -106,9 +107,16 @@ impl<'py, 'data> Input<'py> for JsonValue<'data> {
         }
     }
 
-    fn validate_bytes<'a>(&'a self, _strict: bool) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
+    fn validate_bytes<'a>(
+        &'a self,
+        _strict: bool,
+        mode: BytesMode,
+    ) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
         match self {
-            JsonValue::Str(s) => Ok(ValidationMatch::strict(s.as_bytes().into())),
+            JsonValue::Str(s) => match mode.deserialize_string(s) {
+                Ok(b) => Ok(ValidationMatch::strict(b)),
+                Err(e) => Err(ValError::from(e)),
+            },
             _ => Err(ValError::new(ErrorTypeDefaults::BytesType, self)),
         }
     }
@@ -342,8 +350,15 @@ impl<'py> Input<'py> for str {
         Ok(ValidationMatch::strict(self.into()))
     }
 
-    fn validate_bytes<'a>(&'a self, _strict: bool) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
-        Ok(ValidationMatch::strict(self.as_bytes().into()))
+    fn validate_bytes<'a>(
+        &'a self,
+        _strict: bool,
+        mode: BytesMode,
+    ) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
+        match mode.deserialize_string(self) {
+            Ok(b) => Ok(ValidationMatch::strict(b)),
+            Err(e) => Err(ValError::from(e)),
+        }
     }
 
     fn validate_bool(&self, _strict: bool) -> ValResult<ValidationMatch<bool>> {

src/input/input_python.rs

@@ -14,6 +14,7 @@ use pyo3::PyTypeCheck;
 use speedate::MicrosecondsPrecisionOverflowBehavior;
 
 use crate::errors::{ErrorType, ErrorTypeDefaults, InputValue, LocItem, ValError, ValResult};
+use crate::serializers::config::BytesMode;
 use crate::tools::{extract_i64, safe_repr};
 use crate::validators::decimal::{create_decimal, get_decimal_type};
 use crate::validators::Exactness;
@@ -174,7 +175,7 @@ impl<'py> Input<'py> for Bound<'py, PyAny> {
         Err(ValError::new(ErrorTypeDefaults::StringType, self))
     }
 
-    fn validate_bytes<'a>(&'a self, strict: bool) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
+    fn validate_bytes<'a>(&'a self, strict: bool, mode: BytesMode) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
         if let Ok(py_bytes) = self.downcast_exact::<PyBytes>() {
             return Ok(ValidationMatch::exact(py_bytes.into()));
         } else if let Ok(py_bytes) = self.downcast::<PyBytes>() {
@@ -185,7 +186,10 @@ impl<'py> Input<'py> for Bound<'py, PyAny> {
             if !strict {
                 return if let Ok(py_str) = self.downcast::<PyString>() {
                     let str = py_string_str(py_str)?;
-                    Ok(str.as_bytes().into())
+                    match mode.deserialize_string(str) {
+                        Ok(b) => Ok(b),
+                        Err(e) => Err(ValError::from(e)),
+                    }
                 } else if let Ok(py_byte_array) = self.downcast::<PyByteArray>() {
                     Ok(py_byte_array.to_vec().into())
                 } else {

src/input/input_string.rs

@@ -6,6 +6,7 @@ use speedate::MicrosecondsPrecisionOverflowBehavior;
 use crate::errors::{ErrorTypeDefaults, InputValue, LocItem, ValError, ValResult};
 use crate::input::py_string_str;
 use crate::lookup_key::{LookupKey, LookupPath};
+use crate::serializers::config::BytesMode;
 use crate::tools::safe_repr;
 use crate::validators::decimal::create_decimal;
 
@@ -105,9 +106,16 @@ impl<'py> Input<'py> for StringMapping<'py> {
         }
     }
 
-    fn validate_bytes<'a>(&'a self, _strict: bool) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
+    fn validate_bytes<'a>(
+        &'a self,
+        _strict: bool,
+        mode: BytesMode,
+    ) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
         match self {
-            Self::String(s) => py_string_str(s).map(|b| ValidationMatch::strict(b.as_bytes().into())),
+            Self::String(s) => py_string_str(s).and_then(|b| match mode.deserialize_string(b) {
+                Ok(b) => Ok(ValidationMatch::strict(b)),
+                Err(e) => Err(ValError::from(e)),
+            }),
             Self::Mapping(_) => Err(ValError::new(ErrorTypeDefaults::BytesType, self)),
         }
     }

src/lib.rs

@@ -7,6 +7,7 @@ use std::sync::OnceLock;
 use jiter::StringCacheMode;
 use pyo3::exceptions::PyTypeError;
 use pyo3::{prelude::*, sync::GILOnceCell};
+use serializers::config::BytesMode;
 
 // parse this first to get access to the contained macro
 #[macro_use]
@@ -55,7 +56,7 @@ pub fn from_json<'py>(
     allow_partial: bool,
 ) -> PyResult<Bound<'py, PyAny>> {
     let v_match = data
-        .validate_bytes(false)
+        .validate_bytes(false, BytesMode::Utf8)
         .map_err(|_| PyTypeError::new_err("Expected bytes, bytearray or str"))?;
     let json_either_bytes = v_match.into_inner();
     let json_bytes = json_either_bytes.as_slice();

src/serializers/config.rs

@@ -2,14 +2,15 @@ use std::borrow::Cow;
 use std::str::{from_utf8, FromStr, Utf8Error};
 
 use base64::Engine;
+use pyo3::exceptions::PyValueError;
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::types::{PyDelta, PyDict, PyString};
 
 use serde::ser::Error;
 
 use crate::build_tools::py_schema_err;
-use crate::input::EitherTimedelta;
+use crate::input::{EitherBytes, EitherTimedelta};
 use crate::tools::SchemaDict;
 
 use super::errors::py_err_se_err;
@@ -187,6 +188,17 @@ impl BytesMode {
             }
         }
     }
+
+    pub fn deserialize_string<'a, 'py>(&self, s: &'a str) -> PyResult<EitherBytes<'a, 'py>> {
+        match self {
+            Self::Utf8 => Ok(EitherBytes::Cow(Cow::Borrowed(s.as_bytes()))),
+            Self::Base64 => match base64::engine::general_purpose::URL_SAFE.decode(s) {
+                Ok(bytes) => Ok(EitherBytes::from(bytes)),
+                Err(err) => Err(PyValueError::new_err(format!("Base64 decode error: {}", err))),
+            },
+            Self::Hex => Err(PyValueError::new_err("Hex deserialization is not supported")),
+        }
+    }
 }
 
 pub fn utf8_py_error(py: Python, err: Utf8Error, data: &[u8]) -> PyErr {

src/serializers/mod.rs

@@ -16,7 +16,7 @@ pub use shared::CombinedSerializer;
 use shared::{to_json_bytes, BuildSerializer, TypeSerializer};
 
 mod computed_fields;
-mod config;
+pub(crate) mod config;
 mod errors;
 mod extra;
 mod fields;

src/validators/bytes.rs

@@ -6,13 +6,15 @@ use crate::build_tools::is_strict;
 use crate::errors::{ErrorType, ValError, ValResult};
 use crate::input::Input;
 
+use crate::serializers::config::{BytesMode, FromConfig};
 use crate::tools::SchemaDict;
 
 use super::{BuildValidator, CombinedValidator, DefinitionsBuilder, ValidationState, Validator};
 
 #[derive(Debug, Clone)]
 pub struct BytesValidator {
     strict: bool,
+    bytes_mode: BytesMode,
 }
 
 impl BuildValidator for BytesValidator {
@@ -31,6 +33,7 @@ impl BuildValidator for BytesValidator {
         } else {
             Ok(Self {
                 strict: is_strict(schema, config)?,
+                bytes_mode: BytesMode::from_config(config)?,
             }
             .into())
         }
@@ -47,7 +50,7 @@ impl Validator for BytesValidator {
         state: &mut ValidationState<'_, 'py>,
     ) -> ValResult<PyObject> {
         input
-            .validate_bytes(state.strict_or(self.strict))
+            .validate_bytes(state.strict_or(self.strict), self.bytes_mode.clone())
             .map(|m| m.unpack(state).into_py(py))
     }
 
@@ -59,6 +62,7 @@ impl Validator for BytesValidator {
 #[derive(Debug, Clone)]
 pub struct BytesConstrainedValidator {
     strict: bool,
+    bytes_mode: BytesMode,
     max_length: Option<usize>,
     min_length: Option<usize>,
 }
@@ -72,7 +76,9 @@ impl Validator for BytesConstrainedValidator {
         input: &(impl Input<'py> + ?Sized),
         state: &mut ValidationState<'_, 'py>,
     ) -> ValResult<PyObject> {
-        let either_bytes = input.validate_bytes(state.strict_or(self.strict))?.unpack(state);
+        let either_bytes = input
+            .validate_bytes(state.strict_or(self.strict), self.bytes_mode.clone())?
+            .unpack(state);
         let len = either_bytes.len()?;
 
         if let Some(min_length) = self.min_length {
@@ -110,6 +116,7 @@ impl BytesConstrainedValidator {
         let py = schema.py();
         Ok(Self {
             strict: is_strict(schema, config)?,
+            bytes_mode: BytesMode::from_config(config)?,
             min_length: schema.get_as(intern!(py, "min_length"))?,
             max_length: schema.get_as(intern!(py, "max_length"))?,
         }

src/validators/json.rs

@@ -6,6 +6,7 @@ use jiter::JsonValue;
 
 use crate::errors::{ErrorType, ErrorTypeDefaults, ValError, ValLineError, ValResult};
 use crate::input::{EitherBytes, Input, InputType, ValidationMatch};
+use crate::serializers::config::BytesMode;
 use crate::tools::SchemaDict;
 
 use super::{build_validator, BuildValidator, CombinedValidator, DefinitionsBuilder, ValidationState, Validator};
@@ -79,7 +80,7 @@ impl Validator for JsonValidator {
 pub fn validate_json_bytes<'a, 'py>(
     input: &'a (impl Input<'py> + ?Sized),
 ) -> ValResult<ValidationMatch<EitherBytes<'a, 'py>>> {
-    match input.validate_bytes(false) {
+    match input.validate_bytes(false, BytesMode::Utf8) {
         Ok(v_match) => Ok(v_match),
         Err(ValError::LineErrors(e)) => Err(ValError::LineErrors(
             e.into_iter().map(map_bytes_error).collect::<Vec<_>>(),

src/validators/uuid.rs

@@ -13,6 +13,7 @@ use crate::input::input_as_python_instance;
 use crate::input::Input;
 use crate::input::InputType;
 use crate::input::ValidationMatch;
+use crate::serializers::config::BytesMode;
 use crate::tools::SchemaDict;
 
 use super::model::create_class;
@@ -169,7 +170,7 @@ impl UuidValidator {
             }
             None => {
                 let either_bytes = input
-                    .validate_bytes(true)
+                    .validate_bytes(true, BytesMode::Utf8)
                     .map_err(|_| ValError::new(ErrorTypeDefaults::UuidType, input))?
                     .into_inner();
                 let bytes_slice = either_bytes.as_slice();

tests/test_json.py

@@ -376,3 +376,22 @@ def test_partial_parse():
     with pytest.raises(ValueError, match='EOF while parsing a string at line 1 column 15'):
         from_json(b'["aa", "bb", "c')
     assert from_json(b'["aa", "bb", "c', allow_partial=True) == ['aa', 'bb']
+
+
+def test_json_bytes_base64_round_trip():
+    data = b'hello'
+    encoded = b'"aGVsbG8="'
+    assert to_json(data, bytes_mode='base64') == encoded
+
+    v = SchemaValidator({'type': 'bytes'}, {'ser_json_bytes': 'base64'})
+    assert v.validate_json(encoded) == data
+
+    with pytest.raises(ValueError):
+        v.validate_json('"wrong!"')
+
+    assert to_json({'key': data}, bytes_mode='base64') == b'{"key":"aGVsbG8="}'
+    v = SchemaValidator(
+        {'type': 'dict', 'keys_schema': {'type': 'str'}, 'values_schema': {'type': 'bytes'}},
+        {'ser_json_bytes': 'base64'},
+    )
+    assert v.validate_json('{"key":"aGVsbG8="}') == {'key': data}