Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds a ser_json_datetime config option for configuring how datetimes are serialized #1465

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
4 changes: 4 additions & 0 deletions python/pydantic_core/_pydantic_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ def to_json(
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601',
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -379,6 +380,7 @@ def to_json(
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`.
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'` or `'milliseconds_int'`.
Comment on lines 382 to +383
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`.
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'` or `'milliseconds_int'`.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`.
datetime_mode: How to serialize `datetime` objects, either `'iso8601'`, `'seconds_int'` or `'milliseconds_int'`.

bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down Expand Up @@ -433,6 +435,7 @@ def to_jsonable_python(
exclude_none: bool = False,
round_trip: bool = False,
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601',
datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601',
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
serialize_unknown: bool = False,
Expand All @@ -454,6 +457,7 @@ def to_jsonable_python(
exclude_none: Whether to exclude fields that have a value of `None`.
round_trip: Whether to enable serialization and validation round-trip support.
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`.
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'`, or`'milliseconds_int'`.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'`, or`'milliseconds_int'`.
datetime_mode: How to serialize `datetime` objects, either `'iso8601'`, `'seconds_int'`, or`'milliseconds_int'`.

bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails
Expand Down
3 changes: 3 additions & 0 deletions python/pydantic_core/core_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ class CoreConfig(TypedDict, total=False):
str_to_upper: Whether to convert string fields to uppercase.
allow_inf_nan: Whether to allow infinity and NaN values for float fields. Default is `True`.
ser_json_timedelta: The serialization option for `timedelta` values. Default is 'iso8601'.

ser_json_datetime: The serialization option for `datetime` values. Default is 'iso8601'.
ser_json_bytes: The serialization option for `bytes` values. Default is 'utf8'.
ser_json_inf_nan: The serialization option for infinity and NaN values
in float fields. Default is 'null'.
Expand Down Expand Up @@ -106,6 +108,7 @@ class CoreConfig(TypedDict, total=False):
allow_inf_nan: bool # default: True
# the config options are used to customise serialization to JSON
ser_json_timedelta: Literal['iso8601', 'seconds_float', 'milliseconds_float'] # default: 'iso8601'
ser_json_datetime: Literal['iso8601', 'seconds_int', 'milliseconds_int'] # default: 'iso8601'
ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null'
val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8'
Expand Down
2 changes: 1 addition & 1 deletion src/errors/validation_exception.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ impl ValidationError {
include_context: bool,
include_input: bool,
) -> PyResult<Bound<'py, PyString>> {
let state = SerializationState::new("iso8601", "utf8", "constants")?;
let state = SerializationState::new("iso8601", "iso8601", "utf8", "constants")?;
let extra = state.extra(
py,
&SerMode::Json,
Expand Down
63 changes: 61 additions & 2 deletions src/serializers/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@ use std::str::{from_utf8, FromStr, Utf8Error};
use base64::Engine;
use pyo3::intern;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
use pyo3::types::{PyDateTime, PyDict, PyString};

use serde::ser::Error;

use crate::build_tools::py_schema_err;
use crate::input::EitherTimedelta;
use crate::serializers::type_serializers::datetime_etc::{
datetime_to_milliseconds, datetime_to_seconds, datetime_to_string,
};
use crate::tools::SchemaDict;

use super::errors::py_err_se_err;
Expand All @@ -18,25 +21,34 @@ use super::errors::py_err_se_err;
#[allow(clippy::struct_field_names)]
pub(crate) struct SerializationConfig {
pub timedelta_mode: TimedeltaMode,
pub datetime_mode: DatetimeMode,
pub bytes_mode: BytesMode,
pub inf_nan_mode: InfNanMode,
}

impl SerializationConfig {
pub fn from_config(config: Option<&Bound<'_, PyDict>>) -> PyResult<Self> {
let timedelta_mode = TimedeltaMode::from_config(config)?;
let datetime_mode = DatetimeMode::from_config(config)?;
let bytes_mode = BytesMode::from_config(config)?;
let inf_nan_mode = InfNanMode::from_config(config)?;
Ok(Self {
timedelta_mode,
datetime_mode,
bytes_mode,
inf_nan_mode,
})
}

pub fn from_args(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult<Self> {
pub fn from_args(
timedelta_mode: &str,
datetime_mode: &str,
bytes_mode: &str,
inf_nan_mode: &str,
) -> PyResult<Self> {
Ok(Self {
timedelta_mode: TimedeltaMode::from_str(timedelta_mode)?,
datetime_mode: DatetimeMode::from_str(datetime_mode)?,
bytes_mode: BytesMode::from_str(bytes_mode)?,
inf_nan_mode: InfNanMode::from_str(inf_nan_mode)?,
})
Expand Down Expand Up @@ -92,6 +104,14 @@ serialization_mode! {
MillisecondsFloat => "milliseconds_float"
}

serialization_mode! {
DatetimeMode,
"ser_json_datetime",
Iso8601 => "iso8601",
SecondsInt => "seconds_int",
MillisecondsInt => "milliseconds_int"
}

serialization_mode! {
BytesMode,
"ser_json_bytes",
Expand Down Expand Up @@ -190,6 +210,45 @@ impl BytesMode {
}
}

impl DatetimeMode {
pub fn datetime_to_json(self, py: Python, datetime: &Bound<'_, PyDateTime>) -> PyResult<PyObject> {
match self {
Self::Iso8601 => Ok(datetime_to_string(datetime)?.into_py(py)),
Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.into_py(py)),
Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.into_py(py)),
}
}

pub fn json_key<'py>(self, datetime: &Bound<'_, PyDateTime>) -> PyResult<Cow<'py, str>> {
match self {
Self::Iso8601 => Ok(datetime_to_string(datetime)?.to_string().into()),
Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.to_string().into()),
Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.to_string().into()),
}
}

pub fn datetime_serialize<S: serde::ser::Serializer>(
self,
datetime: &Bound<'_, PyDateTime>,
serializer: S,
) -> Result<S::Ok, S::Error> {
match self {
Self::Iso8601 => {
let s = datetime_to_string(datetime).map_err(py_err_se_err)?;
serializer.serialize_str(&s)
}
Self::SecondsInt => {
let s = datetime_to_seconds(datetime).map_err(py_err_se_err)?;
serializer.serialize_i64(s)
}
Self::MillisecondsInt => {
let s = datetime_to_milliseconds(datetime).map_err(py_err_se_err)?;
serializer.serialize_i64(s)
}
}
}
}

pub fn utf8_py_error(py: Python, err: Utf8Error, data: &[u8]) -> PyErr {
match pyo3::exceptions::PyUnicodeDecodeError::new_utf8_bound(py, data, err) {
Ok(decode_err) => PyErr::from_value_bound(decode_err.into_any()),
Expand Down
4 changes: 2 additions & 2 deletions src/serializers/extra.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@ impl DuckTypingSerMode {
}

impl SerializationState {
pub fn new(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult<Self> {
pub fn new(timedelta_mode: &str, datetime_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult<Self> {
let warnings = CollectWarnings::new(WarningsMode::None);
let rec_guard = SerRecursionState::default();
let config = SerializationConfig::from_args(timedelta_mode, bytes_mode, inf_nan_mode)?;
let config = SerializationConfig::from_args(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?;
Ok(Self {
warnings,
rec_guard,
Expand Down
17 changes: 8 additions & 9 deletions src/serializers/infer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,11 @@ pub(crate) fn infer_to_python_known(
})?
}
ObType::Datetime => {
let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(value.downcast()?)?;
iso_dt.into_py(py)
let datetime = extra
.config
.datetime_mode
.datetime_to_json(value.py(), value.downcast()?)?;
datetime.into_py(py)
}
ObType::Date => {
let iso_date = super::type_serializers::datetime_etc::date_to_string(value.downcast()?)?;
Expand Down Expand Up @@ -458,9 +461,8 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
ObType::Set => serialize_seq!(PySet),
ObType::Frozenset => serialize_seq!(PyFrozenSet),
ObType::Datetime => {
let py_dt = value.downcast().map_err(py_err_se_err)?;
let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(py_dt).map_err(py_err_se_err)?;
serializer.serialize_str(&iso_dt)
let py_datetime = value.downcast().map_err(py_err_se_err)?;
extra.config.datetime_mode.datetime_serialize(py_datetime, serializer)
}
ObType::Date => {
let py_date = value.downcast().map_err(py_err_se_err)?;
Expand Down Expand Up @@ -637,10 +639,7 @@ pub(crate) fn infer_json_key_known<'a>(
.bytes_to_string(key.py(), unsafe { py_byte_array.as_bytes() })
.map(|cow| Cow::Owned(cow.into_owned()))
}
ObType::Datetime => {
let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(key.downcast()?)?;
Ok(Cow::Owned(iso_dt))
}
ObType::Datetime => extra.config.datetime_mode.json_key(key.downcast()?),
ObType::Date => {
let iso_date = super::type_serializers::datetime_etc::date_to_string(key.downcast()?)?;
Ok(Cow::Owned(iso_date))
Expand Down
10 changes: 6 additions & 4 deletions src/serializers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ impl SchemaSerializer {
#[allow(clippy::too_many_arguments)]
#[pyfunction]
#[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = true,
exclude_none = false, round_trip = false, timedelta_mode = "iso8601", bytes_mode = "utf8",
exclude_none = false, round_trip = false, timedelta_mode = "iso8601", datetime_mode = "iso8601", bytes_mode = "utf8",
inf_nan_mode = "constants", serialize_unknown = false, fallback = None, serialize_as_any = false,
context = None))]
pub fn to_json(
Expand All @@ -255,14 +255,15 @@ pub fn to_json(
exclude_none: bool,
round_trip: bool,
timedelta_mode: &str,
datetime_mode: &str,
bytes_mode: &str,
inf_nan_mode: &str,
serialize_unknown: bool,
fallback: Option<&Bound<'_, PyAny>>,
serialize_as_any: bool,
context: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyObject> {
let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?;
let state = SerializationState::new(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?;
let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any);
let extra = state.extra(
py,
Expand All @@ -285,7 +286,7 @@ pub fn to_json(
#[allow(clippy::too_many_arguments)]
#[pyfunction]
#[pyo3(signature = (value, *, include = None, exclude = None, by_alias = true, exclude_none = false, round_trip = false,
timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None,
timedelta_mode = "iso8601", datetime_mode ="iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None,
serialize_as_any = false, context = None))]
pub fn to_jsonable_python(
py: Python,
Expand All @@ -296,14 +297,15 @@ pub fn to_jsonable_python(
exclude_none: bool,
round_trip: bool,
timedelta_mode: &str,
datetime_mode: &str,
bytes_mode: &str,
inf_nan_mode: &str,
serialize_unknown: bool,
fallback: Option<&Bound<'_, PyAny>>,
serialize_as_any: bool,
context: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyObject> {
let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?;
let state = SerializationState::new(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?;
let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any);
let extra = state.extra(
py,
Expand Down
2 changes: 1 addition & 1 deletion src/serializers/shared.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ combined_serializer! {
Decimal: super::type_serializers::decimal::DecimalSerializer;
Str: super::type_serializers::string::StrSerializer;
Bytes: super::type_serializers::bytes::BytesSerializer;
Datetime: super::type_serializers::datetime_etc::DatetimeSerializer;
Datetime: super::type_serializers::datetime_etc::DateTimeSerializer;
TimeDelta: super::type_serializers::timedelta::TimeDeltaSerializer;
Date: super::type_serializers::datetime_etc::DateSerializer;
Time: super::type_serializers::datetime_etc::TimeSerializer;
Expand Down
93 changes: 83 additions & 10 deletions src/serializers/type_serializers/datetime_etc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,27 @@ use std::borrow::Cow;
use pyo3::prelude::*;
use pyo3::types::{PyDate, PyDateTime, PyDict, PyTime};

use crate::definitions::DefinitionsBuilder;
use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time};
use crate::PydanticSerializationUnexpectedValue;

use super::{
infer_json_key, infer_serialize, infer_to_python, py_err_se_err, BuildSerializer, CombinedSerializer, Extra,
SerMode, TypeSerializer,
};
use crate::definitions::DefinitionsBuilder;
use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time};
use crate::serializers::config::{DatetimeMode, FromConfig};
use crate::PydanticSerializationUnexpectedValue;

pub(crate) fn datetime_to_string(py_dt: &Bound<'_, PyDateTime>) -> PyResult<String> {
pydatetime_as_datetime(py_dt).map(|dt| dt.to_string())
}

pub(crate) fn datetime_to_seconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult<i64> {
pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp())
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, seems to be an issue here :(

Looks like dt.timestamp() here doesn't take into account the number of microseconds in the datetime, so anything under a second resolution will get ignored.

Looks like its a speed date thing:
https://github.com/pydantic/speedate/blob/d15ac62b7e98d6e4b8a769b6f5ffab2ad1db5ca5/src/datetime.rs#L580-L582
https://github.com/pydantic/speedate/blob/main/src/time.rs#L388-L393

So i guess a few options:

  1. We don't care, ignore (i dont think this is right)
  2. We care and can calculate the timestamp ourselves here
  3. We care and need to add something upstream to speeddate to allow for returning the timestamp as a f64 with microsecond precision

🤷🏻

}

pub(crate) fn datetime_to_milliseconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult<i64> {
pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp() * 1000)
}

pub(crate) fn date_to_string(py_date: &Bound<'_, PyDate>) -> PyResult<String> {
pydate_as_date(py_date).map(|dt| dt.to_string())
}
Expand Down Expand Up @@ -118,11 +126,76 @@ macro_rules! build_serializer {
};
}

build_serializer!(
DatetimeSerializer,
"datetime",
PyAnyMethods::downcast::<PyDateTime>,
datetime_to_string
);
#[derive(Debug)]
pub struct DateTimeSerializer {
datetime_mode: DatetimeMode,
}

impl BuildSerializer for DateTimeSerializer {
const EXPECTED_TYPE: &'static str = "datetime";

fn build(
_schema: &Bound<'_, PyDict>,
config: Option<&Bound<'_, PyDict>>,
_definitions: &mut DefinitionsBuilder<CombinedSerializer>,
) -> PyResult<CombinedSerializer> {
let datetime_mode = DatetimeMode::from_config(config)?;
Ok(Self { datetime_mode }.into())
}
}
impl_py_gc_traverse!(DateTimeSerializer {});

impl TypeSerializer for DateTimeSerializer {
fn to_python(
&self,
value: &Bound<'_, PyAny>,
include: Option<&Bound<'_, PyAny>>,
exclude: Option<&Bound<'_, PyAny>>,
extra: &Extra,
) -> PyResult<PyObject> {
match extra.mode {
SerMode::Json => match PyAnyMethods::downcast::<PyDateTime>(value) {
Ok(py_value) => Ok(self.datetime_mode.datetime_to_json(value.py(), py_value)?),
Err(_) => {
extra.warnings.on_fallback_py(self.get_name(), value, extra)?;
infer_to_python(value, include, exclude, extra)
}
},
_ => infer_to_python(value, include, exclude, extra),
}
}

fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult<Cow<'a, str>> {
match PyAnyMethods::downcast::<PyDateTime>(key) {
Ok(py_value) => Ok(self.datetime_mode.json_key(py_value)?),
Err(_) => {
extra.warnings.on_fallback_py(self.get_name(), key, extra)?;
infer_json_key(key, extra)
}
}
}

fn serde_serialize<S: serde::ser::Serializer>(
&self,
value: &Bound<'_, PyAny>,
serializer: S,
include: Option<&Bound<'_, PyAny>>,
exclude: Option<&Bound<'_, PyAny>>,
extra: &Extra,
) -> Result<S::Ok, S::Error> {
match PyAnyMethods::downcast::<PyDateTime>(value) {
Ok(py_value) => self.datetime_mode.datetime_serialize(py_value, serializer),
Err(_) => {
extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?;
infer_serialize(value, serializer, include, exclude, extra)
}
}
}

fn get_name(&self) -> &str {
Self::EXPECTED_TYPE
}
}

build_serializer!(DateSerializer, "date", downcast_date_reject_datetime, date_to_string);
build_serializer!(TimeSerializer, "time", PyAnyMethods::downcast::<PyTime>, time_to_string);
Loading
Loading