Encode datetime as Timestamp in msgpack

decoding custom format is still supported for backward compat.

Note that this revision modifies the exception type raised when attempting to
encode a naive datetime object for both formats (json amd msgpack): it now
raises a TypeError instead of a ValueError.

requirements-http.txt

@@ -4,6 +4,6 @@ aiohttp_utils >= 3.1.1
 decorator
 Flask
 iso8601
-msgpack > 0.5
+msgpack >= 1.0.0
 requests
 blinker  # dependency of sentry-sdk[flask]

swh/core/api/serializers.py

@@ -22,7 +22,7 @@ from swh.core.api.classes import PagedResult
 def encode_datetime(dt: datetime.datetime) -> str:
     """Wrapper of datetime.datetime.isoformat() that forbids naive datetimes."""
     if dt.tzinfo is None:
-        raise ValueError(f"{dt} is a naive datetime.")
+        raise TypeError("can not serialize naive 'datetime.datetime' object")
     return dt.isoformat()
 
 
@@ -65,7 +65,6 @@ def encode_timedelta(td: datetime.timedelta) -> Dict[str, int]:
 
 
 ENCODERS: List[Tuple[type, str, Callable]] = [
-    (datetime.datetime, "datetime", encode_datetime),
     (UUID, "uuid", str),
     (datetime.timedelta, "timedelta", encode_timedelta),
     (PagedResult, "paged_result", _encode_paged_result),
@@ -73,15 +72,17 @@ ENCODERS: List[Tuple[type, str, Callable]] = [
 ]
 
 JSON_ENCODERS: List[Tuple[type, str, Callable]] = [
+    (datetime.datetime, "datetime", encode_datetime),
     (bytes, "bytes", lambda o: base64.b85encode(o).decode("ascii")),
 ]
 
 DECODERS: Dict[str, Callable] = {
-    "datetime": lambda d: iso8601.parse_date(d, default_timezone=None),
     "timedelta": lambda d: datetime.timedelta(**d),
     "uuid": UUID,
     "paged_result": _decode_paged_result,
     "exception": dict_to_exception,
+    # for BW compat, to be moved in JSON_DECODERS ASAP
+    "datetime": lambda d: iso8601.parse_date(d, default_timezone=None),
 }
 
 JSON_DECODERS: Dict[str, Callable] = {
@@ -263,7 +264,12 @@ def msgpack_dumps(data: Any, extra_encoders=None) -> bytes:
                 }
         return obj
 
-    return msgpack.packb(data, use_bin_type=True, default=encode_types)
+    return msgpack.packb(
+        data,
+        use_bin_type=True,
+        datetime=True,  # encode datetime as msgpack.Timestamp
+        default=encode_types,
+    )
 
 
 def msgpack_loads(data: bytes, extra_decoders=None) -> Any:
@@ -300,6 +306,7 @@ def msgpack_loads(data: bytes, extra_decoders=None) -> Any:
                 object_hook=decode_types,
                 ext_hook=ext_hook,
                 strict_map_key=False,
+                timestamp=3,  # convert Timestamp in datetime objects (tz UTC)
             )
         except TypeError:  # msgpack < 0.6.0
             return msgpack.unpackb(

swh/core/api/tests/test_serializers.py

@@ -8,6 +8,7 @@ import json
 from typing import Any, Callable, List, Tuple, Union
 from uuid import UUID
 
+import msgpack
 import pytest
 import requests
 from requests.exceptions import ConnectionError
@@ -207,12 +208,33 @@ def test_serializers_decode_response_json(requests_mock):
     assert decode_response(response) == DATA
 
 
-def test_serializers_encode_native_datetime():
+def test_serializers_encode_datetime_msgpack():
+    dt = datetime.datetime.now(tz=datetime.timezone.utc)
+    encmsg = msgpack_dumps(dt)
+    decmsg = msgpack.loads(encmsg, timestamp=0)
+    assert isinstance(decmsg, msgpack.Timestamp)
+    assert decmsg.to_datetime() == dt
+
+
+def test_serializers_decode_datetime_compat_msgpack():
+    dt = datetime.datetime.now(tz=datetime.timezone.utc)
+    encmsg = msgpack_dumps({b"swhtype": "datetime", b"d": dt.isoformat()})
+    decmsg = msgpack_loads(encmsg)
+    assert decmsg == dt
+
+
+def test_serializers_encode_native_datetime_msgpack():
     dt = datetime.datetime(2015, 1, 1, 12, 4, 42, 231455)
-    with pytest.raises(ValueError, match="naive datetime"):
+    with pytest.raises(TypeError, match="datetime"):
         msgpack_dumps(dt)
 
 
+def test_serializers_encode_native_datetime_json():
+    dt = datetime.datetime(2015, 1, 1, 12, 4, 42, 231455)
+    with pytest.raises(TypeError, match="datetime"):
+        json_dumps(dt)
+
+
 def test_serializers_decode_naive_datetime():
     expected_dt = datetime.datetime(2015, 1, 1, 12, 4, 42, 231455)