Can't serialize with a None value

[hemna]

This seems really basic, but I have dataclass instance with a property that has a None value, which is totally legit.

Trying to convert to json fails with

# A printout of the object in question.  You can see the payload is set to None.

MessagePacket(
    from_call='WB4BOR-12',
    to_call='WB4BOR',
    addresse='WB4BOR',
    format='message',
    msgNo='2',
    packet_type='message',
    timestamp=1696885896,
    raw='WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2',
    payload=None,
    path=[],
    via='',
    message_text='broken clouds 65.7F/61.0F Wind 13@290 85%'
)


Traceback (most recent call last):
  File "/Users/i530566/devel/mine/hamradio/aprsd/decode.py", line 56, in <module>
    packet_dict = to_json(packet)
                  ^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/aprsd/.aprsd-venv/lib/python3.12/site-packages/serde/json.py", line 70, in to_json
    return se.serialize(to_dict(obj, c=cls, reuse_instances=False, convert_sets=True), **opts)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/aprsd/.aprsd-venv/lib/python3.12/site-packages/serde/se.py", line 450, in to_dict
    return to_obj(  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/aprsd/.aprsd-venv/lib/python3.12/site-packages/serde/se.py", line 378, in to_obj
    raise SerdeError(e) from None
serde.compat.SerdeError: Unsupported type: NoneType

[opeik]

Do you mind posting the definition of MessagePacket?

[hemna]

https://github.com/craigerl/aprsd/blob/master/aprsd/packets/core.py#L292-L311

The parent class is here:
https://github.com/craigerl/aprsd/blob/master/aprsd/packets/core.py#L49-L71

I'm testing serde to see if I can use it instead of manually calling the json attribute.
I have added the @serde decorator on all of the Packet based classes in that file and it has no affect on the outcome.

Here is my test script. It requires installing aprsd. You can just create a venv and install aprsd that way from either pip or github url here:
https://github.com/craigerl/aprsd.git

import aprslib
from serde.json import from_json, to_json

from aprsd.packets import core
from aprsd.packets import tracker

raw = "WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2"

print(raw)
decoded = aprslib.parse(raw)
print(decoded)
packet = core.Packet.factory(decoded)
print(packet)


# This fails here
packet_dict = to_json(packet)
print(f"Packet json = {packet_dict}")
pkt2 = from_json(packet_dict)

print(pkt2)
print(packet == pkt2)

I added some debug output to serde's se.py to

└─> python serde_test.py
WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2
{'raw': 'WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2', 'from': 'WB4BOR-12', 'to': 'APZ100', 'path': [], 'via': '', 'addresse': 'WB4BOR', 'format': 'message', 'message_text': 'broken clouds 65.7F/61.0F Wind 13@290 85%', 'msgNo': '2'}
WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2
XXX to_dict called with c None
SERDE:se.py::to_obj(o=WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2, named=True, reuse_instances=False, convert_sets=True, c=None)
SERDE:se.py::serializable_to_obj  serde_Scope ==================================================
                  MessagePacket
==================================================

--------------------------------------------------
           Function references in scope
--------------------------------------------------
to_iter: <function to_iter at 0x10a4f1260>
to_dict: <function to_dict at 0x10a599da0>
typecheck: <function typecheck at 0x10a59a840>
 func_name to_dict
SERDE:se.py::serializable_to_obj  scope.functs[func_name] <function to_dict at 0x10a599da0>
SERDE:se.py::serializable_to_obj  object WB4BOR-12>APZ100::WB4BOR   :broken clouds 65.7F/61.0F Wind 13@290 85%{2
Traceback (most recent call last):
  File "/Users/i530566/devel/mine/hamradio/aprsd/serde_test.py", line 19, in <module>
    packet_dict = to_json(packet)
                  ^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/json.py", line 70, in to_json
    return se.serialize(to_dict(obj, c=cls, reuse_instances=False, convert_sets=True), **opts)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 456, in to_dict
    return to_obj(  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 383, in to_obj
    raise SerdeError(e) from None
serde.compat.SerdeError: Unsupported type: NoneType

[hemna]

After further hacking on serde, I was able to modify the to_dict() template generation to print out the assignments inside of the generated function. as well as what the result of the generated function itself was.

def to_dict(obj, reuse_instances = None, convert_sets = None):
  print(f"to_dict called")
  if reuse_instances is None:
    reuse_instances = True
  if convert_sets is None:
    convert_sets = False
  if not is_dataclass(obj):
    return copy.deepcopy(obj)



  print(f"set RES")
  res = {}
  res["from_call"] = obj.from_call
  print(f'res["from_call"] = obj.from_call')


  res["to_call"] = obj.to_call
  print(f'res["to_call"] = obj.to_call')


  res["addresse"] = obj.addresse
  print(f'res["addresse"] = obj.addresse')


  res["format"] = obj.format
  print(f'res["format"] = obj.format')


  res["msgNo"] = obj.msgNo
  print(f'res["msgNo"] = obj.msgNo')


  res["packet_type"] = obj.packet_type
  print(f'res["packet_type"] = obj.packet_type')


  res["timestamp"] = obj.timestamp
  print(f'res["timestamp"] = obj.timestamp')


  res["raw"] = obj.raw
  print(f'res["raw"] = obj.raw')


  res["raw_dict"] = obj.raw_dict
  print(f'res["raw_dict"] = obj.raw_dict')


  res["payload"] = obj.payload
  print(f'res["payload"] = obj.payload')


  res["send_count"] = obj.send_count
  print(f'res["send_count"] = obj.send_count')


  res["retry_count"] = obj.retry_count
  print(f'res["retry_count"] = obj.retry_count')


  res["last_send_time"] = raise_unsupported_type(obj.last_send_time)
  print(f'res["last_send_time"] = raise_unsupported_type(obj.last_send_time)')


  res["allow_delay"] = obj.allow_delay
  print(f'res["allow_delay"] = obj.allow_delay')


  res["path"] = [v for v in obj.path]
  print(f'res["path"] = [v for v in obj.path]')


  res["via"] = obj.via
  print(f'res["via"] = obj.via')


  res["message_text"] = obj.message_text
  print(f'res["message_text"] = obj.message_text')


  return res

and the result is

to_dict called
set RES
res["from_call"] = obj.from_call
res["to_call"] = obj.to_call
res["addresse"] = obj.addresse
res["format"] = obj.format
res["msgNo"] = obj.msgNo
res["packet_type"] = obj.packet_type
res["timestamp"] = obj.timestamp
res["raw"] = obj.raw
res["raw_dict"] = obj.raw_dict
res["payload"] = obj.payload
res["send_count"] = obj.send_count
res["retry_count"] = obj.retry_count
Traceback (most recent call last):
  File "/Users/i530566/devel/mine/hamradio/aprsd/serde_test.py", line 19, in <module>
    packet_dict = to_json(packet)
                  ^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/json.py", line 70, in to_json
    return se.serialize(to_dict(obj, c=cls, reuse_instances=False, convert_sets=True), **opts)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 464, in to_dict
    return to_obj(  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 390, in to_obj
    raise SerdeError(e) from None
serde.compat.SerdeError: Unsupported type: NoneType

It looks like it can't handle the datetime field as defined here:
https://github.com/craigerl/aprsd/blob/master/aprsd/packets/core.py#L67

[hemna]

So it looks like there are a few problems here:

1. serde doesn't support datetime.timedelta
2. serde doesn't handle when a default value of a field is set to None. Instead of just setting it to none when to_json(obj) is run it tries to serialize a NoneType. why?
3. you can't use a default_factory=_my_custom_factory

└─> cat serde_tests.py
import datetime
from dataclasses import dataclass, field

from serde import serde
from serde.json import to_json, from_json

def _init_date():
    return datetime.datetime(1970, 10, 24)

@serde
@dataclass
class packet:
    msgNo: str = field(default="1")
    payload: str = field(default='')

    # This one fails because serde just tries to dereference None, when None is a valid value.
    # last_send_time: datetime.datetime = field(repr=False, default=None, compare=False, hash=False)

    # This one fails, because the default_factory is used that points to a function.
    # last_send_time: datetime.datetime = field(repr=False, default=_init_date, compare=False, hash=False)

    # This one works, but having a last_send_time of None is the correct logic, when the packet object has 
    # never been sent.
    # last_send_time: datetime.datetime = field(repr=False, default=datetime.datetime(1970, 10, 24), compare=False, hash=False)

m = packet(msgNo="1232")

print(to_json(m))

Traceback (most recent call last):
  File "/Users/i530566/devel/mine/hamradio/aprsd/serde_tests.py", line 22, in <module>
    print(to_json(m))
          ^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/json.py", line 70, in to_json
    return se.serialize(to_dict(obj, c=cls, reuse_instances=False, convert_sets=True), **opts)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 472, in to_dict
    return to_obj(  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 398, in to_obj
    raise SerdeError(e) from None
serde.compat.SerdeError: 'NoneType' object has no attribute 'isoformat'

or using the default_factory

Traceback (most recent call last):
  File "/Users/i530566/devel/mine/hamradio/aprsd/serde_tests.py", line 22, in <module>
    print(to_json(m))
          ^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/json.py", line 70, in to_json
    return se.serialize(to_dict(obj, c=cls, reuse_instances=False, convert_sets=True), **opts)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 472, in to_dict
    return to_obj(  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/i530566/devel/mine/hamradio/tmp/pyserde/serde/se.py", line 398, in to_obj
    raise SerdeError(e) from None
serde.compat.SerdeError: 'function' object has no attribute 'isoformat'