muutils.json_serialize.json_serialize
provides the basic framework for json serialization of objects
notably:
SerializerHandler
defines how to serialize a specific type of objectJsonSerializer
handles configuration for which handlers to usejson_serialize
provides the default configuration if you don't care -- call it on any object!
1"""provides the basic framework for json serialization of objects 2 3notably: 4 5- `SerializerHandler` defines how to serialize a specific type of object 6- `JsonSerializer` handles configuration for which handlers to use 7- `json_serialize` provides the default configuration if you don't care -- call it on any object! 8 9""" 10 11from __future__ import annotations 12 13import inspect 14import warnings 15from dataclasses import dataclass, is_dataclass 16from pathlib import Path 17from typing import Any, Callable, Iterable, Mapping, Set, Union 18 19from muutils.errormode import ErrorMode 20 21try: 22 from muutils.json_serialize.array import ArrayMode, serialize_array 23except ImportError as e: 24 ArrayMode = str # type: ignore[misc] 25 serialize_array = lambda *args, **kwargs: None # noqa: E731 26 warnings.warn( 27 f"muutils.json_serialize.array could not be imported probably because missing numpy, array serialization will not work: \n{e}", 28 ImportWarning, 29 ) 30 31from muutils.json_serialize.util import ( 32 Hashableitem, 33 JSONitem, 34 MonoTuple, 35 SerializationException, 36 _recursive_hashify, 37 isinstance_namedtuple, 38 safe_getsource, 39 string_as_lines, 40 try_catch, 41) 42 43# pylint: disable=protected-access 44 45SERIALIZER_SPECIAL_KEYS: MonoTuple[str] = ( 46 "__name__", 47 "__doc__", 48 "__module__", 49 "__class__", 50 "__dict__", 51 "__annotations__", 52) 53 54SERIALIZER_SPECIAL_FUNCS: dict[str, Callable] = { 55 "str": str, 56 "dir": dir, 57 "type": try_catch(lambda x: str(type(x).__name__)), 58 "repr": try_catch(lambda x: repr(x)), 59 "code": try_catch(lambda x: inspect.getsource(x)), 60 "sourcefile": try_catch(lambda x: inspect.getsourcefile(x)), 61} 62 63SERIALIZE_DIRECT_AS_STR: Set[str] = { 64 "<class 'torch.device'>", 65 "<class 'torch.dtype'>", 66} 67 68ObjectPath = MonoTuple[Union[str, int]] 69 70 71@dataclass 72class SerializerHandler: 73 """a handler for a specific type of object 74 75 # Parameters: 76 - `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler 77 - `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object 78 - `desc : str` description of the handler (optional) 79 """ 80 81 # (self_config, object) -> whether to use this handler 82 check: Callable[["JsonSerializer", Any, ObjectPath], bool] 83 # (self_config, object, path) -> serialized object 84 serialize_func: Callable[["JsonSerializer", Any, ObjectPath], JSONitem] 85 # unique identifier for the handler 86 uid: str 87 # description of this serializer 88 desc: str 89 90 def serialize(self) -> dict: 91 """serialize the handler info""" 92 return { 93 # get the code and doc of the check function 94 "check": { 95 "code": safe_getsource(self.check), 96 "doc": string_as_lines(self.check.__doc__), 97 }, 98 # get the code and doc of the load function 99 "serialize_func": { 100 "code": safe_getsource(self.serialize_func), 101 "doc": string_as_lines(self.serialize_func.__doc__), 102 }, 103 # get the uid, source_pckg, priority, and desc 104 "uid": str(self.uid), 105 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 106 "__module__": getattr(self.serialize_func, "__module__", None), 107 "desc": str(self.desc), 108 } 109 110 111BASE_HANDLERS: MonoTuple[SerializerHandler] = ( 112 SerializerHandler( 113 check=lambda self, obj, path: isinstance( 114 obj, (bool, int, float, str, type(None)) 115 ), 116 serialize_func=lambda self, obj, path: obj, 117 uid="base types", 118 desc="base types (bool, int, float, str, None)", 119 ), 120 SerializerHandler( 121 check=lambda self, obj, path: isinstance(obj, Mapping), 122 serialize_func=lambda self, obj, path: { 123 str(k): self.json_serialize(v, tuple(path) + (k,)) for k, v in obj.items() 124 }, 125 uid="dictionaries", 126 desc="dictionaries", 127 ), 128 SerializerHandler( 129 check=lambda self, obj, path: isinstance(obj, (list, tuple)), 130 serialize_func=lambda self, obj, path: [ 131 self.json_serialize(x, tuple(path) + (i,)) for i, x in enumerate(obj) 132 ], 133 uid="(list, tuple) -> list", 134 desc="lists and tuples as lists", 135 ), 136) 137 138 139def _serialize_override_serialize_func( 140 self: "JsonSerializer", obj: Any, path: ObjectPath 141) -> JSONitem: 142 # obj_cls: type = type(obj) 143 # if hasattr(obj_cls, "_register_self") and callable(obj_cls._register_self): 144 # obj_cls._register_self() 145 146 # get the serialized object 147 return obj.serialize() 148 149 150DEFAULT_HANDLERS: MonoTuple[SerializerHandler] = tuple(BASE_HANDLERS) + ( 151 SerializerHandler( 152 # TODO: allow for custom serialization handler name 153 check=lambda self, obj, path: hasattr(obj, "serialize") 154 and callable(obj.serialize), 155 serialize_func=_serialize_override_serialize_func, 156 uid=".serialize override", 157 desc="objects with .serialize method", 158 ), 159 SerializerHandler( 160 check=lambda self, obj, path: isinstance_namedtuple(obj), 161 serialize_func=lambda self, obj, path: self.json_serialize(dict(obj._asdict())), 162 uid="namedtuple -> dict", 163 desc="namedtuples as dicts", 164 ), 165 SerializerHandler( 166 check=lambda self, obj, path: is_dataclass(obj), 167 serialize_func=lambda self, obj, path: { 168 k: self.json_serialize(getattr(obj, k), tuple(path) + (k,)) 169 for k in obj.__dataclass_fields__ 170 }, 171 uid="dataclass -> dict", 172 desc="dataclasses as dicts", 173 ), 174 SerializerHandler( 175 check=lambda self, obj, path: isinstance(obj, Path), 176 serialize_func=lambda self, obj, path: obj.as_posix(), 177 uid="path -> str", 178 desc="Path objects as posix strings", 179 ), 180 SerializerHandler( 181 check=lambda self, obj, path: str(type(obj)) in SERIALIZE_DIRECT_AS_STR, 182 serialize_func=lambda self, obj, path: str(obj), 183 uid="obj -> str(obj)", 184 desc="directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings", 185 ), 186 SerializerHandler( 187 check=lambda self, obj, path: str(type(obj)) == "<class 'numpy.ndarray'>", 188 serialize_func=lambda self, obj, path: serialize_array(self, obj, path=path), 189 uid="numpy.ndarray", 190 desc="numpy arrays", 191 ), 192 SerializerHandler( 193 check=lambda self, obj, path: str(type(obj)) == "<class 'torch.Tensor'>", 194 serialize_func=lambda self, obj, path: serialize_array( 195 self, obj.detach().cpu(), path=path 196 ), 197 uid="torch.Tensor", 198 desc="pytorch tensors", 199 ), 200 SerializerHandler( 201 check=lambda self, obj, path: str(type(obj)) 202 == "<class 'pandas.core.frame.DataFrame'>", 203 serialize_func=lambda self, obj, path: dict( 204 __format__="pandas.DataFrame", 205 columns=obj.columns.tolist(), 206 data=obj.to_dict(orient="records"), 207 path=path, 208 ), 209 uid="pandas.DataFrame", 210 desc="pandas DataFrames", 211 ), 212 SerializerHandler( 213 check=lambda self, obj, path: isinstance(obj, (set, list, tuple)) 214 or isinstance(obj, Iterable), 215 serialize_func=lambda self, obj, path: [ 216 self.json_serialize(x, tuple(path) + (i,)) for i, x in enumerate(obj) 217 ], 218 uid="(set, list, tuple, Iterable) -> list", 219 desc="sets, lists, tuples, and Iterables as lists", 220 ), 221 SerializerHandler( 222 check=lambda self, obj, path: True, 223 serialize_func=lambda self, obj, path: { 224 **{k: str(getattr(obj, k, None)) for k in SERIALIZER_SPECIAL_KEYS}, 225 **{k: f(obj) for k, f in SERIALIZER_SPECIAL_FUNCS.items()}, 226 }, 227 uid="fallback", 228 desc="fallback handler -- serialize object attributes and special functions as strings", 229 ), 230) 231 232 233class JsonSerializer: 234 """Json serialization class (holds configs) 235 236 # Parameters: 237 - `array_mode : ArrayMode` 238 how to write arrays 239 (defaults to `"array_list_meta"`) 240 - `error_mode : ErrorMode` 241 what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") 242 (defaults to `"except"`) 243 - `handlers_pre : MonoTuple[SerializerHandler]` 244 handlers to use before the default handlers 245 (defaults to `tuple()`) 246 - `handlers_default : MonoTuple[SerializerHandler]` 247 default handlers to use 248 (defaults to `DEFAULT_HANDLERS`) 249 - `write_only_format : bool` 250 changes "__format__" keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) 251 (defaults to `False`) 252 253 # Raises: 254 - `ValueError`: on init, if `args` is not empty 255 - `SerializationException`: on `json_serialize()`, if any error occurs when trying to serialize an object and `error_mode` is set to `ErrorMode.EXCEPT"` 256 257 """ 258 259 def __init__( 260 self, 261 *args, 262 array_mode: ArrayMode = "array_list_meta", 263 error_mode: ErrorMode = ErrorMode.EXCEPT, 264 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 265 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 266 write_only_format: bool = False, 267 ): 268 if len(args) > 0: 269 raise ValueError( 270 f"JsonSerializer takes no positional arguments!\n{args = }" 271 ) 272 273 self.array_mode: ArrayMode = array_mode 274 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 275 self.write_only_format: bool = write_only_format 276 # join up the handlers 277 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 278 handlers_default 279 ) 280 281 def json_serialize( 282 self, 283 obj: Any, 284 path: ObjectPath = tuple(), 285 ) -> JSONitem: 286 try: 287 for handler in self.handlers: 288 if handler.check(self, obj, path): 289 output: JSONitem = handler.serialize_func(self, obj, path) 290 if self.write_only_format: 291 if isinstance(output, dict) and "__format__" in output: 292 new_fmt: JSONitem = output.pop("__format__") 293 output["__write_format__"] = new_fmt 294 return output 295 296 raise ValueError(f"no handler found for object with {type(obj) = }") 297 298 except Exception as e: 299 if self.error_mode == "except": 300 obj_str: str = repr(obj) 301 if len(obj_str) > 1000: 302 obj_str = obj_str[:1000] + "..." 303 raise SerializationException( 304 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 305 ) from e 306 elif self.error_mode == "warn": 307 warnings.warn( 308 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 309 ) 310 311 return repr(obj) 312 313 def hashify( 314 self, 315 obj: Any, 316 path: ObjectPath = tuple(), 317 force: bool = True, 318 ) -> Hashableitem: 319 """try to turn any object into something hashable""" 320 data = self.json_serialize(obj, path=path) 321 322 # recursive hashify, turning dicts and lists into tuples 323 return _recursive_hashify(data, force=force) 324 325 326GLOBAL_JSON_SERIALIZER: JsonSerializer = JsonSerializer() 327 328 329def json_serialize(obj: Any, path: ObjectPath = tuple()) -> JSONitem: 330 """serialize object to json-serializable object with default config""" 331 return GLOBAL_JSON_SERIALIZER.json_serialize(obj, path=path)
SERIALIZER_SPECIAL_KEYS: None =
('__name__', '__doc__', '__module__', '__class__', '__dict__', '__annotations__')
SERIALIZER_SPECIAL_FUNCS: dict[str, typing.Callable] =
{'str': <class 'str'>, 'dir': <built-in function dir>, 'type': <function <lambda>>, 'repr': <function <lambda>>, 'code': <function <lambda>>, 'sourcefile': <function <lambda>>}
SERIALIZE_DIRECT_AS_STR: Set[str] =
{"<class 'torch.device'>", "<class 'torch.dtype'>"}
ObjectPath =
tuple[typing.Union[str, int], ...]
@dataclass
class
SerializerHandler:
72@dataclass 73class SerializerHandler: 74 """a handler for a specific type of object 75 76 # Parameters: 77 - `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler 78 - `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object 79 - `desc : str` description of the handler (optional) 80 """ 81 82 # (self_config, object) -> whether to use this handler 83 check: Callable[["JsonSerializer", Any, ObjectPath], bool] 84 # (self_config, object, path) -> serialized object 85 serialize_func: Callable[["JsonSerializer", Any, ObjectPath], JSONitem] 86 # unique identifier for the handler 87 uid: str 88 # description of this serializer 89 desc: str 90 91 def serialize(self) -> dict: 92 """serialize the handler info""" 93 return { 94 # get the code and doc of the check function 95 "check": { 96 "code": safe_getsource(self.check), 97 "doc": string_as_lines(self.check.__doc__), 98 }, 99 # get the code and doc of the load function 100 "serialize_func": { 101 "code": safe_getsource(self.serialize_func), 102 "doc": string_as_lines(self.serialize_func.__doc__), 103 }, 104 # get the uid, source_pckg, priority, and desc 105 "uid": str(self.uid), 106 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 107 "__module__": getattr(self.serialize_func, "__module__", None), 108 "desc": str(self.desc), 109 }
a handler for a specific type of object
Parameters:
- `check : Callable[[JsonSerializer, Any], bool]` takes a JsonSerializer and an object, returns whether to use this handler
- `serialize : Callable[[JsonSerializer, Any, ObjectPath], JSONitem]` takes a JsonSerializer, an object, and the current path, returns the serialized object
- `desc : str` description of the handler (optional)
SerializerHandler( check: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], bool], serialize_func: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], Union[bool, int, float, str, list, Dict[str, Any], NoneType]], uid: str, desc: str)
check: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], bool]
serialize_func: Callable[[JsonSerializer, Any, tuple[Union[str, int], ...]], Union[bool, int, float, str, list, Dict[str, Any], NoneType]]
def
serialize(self) -> dict:
91 def serialize(self) -> dict: 92 """serialize the handler info""" 93 return { 94 # get the code and doc of the check function 95 "check": { 96 "code": safe_getsource(self.check), 97 "doc": string_as_lines(self.check.__doc__), 98 }, 99 # get the code and doc of the load function 100 "serialize_func": { 101 "code": safe_getsource(self.serialize_func), 102 "doc": string_as_lines(self.serialize_func.__doc__), 103 }, 104 # get the uid, source_pckg, priority, and desc 105 "uid": str(self.uid), 106 "source_pckg": getattr(self.serialize_func, "source_pckg", None), 107 "__module__": getattr(self.serialize_func, "__module__", None), 108 "desc": str(self.desc), 109 }
serialize the handler info
BASE_HANDLERS: None =
(SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'))
DEFAULT_HANDLERS: None =
(SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function _serialize_override_serialize_func>, uid='.serialize override', desc='objects with .serialize method'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='namedtuple -> dict', desc='namedtuples as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dataclass -> dict', desc='dataclasses as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='path -> str', desc='Path objects as posix strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='obj -> str(obj)', desc='directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='numpy.ndarray', desc='numpy arrays'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='torch.Tensor', desc='pytorch tensors'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='pandas.DataFrame', desc='pandas DataFrames'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(set, list, tuple, Iterable) -> list', desc='sets, lists, tuples, and Iterables as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='fallback', desc='fallback handler -- serialize object attributes and special functions as strings'))
class
JsonSerializer:
234class JsonSerializer: 235 """Json serialization class (holds configs) 236 237 # Parameters: 238 - `array_mode : ArrayMode` 239 how to write arrays 240 (defaults to `"array_list_meta"`) 241 - `error_mode : ErrorMode` 242 what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") 243 (defaults to `"except"`) 244 - `handlers_pre : MonoTuple[SerializerHandler]` 245 handlers to use before the default handlers 246 (defaults to `tuple()`) 247 - `handlers_default : MonoTuple[SerializerHandler]` 248 default handlers to use 249 (defaults to `DEFAULT_HANDLERS`) 250 - `write_only_format : bool` 251 changes "__format__" keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) 252 (defaults to `False`) 253 254 # Raises: 255 - `ValueError`: on init, if `args` is not empty 256 - `SerializationException`: on `json_serialize()`, if any error occurs when trying to serialize an object and `error_mode` is set to `ErrorMode.EXCEPT"` 257 258 """ 259 260 def __init__( 261 self, 262 *args, 263 array_mode: ArrayMode = "array_list_meta", 264 error_mode: ErrorMode = ErrorMode.EXCEPT, 265 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 266 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 267 write_only_format: bool = False, 268 ): 269 if len(args) > 0: 270 raise ValueError( 271 f"JsonSerializer takes no positional arguments!\n{args = }" 272 ) 273 274 self.array_mode: ArrayMode = array_mode 275 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 276 self.write_only_format: bool = write_only_format 277 # join up the handlers 278 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 279 handlers_default 280 ) 281 282 def json_serialize( 283 self, 284 obj: Any, 285 path: ObjectPath = tuple(), 286 ) -> JSONitem: 287 try: 288 for handler in self.handlers: 289 if handler.check(self, obj, path): 290 output: JSONitem = handler.serialize_func(self, obj, path) 291 if self.write_only_format: 292 if isinstance(output, dict) and "__format__" in output: 293 new_fmt: JSONitem = output.pop("__format__") 294 output["__write_format__"] = new_fmt 295 return output 296 297 raise ValueError(f"no handler found for object with {type(obj) = }") 298 299 except Exception as e: 300 if self.error_mode == "except": 301 obj_str: str = repr(obj) 302 if len(obj_str) > 1000: 303 obj_str = obj_str[:1000] + "..." 304 raise SerializationException( 305 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 306 ) from e 307 elif self.error_mode == "warn": 308 warnings.warn( 309 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 310 ) 311 312 return repr(obj) 313 314 def hashify( 315 self, 316 obj: Any, 317 path: ObjectPath = tuple(), 318 force: bool = True, 319 ) -> Hashableitem: 320 """try to turn any object into something hashable""" 321 data = self.json_serialize(obj, path=path) 322 323 # recursive hashify, turning dicts and lists into tuples 324 return _recursive_hashify(data, force=force)
Json serialization class (holds configs)
Parameters:
array_mode : ArrayMode
how to write arrays (defaults to"array_list_meta"
)error_mode : ErrorMode
what to do when we can't serialize an object (will use repr as fallback if "ignore" or "warn") (defaults to"except"
)handlers_pre : MonoTuple[SerializerHandler]
handlers to use before the default handlers (defaults totuple()
)handlers_default : MonoTuple[SerializerHandler]
default handlers to use (defaults toDEFAULT_HANDLERS
)write_only_format : bool
changes "__format__" keys in output to "__write_format__" (when you want to serialize something in a way that zanj won't try to recover the object when loading) (defaults toFalse
)
Raises:
ValueError
: on init, ifargs
is not emptySerializationException
: onjson_serialize()
, if any error occurs when trying to serialize an object anderror_mode
is set toErrorMode.EXCEPT"
JsonSerializer( *args, array_mode: Literal['list', 'array_list_meta', 'array_hex_meta', 'array_b64_meta', 'external', 'zero_dim'] = 'array_list_meta', error_mode: muutils.errormode.ErrorMode = ErrorMode.Except, handlers_pre: None = (), handlers_default: None = (SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='base types', desc='base types (bool, int, float, str, None)'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dictionaries', desc='dictionaries'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(list, tuple) -> list', desc='lists and tuples as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function _serialize_override_serialize_func>, uid='.serialize override', desc='objects with .serialize method'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='namedtuple -> dict', desc='namedtuples as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='dataclass -> dict', desc='dataclasses as dicts'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='path -> str', desc='Path objects as posix strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='obj -> str(obj)', desc='directly serialize objects in `SERIALIZE_DIRECT_AS_STR` to strings'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='numpy.ndarray', desc='numpy arrays'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='torch.Tensor', desc='pytorch tensors'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='pandas.DataFrame', desc='pandas DataFrames'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='(set, list, tuple, Iterable) -> list', desc='sets, lists, tuples, and Iterables as lists'), SerializerHandler(check=<function <lambda>>, serialize_func=<function <lambda>>, uid='fallback', desc='fallback handler -- serialize object attributes and special functions as strings')), write_only_format: bool = False)
260 def __init__( 261 self, 262 *args, 263 array_mode: ArrayMode = "array_list_meta", 264 error_mode: ErrorMode = ErrorMode.EXCEPT, 265 handlers_pre: MonoTuple[SerializerHandler] = tuple(), 266 handlers_default: MonoTuple[SerializerHandler] = DEFAULT_HANDLERS, 267 write_only_format: bool = False, 268 ): 269 if len(args) > 0: 270 raise ValueError( 271 f"JsonSerializer takes no positional arguments!\n{args = }" 272 ) 273 274 self.array_mode: ArrayMode = array_mode 275 self.error_mode: ErrorMode = ErrorMode.from_any(error_mode) 276 self.write_only_format: bool = write_only_format 277 # join up the handlers 278 self.handlers: MonoTuple[SerializerHandler] = tuple(handlers_pre) + tuple( 279 handlers_default 280 )
array_mode: Literal['list', 'array_list_meta', 'array_hex_meta', 'array_b64_meta', 'external', 'zero_dim']
error_mode: muutils.errormode.ErrorMode
def
json_serialize( self, obj: Any, path: tuple[typing.Union[str, int], ...] = ()) -> Union[bool, int, float, str, list, Dict[str, Any], NoneType]:
282 def json_serialize( 283 self, 284 obj: Any, 285 path: ObjectPath = tuple(), 286 ) -> JSONitem: 287 try: 288 for handler in self.handlers: 289 if handler.check(self, obj, path): 290 output: JSONitem = handler.serialize_func(self, obj, path) 291 if self.write_only_format: 292 if isinstance(output, dict) and "__format__" in output: 293 new_fmt: JSONitem = output.pop("__format__") 294 output["__write_format__"] = new_fmt 295 return output 296 297 raise ValueError(f"no handler found for object with {type(obj) = }") 298 299 except Exception as e: 300 if self.error_mode == "except": 301 obj_str: str = repr(obj) 302 if len(obj_str) > 1000: 303 obj_str = obj_str[:1000] + "..." 304 raise SerializationException( 305 f"error serializing at {path = } with last handler: '{handler.uid}'\nfrom: {e}\nobj: {obj_str}" 306 ) from e 307 elif self.error_mode == "warn": 308 warnings.warn( 309 f"error serializing at {path = }, will return as string\n{obj = }\nexception = {e}" 310 ) 311 312 return repr(obj)
def
hashify( self, obj: Any, path: tuple[typing.Union[str, int], ...] = (), force: bool = True) -> Union[bool, int, float, str, tuple]:
314 def hashify( 315 self, 316 obj: Any, 317 path: ObjectPath = tuple(), 318 force: bool = True, 319 ) -> Hashableitem: 320 """try to turn any object into something hashable""" 321 data = self.json_serialize(obj, path=path) 322 323 # recursive hashify, turning dicts and lists into tuples 324 return _recursive_hashify(data, force=force)
try to turn any object into something hashable
def
json_serialize( obj: Any, path: tuple[typing.Union[str, int], ...] = ()) -> Union[bool, int, float, str, list, Dict[str, Any], NoneType]:
330def json_serialize(obj: Any, path: ObjectPath = tuple()) -> JSONitem: 331 """serialize object to json-serializable object with default config""" 332 return GLOBAL_JSON_SERIALIZER.json_serialize(obj, path=path)
serialize object to json-serializable object with default config