Stowage/msgpack-python
2
0
Fork 0
mirror of https://github.com/msgpack/msgpack-python.git synced 2025-10-28 16:14:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

precise_mode instead of distinguish_tuple

Browse source 
When precise_mode flag is set, serialization will be as precise as
possible - type checks will be exact (type(..) is ... instead of
isinstance(..., ...) and tuple will be treated as undefined type. This
mode is to make accurate object serialization possible.
This commit is contained in:
faerot 2014-05-22 16:45:26 +03:00
parent 3b933f0966
commit b877ce2afa
2 changed files with 51 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

37
msgpack/_packer.pyx
View file

@ -56,10 +56,13 @@ cdef class Packer(object):
Convert unicode to bytes with this encoding. (default: 'utf-8') Convert unicode to bytes with this encoding. (default: 'utf-8')
:param str unicode_errors: :param str unicode_errors:
Error handler for encoding unicode. (default: 'strict') Error handler for encoding unicode. (default: 'strict')
:param bool distinguish_tuple: :param bool precise_mode:
If set to true, tuples will not be serialized as lists If set to true, types will be checked to be exact. Derived classes
and will be treated as unsupported type. This is useful when trying from serializeable types will not be serialized and will be
to implement accurate serialization for python types. treated as unsupported type and forwarded to default.
Additionally tuples will not be serialized as lists.
This is useful when trying to implement accurate serialization
for python types.
:param bool use_single_float: :param bool use_single_float:
Use single precision float type for float. (default: False) Use single precision float type for float. (default: False)
:param bool autoreset: :param bool autoreset:
@ -75,7 +78,7 @@ cdef class Packer(object):
cdef object _berrors cdef object _berrors
cdef char *encoding cdef char *encoding
cdef char *unicode_errors cdef char *unicode_errors
cdef bool distinguish_tuple cdef bint precise_mode
cdef bool use_float cdef bool use_float
cdef bint autoreset cdef bint autoreset
@ -88,12 +91,12 @@ cdef class Packer(object):
self.pk.length = 0 self.pk.length = 0
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
distinguish_tuple=False, use_single_float=False, bint autoreset=1, use_single_float=False, bint autoreset=1, bint use_bin_type=0,
bint use_bin_type=0): bint precise_mode=0):
""" """
""" """
self.use_float = use_single_float self.use_float = use_single_float
self.distinguish_tuple = distinguish_tuple self.precise_mode = precise_mode
self.autoreset = autoreset self.autoreset = autoreset
self.pk.use_bin_type = use_bin_type self.pk.use_bin_type = use_bin_type
if default is not None: if default is not None:
@ -129,7 +132,7 @@ cdef class Packer(object):
cdef dict d cdef dict d
cdef size_t L cdef size_t L
cdef int default_used = 0 cdef int default_used = 0
cdef bool distinguish_tuple = self.distinguish_tuple cdef bint precise = self.precise_mode
if nest_limit < 0: if nest_limit < 0:
raise PackValueError("recursion limit exceeded.") raise PackValueError("recursion limit exceeded.")
@ -137,12 +140,12 @@ cdef class Packer(object):
while True: while True:
if o is None: if o is None:
ret = msgpack_pack_nil(&self.pk) ret = msgpack_pack_nil(&self.pk)
elif isinstance(o, bool): elif PyBool_Check(o) if precise else isinstance(o, bool):
if o: if o:
ret = msgpack_pack_true(&self.pk) ret = msgpack_pack_true(&self.pk)
else: else:
ret = msgpack_pack_false(&self.pk) ret = msgpack_pack_false(&self.pk)
elif PyLong_Check(o): elif PyLong_CheckExact(o) if precise else PyLong_Check(o):
# PyInt_Check(long) is True for Python 3. # PyInt_Check(long) is True for Python 3.
# Sow we should test long before int. # Sow we should test long before int.
if o > 0: if o > 0:
@ -151,17 +154,17 @@ cdef class Packer(object):
else: else:
llval = o llval = o
ret = msgpack_pack_long_long(&self.pk, llval) ret = msgpack_pack_long_long(&self.pk, llval)
elif PyInt_Check(o): elif PyInt_CheckExact(o) if precise else PyInt_Check(o):
longval = o longval = o
ret = msgpack_pack_long(&self.pk, longval) ret = msgpack_pack_long(&self.pk, longval)
elif PyFloat_Check(o): elif PyFloat_CheckExact(o) if precise else PyFloat_Check(o):
if self.use_float: if self.use_float:
fval = o fval = o
ret = msgpack_pack_float(&self.pk, fval) ret = msgpack_pack_float(&self.pk, fval)
else: else:
dval = o dval = o
ret = msgpack_pack_double(&self.pk, dval) ret = msgpack_pack_double(&self.pk, dval)
elif PyBytes_Check(o): elif PyBytes_CheckExact(o) if precise else PyBytes_Check(o):
L = len(o) L = len(o)
if L > (2**32)-1: if L > (2**32)-1:
raise ValueError("bytes is too large") raise ValueError("bytes is too large")
@ -169,7 +172,7 @@ cdef class Packer(object):
ret = msgpack_pack_bin(&self.pk, L) ret = msgpack_pack_bin(&self.pk, L)
if ret == 0: if ret == 0:
ret = msgpack_pack_raw_body(&self.pk, rawval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L)
elif PyUnicode_Check(o): elif PyUnicode_CheckExact(o) if precise else PyUnicode_Check(o):
if not self.encoding: if not self.encoding:
raise TypeError("Can't encode unicode string: no encoding is specified") raise TypeError("Can't encode unicode string: no encoding is specified")
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
@ -192,7 +195,7 @@ cdef class Packer(object):
if ret != 0: break if ret != 0: break
ret = self._pack(v, nest_limit-1) ret = self._pack(v, nest_limit-1)
if ret != 0: break if ret != 0: break
elif PyDict_Check(o): elif not precise and PyDict_Check(o):
L = len(o) L = len(o)
if L > (2**32)-1: if L > (2**32)-1:
raise ValueError("dict is too large") raise ValueError("dict is too large")
@ -212,7 +215,7 @@ cdef class Packer(object):
raise ValueError("EXT data is too large") raise ValueError("EXT data is too large")
ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_ext(&self.pk, longval, L)
ret = msgpack_pack_raw_body(&self.pk, rawval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L)
elif (PyTuple_Check(o) and not distinguish_tuple) or PyList_Check(o): elif PyList_CheckExact(o) if precise else (PyTuple_Check(o) or PyList_Check(o)):
L = len(o) L = len(o)
if L > (2**32)-1: if L > (2**32)-1:
raise ValueError("list is too large") raise ValueError("list is too large")

48
msgpack/fallback.py
View file

@ -485,10 +485,13 @@ class Packer(object):
Convert unicode to bytes with this encoding. (default: 'utf-8') Convert unicode to bytes with this encoding. (default: 'utf-8')
:param str unicode_errors: :param str unicode_errors:
Error handler for encoding unicode. (default: 'strict') Error handler for encoding unicode. (default: 'strict')
:param bool distinguish_tuple: :param bool precise_mode:
If set to true, tuples will not be serialized as lists If set to true, types will be checked to be exact. Derived classes
and will be treated as unsupported type. This is useful when trying from serializeable types will not be serialized and will be
to implement accurate serialization for python types. treated as unsupported type and forwarded to default.
Additionally tuples will not be serialized as lists.
This is useful when trying to implement accurate serialization
for python types.
:param bool use_single_float: :param bool use_single_float:
Use single precision float type for float. (default: False) Use single precision float type for float. (default: False)
:param bool autoreset: :param bool autoreset:
@ -499,9 +502,9 @@ class Packer(object):
It also enable str8 type for unicode. It also enable str8 type for unicode.
""" """
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
distinguish_tuple=False, use_single_float=False, autoreset=True, precise_mode=False, use_single_float=False, autoreset=True,
use_bin_type=False): use_bin_type=False):
self._distinguish_tuple = distinguish_tuple self._precise_mode = precise_mode
self._use_float = use_single_float self._use_float = use_single_float
self._autoreset = autoreset self._autoreset = autoreset
self._use_bin_type = use_bin_type self._use_bin_type = use_bin_type
@ -513,19 +516,30 @@ class Packer(object):
raise TypeError("default must be callable") raise TypeError("default must be callable")
self._default = default self._default = default
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): def _check_precise(obj, t, type=type, tuple=tuple):
if type(t) is tuple:
return type(obj) in t
else:
return type(obj) is t
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
check=isinstance, check_precise=_check_precise):
default_used = False default_used = False
list_type = list if self._distinguish_tuple else (list, tuple) if self._precise_mode:
check = check_precise
list_types = list
else:
list_types = (list, tuple)
while True: while True:
if nest_limit < 0: if nest_limit < 0:
raise PackValueError("recursion limit exceeded") raise PackValueError("recursion limit exceeded")
if obj is None: if obj is None:
return self._buffer.write(b"\xc0") return self._buffer.write(b"\xc0")
if isinstance(obj, bool): if check(obj, bool):
if obj: if obj:
return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc3")
return self._buffer.write(b"\xc2") return self._buffer.write(b"\xc2")
if isinstance(obj, int_types): if check(obj, int_types):
if 0 <= obj < 0x80: if 0 <= obj < 0x80:
return self._buffer.write(struct.pack("B", obj)) return self._buffer.write(struct.pack("B", obj))
if -0x20 <= obj < 0: if -0x20 <= obj < 0:
@ -547,7 +561,7 @@ class Packer(object):
if -0x8000000000000000 <= obj < -0x80000000: if -0x8000000000000000 <= obj < -0x80000000:
return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
raise PackValueError("Integer value out of range") raise PackValueError("Integer value out of range")
if self._use_bin_type and isinstance(obj, bytes): if self._use_bin_type and check(obj, bytes):
n = len(obj) n = len(obj)
if n <= 0xff: if n <= 0xff:
self._buffer.write(struct.pack('>BB', 0xc4, n)) self._buffer.write(struct.pack('>BB', 0xc4, n))
@ -558,8 +572,8 @@ class Packer(object):
else: else:
raise PackValueError("Bytes is too large") raise PackValueError("Bytes is too large")
return self._buffer.write(obj) return self._buffer.write(obj)
if isinstance(obj, (Unicode, bytes)): if check(obj, (Unicode, bytes)):
if isinstance(obj, Unicode): if check(obj, Unicode):
if self._encoding is None: if self._encoding is None:
raise TypeError( raise TypeError(
"Can't encode unicode string: " "Can't encode unicode string: "
@ -577,11 +591,11 @@ class Packer(object):
else: else:
raise PackValueError("String is too large") raise PackValueError("String is too large")
return self._buffer.write(obj) return self._buffer.write(obj)
if isinstance(obj, float): if check(obj, float):
if self._use_float: if self._use_float:
return self._buffer.write(struct.pack(">Bf", 0xca, obj)) return self._buffer.write(struct.pack(">Bf", 0xca, obj))
return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
if isinstance(obj, ExtType): if check(obj, ExtType):
code = obj.code code = obj.code
data = obj.data data = obj.data
assert isinstance(code, int) assert isinstance(code, int)
@ -606,13 +620,13 @@ class Packer(object):
self._buffer.write(struct.pack("b", code)) self._buffer.write(struct.pack("b", code))
self._buffer.write(data) self._buffer.write(data)
return return
if isinstance(obj, list_type): if check(obj, list_types):
n = len(obj) n = len(obj)
self._fb_pack_array_header(n) self._fb_pack_array_header(n)
for i in xrange(n): for i in xrange(n):
self._pack(obj[i], nest_limit - 1) self._pack(obj[i], nest_limit - 1)
return return
if isinstance(obj, dict): if check(obj, dict):
return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
nest_limit - 1) nest_limit - 1)
if not default_used and self._default is not None: if not default_used and self._default is not None: