mirror of
https://github.com/msgpack/msgpack-python.git
synced 2025-11-01 10:00:54 +00:00
Add raw_as_bytes option to Unpacker. (#265)
This commit is contained in:
INADA Naoki
GitHub
parent
50ea49c86f
commit
5534d0c7af
11 changed files with 199 additions and 93 deletions
|
|
@ -43,8 +43,9 @@ from msgpack import ExtType
|
|||
cdef extern from "unpack.h":
|
||||
ctypedef struct msgpack_user:
|
||||
bint use_list
|
||||
PyObject* object_hook
|
||||
bint raw_as_bytes
|
||||
bint has_pairs_hook # call object_hook with k-v pairs
|
||||
PyObject* object_hook
|
||||
PyObject* list_hook
|
||||
PyObject* ext_hook
|
||||
char *encoding
|
||||
|
|
@ -73,12 +74,14 @@ cdef extern from "unpack.h":
|
|||
cdef inline init_ctx(unpack_context *ctx,
|
||||
object object_hook, object object_pairs_hook,
|
||||
object list_hook, object ext_hook,
|
||||
bint use_list, char* encoding, char* unicode_errors,
|
||||
bint use_list, bint raw_as_bytes,
|
||||
char* encoding, char* unicode_errors,
|
||||
Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
|
||||
Py_ssize_t max_array_len, Py_ssize_t max_map_len,
|
||||
Py_ssize_t max_ext_len):
|
||||
unpack_init(ctx)
|
||||
ctx.user.use_list = use_list
|
||||
ctx.user.raw_as_bytes = raw_as_bytes
|
||||
ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
|
||||
ctx.user.max_str_len = max_str_len
|
||||
ctx.user.max_bin_len = max_bin_len
|
||||
|
|
@ -155,7 +158,8 @@ cdef inline int get_data_from_buffer(object obj,
|
|||
return 1
|
||||
|
||||
def unpackb(object packed, object object_hook=None, object list_hook=None,
|
||||
bint use_list=1, encoding=None, unicode_errors="strict",
|
||||
bint use_list=True, bint raw_as_bytes=True,
|
||||
encoding=None, unicode_errors="strict",
|
||||
object_pairs_hook=None, ext_hook=ExtType,
|
||||
Py_ssize_t max_str_len=2147483647, # 2**32-1
|
||||
Py_ssize_t max_bin_len=2147483647,
|
||||
|
|
@ -180,21 +184,26 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
|
|||
cdef char* cerr = NULL
|
||||
cdef int new_protocol = 0
|
||||
|
||||
if encoding is not None:
|
||||
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
|
||||
if isinstance(encoding, unicode):
|
||||
encoding = encoding.encode('ascii')
|
||||
elif not isinstance(encoding, bytes):
|
||||
raise TypeError("encoding should be bytes or unicode")
|
||||
cenc = PyBytes_AsString(encoding)
|
||||
|
||||
if unicode_errors is not None:
|
||||
PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
|
||||
if isinstance(unicode_errors, unicode):
|
||||
unicode_errors = unicode_errors.encode('ascii')
|
||||
elif not isinstance(unicode_errors, bytes):
|
||||
raise TypeError("unicode_errors should be bytes or unicode")
|
||||
cerr = PyBytes_AsString(unicode_errors)
|
||||
|
||||
get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
|
||||
|
||||
try:
|
||||
if encoding is not None:
|
||||
if isinstance(encoding, unicode):
|
||||
encoding = encoding.encode('ascii')
|
||||
cenc = PyBytes_AsString(encoding)
|
||||
|
||||
if unicode_errors is not None:
|
||||
if isinstance(unicode_errors, unicode):
|
||||
unicode_errors = unicode_errors.encode('ascii')
|
||||
cerr = PyBytes_AsString(unicode_errors)
|
||||
|
||||
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
|
||||
use_list, cenc, cerr,
|
||||
use_list, raw_as_bytes, cenc, cerr,
|
||||
max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
|
||||
ret = unpack_construct(&ctx, buf, buf_len, &off)
|
||||
finally:
|
||||
|
|
@ -252,6 +261,16 @@ cdef class Unpacker(object):
|
|||
If true, unpack msgpack array to Python list.
|
||||
Otherwise, unpack to Python tuple. (default: True)
|
||||
|
||||
:param bool raw_as_bytes:
|
||||
If true, unpack msgpack raw to Python bytes (default).
|
||||
Otherwise, unpack to Python str (or unicode on Python 2) by decoding
|
||||
with UTF-8 encoding (recommended).
|
||||
Currently, the default is true, but it will be changed to false in
|
||||
near future. So you must specify it explicitly for keeping backward
|
||||
compatibility.
|
||||
|
||||
*encoding* option which is deprecated overrides this option.
|
||||
|
||||
:param callable object_hook:
|
||||
When specified, it should be callable.
|
||||
Unpacker calls it with a dict argument after unpacking msgpack map.
|
||||
|
|
@ -262,14 +281,6 @@ cdef class Unpacker(object):
|
|||
Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
|
||||
(See also simplejson)
|
||||
|
||||
:param str encoding:
|
||||
Encoding used for decoding msgpack raw.
|
||||
If it is None (default), msgpack raw is deserialized to Python bytes.
|
||||
|
||||
:param str unicode_errors:
|
||||
Used for decoding msgpack raw with *encoding*.
|
||||
(default: `'strict'`)
|
||||
|
||||
:param int max_buffer_size:
|
||||
Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
|
||||
Raises `BufferFull` exception when it is insufficient.
|
||||
|
|
@ -287,16 +298,25 @@ cdef class Unpacker(object):
|
|||
:param int max_map_len:
|
||||
Limits max length of map. (default: 2**31-1)
|
||||
|
||||
:param str encoding:
|
||||
Deprecated, use raw_as_bytes instead.
|
||||
Encoding used for decoding msgpack raw.
|
||||
If it is None (default), msgpack raw is deserialized to Python bytes.
|
||||
|
||||
example of streaming deserialize from file-like object::
|
||||
:param str unicode_errors:
|
||||
Deprecated. Used for decoding msgpack raw with *encoding*.
|
||||
(default: `'strict'`)
|
||||
|
||||
unpacker = Unpacker(file_like)
|
||||
|
||||
Example of streaming deserialize from file-like object::
|
||||
|
||||
unpacker = Unpacker(file_like, raw_as_bytes=False)
|
||||
for o in unpacker:
|
||||
process(o)
|
||||
|
||||
example of streaming deserialize from socket::
|
||||
Example of streaming deserialize from socket::
|
||||
|
||||
unpacker = Unpacker()
|
||||
unpacker = Unpacker(raw_as_bytes=False)
|
||||
while True:
|
||||
buf = sock.recv(1024**2)
|
||||
if not buf:
|
||||
|
|
@ -324,7 +344,8 @@ cdef class Unpacker(object):
|
|||
PyMem_Free(self.buf)
|
||||
self.buf = NULL
|
||||
|
||||
def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
|
||||
def __init__(self, file_like=None, Py_ssize_t read_size=0,
|
||||
bint use_list=True, bint raw_as_bytes=True,
|
||||
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
|
||||
encoding=None, unicode_errors='strict', int max_buffer_size=0,
|
||||
object ext_hook=ExtType,
|
||||
|
|
@ -363,6 +384,7 @@ cdef class Unpacker(object):
|
|||
self.stream_offset = 0
|
||||
|
||||
if encoding is not None:
|
||||
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
|
||||
if isinstance(encoding, unicode):
|
||||
self.encoding = encoding.encode('ascii')
|
||||
elif isinstance(encoding, bytes):
|
||||
|
|
@ -372,6 +394,7 @@ cdef class Unpacker(object):
|
|||
cenc = PyBytes_AsString(self.encoding)
|
||||
|
||||
if unicode_errors is not None:
|
||||
PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
|
||||
if isinstance(unicode_errors, unicode):
|
||||
self.unicode_errors = unicode_errors.encode('ascii')
|
||||
elif isinstance(unicode_errors, bytes):
|
||||
|
|
@ -381,7 +404,7 @@ cdef class Unpacker(object):
|
|||
cerr = PyBytes_AsString(self.unicode_errors)
|
||||
|
||||
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
|
||||
ext_hook, use_list, cenc, cerr,
|
||||
ext_hook, use_list, raw_as_bytes, cenc, cerr,
|
||||
max_str_len, max_bin_len, max_array_len,
|
||||
max_map_len, max_ext_len)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue