Stowage/msgpack-python
2
0
Fork 0
mirror of https://github.com/msgpack/msgpack-python.git synced 2025-10-24 06:13:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

Unpacker: Change max_buffer_size to 100MiB (#391)

Browse source
This commit is contained in:
Inada Naoki 2019-12-09 17:03:12 +09:00 • committed by GitHub
parent 5399f8180d
commit c356035a57
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 40 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

42
msgpack/_unpacker.pyx
View file

@ -234,27 +234,28 @@ cdef class Unpacker(object):
(See also simplejson) (See also simplejson)
:param int max_buffer_size: :param int max_buffer_size:
Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Limits size of data waiting unpacked. 0 means system's INT_MAX.
The default value is 100*1024*1024 (100MiB).
Raises `BufferFull` exception when it is insufficient. Raises `BufferFull` exception when it is insufficient.
You should set this parameter when unpacking data from untrusted source. You should set this parameter when unpacking data from untrusted source.
:param int max_str_len: :param int max_str_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max length of str. (default: max_buffer_size or 1024*1024) Limits max length of str. (default: max_buffer_size)
:param int max_bin_len: :param int max_bin_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max length of bin. (default: max_buffer_size or 1024*1024) Limits max length of bin. (default: max_buffer_size)
:param int max_array_len: :param int max_array_len:
Limits max length of array. (default: max_buffer_size or 128*1024) Limits max length of array. (default: max_buffer_size)
:param int max_map_len: :param int max_map_len:
Limits max length of map. (default: max_buffer_size//2 or 32*1024) Limits max length of map. (default: max_buffer_size//2)
:param int max_ext_len: :param int max_ext_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max size of ext type. (default: max_buffer_size or 1024*1024) Limits max size of ext type. (default: max_buffer_size)
:param str unicode_errors: :param str unicode_errors:
Error handler used for decoding str type. (default: `'strict'`) Error handler used for decoding str type. (default: `'strict'`)
@ -262,13 +263,13 @@ cdef class Unpacker(object):
Example of streaming deserialize from file-like object:: Example of streaming deserialize from file-like object::
unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) unpacker = Unpacker(file_like)
for o in unpacker: for o in unpacker:
process(o) process(o)
Example of streaming deserialize from socket:: Example of streaming deserialize from socket::
unpacker = Unpacker(max_buffer_size=10*1024*1024) unpacker = Unpacker()
while True: while True:
buf = sock.recv(1024**2) buf = sock.recv(1024**2)
if not buf: if not buf:
@ -305,7 +306,7 @@ cdef class Unpacker(object):
def __init__(self, file_like=None, *, Py_ssize_t read_size=0, def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
bint use_list=True, bint raw=False, bint strict_map_key=True, bint use_list=True, bint raw=False, bint strict_map_key=True,
object object_hook=None, object object_pairs_hook=None, object list_hook=None, object object_hook=None, object object_pairs_hook=None, object list_hook=None,
unicode_errors=None, Py_ssize_t max_buffer_size=0, unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024,
object ext_hook=ExtType, object ext_hook=ExtType,
Py_ssize_t max_str_len=-1, Py_ssize_t max_str_len=-1,
Py_ssize_t max_bin_len=-1, Py_ssize_t max_bin_len=-1,
@ -325,23 +326,24 @@ cdef class Unpacker(object):
if not PyCallable_Check(self.file_like_read): if not PyCallable_Check(self.file_like_read):
raise TypeError("`file_like.read` must be a callable.") raise TypeError("`file_like.read` must be a callable.")
if max_str_len == -1:
max_str_len = max_buffer_size or 1024*1024
if max_bin_len == -1:
max_bin_len = max_buffer_size or 1024*1024
if max_array_len == -1:
max_array_len = max_buffer_size or 128*1024
if max_map_len == -1:
max_map_len = max_buffer_size//2 or 32*1024
if max_ext_len == -1:
max_ext_len = max_buffer_size or 1024*1024
if not max_buffer_size: if not max_buffer_size:
max_buffer_size = INT_MAX max_buffer_size = INT_MAX
if max_str_len == -1:
max_str_len = max_buffer_size
if max_bin_len == -1:
max_bin_len = max_buffer_size
if max_array_len == -1:
max_array_len = max_buffer_size
if max_map_len == -1:
max_map_len = max_buffer_size//2
if max_ext_len == -1:
max_ext_len = max_buffer_size
if read_size > max_buffer_size: if read_size > max_buffer_size:
raise ValueError("read_size should be less or equal to max_buffer_size") raise ValueError("read_size should be less or equal to max_buffer_size")
if not read_size: if not read_size:
read_size = min(max_buffer_size, 1024**2) read_size = min(max_buffer_size, 1024**2)
self.max_buffer_size = max_buffer_size self.max_buffer_size = max_buffer_size
self.read_size = read_size self.read_size = read_size
self.buf = <char*>PyMem_Malloc(read_size) self.buf = <char*>PyMem_Malloc(read_size)

33
msgpack/fallback.py
View file

@ -193,39 +193,40 @@ class Unpacker(object):
contains invalid UTF-8 string. contains invalid UTF-8 string.
:param int max_buffer_size: :param int max_buffer_size:
Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Limits size of data waiting unpacked. 0 means 2**32-1.
The default value is 100*1024*1024 (100MiB).
Raises `BufferFull` exception when it is insufficient. Raises `BufferFull` exception when it is insufficient.
You should set this parameter when unpacking data from untrusted source. You should set this parameter when unpacking data from untrusted source.
:param int max_str_len: :param int max_str_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max length of str. (default: max_buffer_size or 1024*1024) Limits max length of str. (default: max_buffer_size)
:param int max_bin_len: :param int max_bin_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max length of bin. (default: max_buffer_size or 1024*1024) Limits max length of bin. (default: max_buffer_size)
:param int max_array_len: :param int max_array_len:
Limits max length of array. Limits max length of array.
(default: max_buffer_size or 128*1024) (default: max_buffer_size)
:param int max_map_len: :param int max_map_len:
Limits max length of map. Limits max length of map.
(default: max_buffer_size//2 or 32*1024) (default: max_buffer_size//2)
:param int max_ext_len: :param int max_ext_len:
Deprecated, use *max_buffer_size* instead. Deprecated, use *max_buffer_size* instead.
Limits max size of ext type. (default: max_buffer_size or 1024*1024) Limits max size of ext type. (default: max_buffer_size)
Example of streaming deserialize from file-like object:: Example of streaming deserialize from file-like object::
unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) unpacker = Unpacker(file_like)
for o in unpacker: for o in unpacker:
process(o) process(o)
Example of streaming deserialize from socket:: Example of streaming deserialize from socket::
unpacker = Unpacker(max_buffer_size=10*1024*1024) unpacker = Unpacker(max_buffer_size)
while True: while True:
buf = sock.recv(1024**2) buf = sock.recv(1024**2)
if not buf: if not buf:
@ -252,7 +253,7 @@ class Unpacker(object):
object_pairs_hook=None, object_pairs_hook=None,
list_hook=None, list_hook=None,
unicode_errors=None, unicode_errors=None,
max_buffer_size=0, max_buffer_size=100 * 1024 * 1024,
ext_hook=ExtType, ext_hook=ExtType,
max_str_len=-1, max_str_len=-1,
max_bin_len=-1, max_bin_len=-1,
@ -285,18 +286,20 @@ class Unpacker(object):
# state, which _buf_checkpoint records. # state, which _buf_checkpoint records.
self._buf_checkpoint = 0 self._buf_checkpoint = 0
if not max_buffer_size:
max_buffer_size = 2 ** 31 - 1
if max_str_len == -1: if max_str_len == -1:
max_str_len = max_buffer_size or 1024 * 1024 max_str_len = max_buffer_size
if max_bin_len == -1: if max_bin_len == -1:
max_bin_len = max_buffer_size or 1024 * 1024 max_bin_len = max_buffer_size
if max_array_len == -1: if max_array_len == -1:
max_array_len = max_buffer_size or 128 * 1024 max_array_len = max_buffer_size
if max_map_len == -1: if max_map_len == -1:
max_map_len = max_buffer_size // 2 or 32 * 1024 max_map_len = max_buffer_size // 2
if max_ext_len == -1: if max_ext_len == -1:
max_ext_len = max_buffer_size or 1024 * 1024 max_ext_len = max_buffer_size
self._max_buffer_size = max_buffer_size or 2 ** 31 - 1 self._max_buffer_size = max_buffer_size
if read_size > self._max_buffer_size: if read_size > self._max_buffer_size:
raise ValueError("read_size must be smaller than max_buffer_size") raise ValueError("read_size must be smaller than max_buffer_size")
self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)