Stowage/msgpack-python
2
0
Fork 0
mirror of https://github.com/msgpack/msgpack-python.git synced 2025-10-25 14:44:10 +00:00
Code Issues Projects Releases Packages Wiki Activity

s/raw_as_bytes/raw/g (#276)

Browse source 
fixes #273
This commit is contained in:
INADA Naoki 2018-01-12 19:22:36 +09:00 • committed by GitHub
parent d9ec8fc905
commit 5569a4efcd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 48 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -18,7 +18,8 @@ serve-doc: all
.PHONY: clean
clean:
rm -rf build
rm msgpack/*.so
rm -f msgpack/_packer.cpp
rm -f msgpack/_unpacker.cpp
rm -rf msgpack/__pycache__
rm -rf test/__pycache__

24
README.rst
View file

@ -47,9 +47,9 @@ In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommende
For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes``
object into msgpack raw type.
In case of unpacker, there is new ``raw_as_bytes`` option. It is ``True`` by default
In case of unpacker, there is new ``raw`` option. It is ``True`` by default
for backward compatibility, but it is changed to ``False`` in near future.
You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``.
You can use ``raw=False`` instead of ``encoding='utf-8'``.
Planned backward incompatible changes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -58,14 +58,14 @@ When msgpack 1.0, I planning these breaking changes:
* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option.
* packer: Change default of ``use_bin_type`` option from False to True.
* unpacker: Change default of ``raw_as_bytes`` option from True to False.
* unpacker: Change default of ``raw`` option from True to False.
* unpacker: Reduce all ``max_xxx_len`` options for typical usage.
* unpacker: Remove ``write_bytes`` option from all methods.
To avoid these breaking changes breaks your application, please:
* Don't use deprecated options.
* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly.
* Pass ``use_bin_type`` and ``raw`` options explicitly.
* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too.
@ -113,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
>>> import msgpack
>>> msgpack.packb([1, 2, 3], use_bin_type=True)
'\x93\x01\x02\x03'
>>> msgpack.unpackb(_, raw_as_bytes=False)
>>> msgpack.unpackb(_, raw=False)
[1, 2, 3]
``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
.. code-block:: pycon
>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False)
>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
(1, 2, 3)
You should always specify the ``use_list`` keyword argument for backward compatibility.
@ -146,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method).
buf.seek(0)
unpacker = msgpack.Unpacker(buf, raw_as_bytes=False)
unpacker = msgpack.Unpacker(buf, raw=False)
for unpacked in unpacker:
print(unpacked)
@ -179,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False)
this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
``Unpacker``'s ``object_hook`` callback receives a dict; the
``object_pairs_hook`` callback may instead be used to receive a list of
@ -209,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type.
...
>>> data = array.array('d', [1.2, 3.4])
>>> packed = msgpack.packb(data, default=default, use_bin_type=True)
>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False)
>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
>>> data == unpacked
True
@ -257,7 +257,7 @@ For backward compatibility reasons, msgpack-python will still default all
strings to byte strings, unless you specify the ``use_bin_type=True`` option in
the packer. If you do so, it will use a non-standard type called **bin** to
serialize byte arrays, and **raw** becomes to mean **str**. If you want to
distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``.
distinguish **bin** and **raw** in the unpacker, specify ``raw=False``.
Note that Python 2 defaults to byte-arrays over Unicode strings:
@ -267,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings:
>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
['spam', 'eggs']
>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
raw_as_bytes=False)
raw=False)
['spam', u'eggs']
This is the same code in Python 3 (same behaviour, but Python 3 has a
@ -279,7 +279,7 @@ different default):
>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
[b'spam', b'eggs']
>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
raw_as_bytes=False)
raw=False)
[b'spam', 'eggs']

26
msgpack/_unpacker.pyx
View file

@ -43,7 +43,7 @@ from msgpack import ExtType
cdef extern from "unpack.h":
ctypedef struct msgpack_user:
bint use_list
bint raw_as_bytes
bint raw
bint has_pairs_hook # call object_hook with k-v pairs
PyObject* object_hook
PyObject* list_hook
@ -74,14 +74,14 @@ cdef extern from "unpack.h":
cdef inline init_ctx(unpack_context *ctx,
object object_hook, object object_pairs_hook,
object list_hook, object ext_hook,
bint use_list, bint raw_as_bytes,
bint use_list, bint raw,
char* encoding, char* unicode_errors,
Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
Py_ssize_t max_array_len, Py_ssize_t max_map_len,
Py_ssize_t max_ext_len):
unpack_init(ctx)
ctx.user.use_list = use_list
ctx.user.raw_as_bytes = raw_as_bytes
ctx.user.raw = raw
ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
ctx.user.max_str_len = max_str_len
ctx.user.max_bin_len = max_bin_len
@ -158,7 +158,7 @@ cdef inline int get_data_from_buffer(object obj,
return 1
def unpackb(object packed, object object_hook=None, object list_hook=None,
bint use_list=True, bint raw_as_bytes=True,
bint use_list=True, bint raw=True,
encoding=None, unicode_errors="strict",
object_pairs_hook=None, ext_hook=ExtType,
Py_ssize_t max_str_len=2147483647, # 2**32-1
@ -185,7 +185,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
cdef int new_protocol = 0
if encoding is not None:
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
if isinstance(encoding, unicode):
encoding = encoding.encode('ascii')
elif not isinstance(encoding, bytes):
@ -203,7 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
try:
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
use_list, raw_as_bytes, cenc, cerr,
use_list, raw, cenc, cerr,
max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
ret = unpack_construct(&ctx, buf, buf_len, &off)
finally:
@ -261,7 +261,7 @@ cdef class Unpacker(object):
If true, unpack msgpack array to Python list.
Otherwise, unpack to Python tuple. (default: True)
:param bool raw_as_bytes:
:param bool raw:
If true, unpack msgpack raw to Python bytes (default).
Otherwise, unpack to Python str (or unicode on Python 2) by decoding
with UTF-8 encoding (recommended).
@ -299,7 +299,7 @@ cdef class Unpacker(object):
Limits max length of map. (default: 2**31-1)
:param str encoding:
Deprecated, use raw_as_bytes instead.
Deprecated, use raw instead.
Encoding used for decoding msgpack raw.
If it is None (default), msgpack raw is deserialized to Python bytes.
@ -310,13 +310,13 @@ cdef class Unpacker(object):
Example of streaming deserialize from file-like object::
unpacker = Unpacker(file_like, raw_as_bytes=False)
unpacker = Unpacker(file_like, raw=False)
for o in unpacker:
process(o)
Example of streaming deserialize from socket::
unpacker = Unpacker(raw_as_bytes=False)
unpacker = Unpacker(raw=False)
while True:
buf = sock.recv(1024**2)
if not buf:
@ -345,7 +345,7 @@ cdef class Unpacker(object):
self.buf = NULL
def __init__(self, file_like=None, Py_ssize_t read_size=0,
bint use_list=True, bint raw_as_bytes=True,
bint use_list=True, bint raw=True,
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
encoding=None, unicode_errors='strict', int max_buffer_size=0,
object ext_hook=ExtType,
@ -384,7 +384,7 @@ cdef class Unpacker(object):
self.stream_offset = 0
if encoding is not None:
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
if isinstance(encoding, unicode):
self.encoding = encoding.encode('ascii')
elif isinstance(encoding, bytes):
@ -404,7 +404,7 @@ cdef class Unpacker(object):
cerr = PyBytes_AsString(self.unicode_errors)
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
ext_hook, use_list, raw_as_bytes, cenc, cerr,
ext_hook, use_list, raw, cenc, cerr,
max_str_len, max_bin_len, max_array_len,
max_map_len, max_ext_len)

16
msgpack/fallback.py
View file

@ -145,7 +145,7 @@ class Unpacker(object):
If true, unpack msgpack array to Python list.
Otherwise, unpack to Python tuple. (default: True)
:param bool raw_as_bytes:
:param bool raw:
If true, unpack msgpack raw to Python bytes (default).
Otherwise, unpack to Python str (or unicode on Python 2) by decoding
with UTF-8 encoding (recommended).
@ -193,13 +193,13 @@ class Unpacker(object):
example of streaming deserialize from file-like object::
unpacker = Unpacker(file_like, raw_as_bytes=False)
unpacker = Unpacker(file_like, raw=False)
for o in unpacker:
process(o)
example of streaming deserialize from socket::
unpacker = Unpacker(raw_as_bytes=False)
unpacker = Unpacker(raw=False)
while True:
buf = sock.recv(1024**2)
if not buf:
@ -209,7 +209,7 @@ class Unpacker(object):
process(o)
"""
def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True,
def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
object_hook=None, object_pairs_hook=None, list_hook=None,
encoding=None, unicode_errors=None, max_buffer_size=0,
ext_hook=ExtType,
@ -221,7 +221,7 @@ class Unpacker(object):
if encoding is not None:
warnings.warn(
"encoding is deprecated, Use raw_as_bytes=False instead.",
"encoding is deprecated, Use raw=False instead.",
PendingDeprecationWarning)
if unicode_errors is not None:
@ -257,7 +257,7 @@ class Unpacker(object):
if read_size > self._max_buffer_size:
raise ValueError("read_size must be smaller than max_buffer_size")
self._read_size = read_size or min(self._max_buffer_size, 16*1024)
self._raw_as_bytes = bool(raw_as_bytes)
self._raw = bool(raw)
self._encoding = encoding
self._unicode_errors = unicode_errors
self._use_list = use_list
@ -606,7 +606,7 @@ class Unpacker(object):
if typ == TYPE_RAW:
if self._encoding is not None:
obj = obj.decode(self._encoding, self._unicode_errors)
elif self._raw_as_bytes:
elif self._raw:
obj = bytes(obj)
else:
obj = obj.decode('utf_8')
@ -715,7 +715,7 @@ class Packer(object):
encoding = 'utf_8'
else:
warnings.warn(
"encoding is deprecated, Use raw_as_bytes=False instead.",
"encoding is deprecated, Use raw=False instead.",
PendingDeprecationWarning)
if unicode_errors is None:

4
msgpack/unpack.h
View file

@ -21,7 +21,7 @@
typedef struct unpack_user {
bool use_list;
bool raw_as_bytes;
bool raw;
bool has_pairs_hook;
PyObject *object_hook;
PyObject *list_hook;
@ -229,7 +229,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
if (u->encoding) {
py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
} else if (u->raw_as_bytes) {
} else if (u->raw) {
py = PyBytes_FromStringAndSize(p, l);
} else {
py = PyUnicode_DecodeUTF8(p, l, NULL);

4
test/test_limits.py
View file

@ -39,11 +39,11 @@ def test_max_str_len():
d = 'x' * 3
packed = packb(d)
unpacker = Unpacker(max_str_len=3, raw_as_bytes=False)
unpacker = Unpacker(max_str_len=3, raw=False)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_str_len=2, raw_as_bytes=False)
unpacker = Unpacker(max_str_len=2, raw=False)
with pytest.raises(UnpackValueError):
unpacker.feed(packed)
unpacker.unpack()

8
test/test_pack.py
View file

@ -31,11 +31,11 @@ def testPack():
def testPackUnicode():
test_data = ["", "abcd", ["defgh"], "Русский текст"]
for td in test_data:
re = unpackb(packb(td), use_list=1, raw_as_bytes=False)
re = unpackb(packb(td), use_list=1, raw=False)
assert re == td
packer = Packer()
data = packer.pack(td)
re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack()
re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack()
assert re == td
def testPackUTF32(): # deprecated
@ -72,14 +72,14 @@ def testIgnoreUnicodeErrors(): # deprecated
def testStrictUnicodeUnpack():
with raises(UnicodeDecodeError):
unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1)
unpackb(packb(b'abc\xeddef'), raw=False, use_list=1)
def testStrictUnicodePack(): # deprecated
with raises(UnicodeEncodeError):
packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
def testIgnoreErrorsPack(): # deprecated
re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1)
re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
assert re == "abcdef"
def testDecodeBinary():

8
test/test_stricttype.py
View file

@ -11,7 +11,7 @@ def test_namedtuple():
return dict(o._asdict())
raise TypeError('Unsupported type %s' % (type(o),))
packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
unpacked = unpackb(packed, raw_as_bytes=False)
unpacked = unpackb(packed, raw=False)
assert unpacked == {'foo': 1, 'bar': 42}
@ -32,7 +32,7 @@ def test_tuple():
return o
data = packb(t, strict_types=True, use_bin_type=True, default=default)
expected = unpackb(data, raw_as_bytes=False, object_hook=convert)
expected = unpackb(data, raw=False, object_hook=convert)
assert expected == t
@ -53,10 +53,10 @@ def test_tuple_ext():
def convert(code, payload):
if code == MSGPACK_EXT_TYPE_TUPLE:
# Unpack and convert to tuple
return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert))
return tuple(unpackb(payload, raw=False, ext_hook=convert))
raise ValueError('Unknown Ext code {}'.format(code))
data = packb(t, strict_types=True, use_bin_type=True, default=default)
expected = unpackb(data, raw_as_bytes=False, ext_hook=convert)
expected = unpackb(data, raw=False, ext_hook=convert)
assert expected == t

2
test/test_unpack.py
View file

@ -48,7 +48,7 @@ def test_unpacker_ext_hook():
def __init__(self):
super(MyUnpacker, self).__init__(
ext_hook=self._hook, raw_as_bytes=False)
ext_hook=self._hook, raw=False)
def _hook(self, code, data):
if code == 1: