mirror of
https://github.com/msgpack/msgpack-python.git
synced 2025-11-02 10:30:54 +00:00
Joel Nothman
commit
df4f23779d
16 changed files with 437 additions and 142 deletions
|
|
@ -1,3 +1,24 @@
|
|||
0.3.0
|
||||
=====
|
||||
:release date: in development
|
||||
|
||||
Changes
|
||||
-------
|
||||
* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
|
||||
|
||||
|
||||
0.2.3
|
||||
=======
|
||||
:release date: in development
|
||||
|
||||
Changes
|
||||
-------
|
||||
* Warn when use_list is not specified. It's default value will be changed in 0.3.
|
||||
|
||||
Bugs fixed
|
||||
-----------
|
||||
* Can't pack subclass of dict.
|
||||
|
||||
0.2.2
|
||||
=======
|
||||
:release date: 2012-09-21
|
||||
|
|
|
|||
7
ROADMAP.md
Normal file
7
ROADMAP.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
0.2 series
|
||||
==========
|
||||
Improve compatibility to simplejson.
|
||||
|
||||
0.3 series
|
||||
==========
|
||||
Add features msgpack-ruby has.
|
||||
|
|
@ -1,12 +1,16 @@
|
|||
# coding: utf-8
|
||||
#cython: embedsignature=True
|
||||
|
||||
import warnings
|
||||
|
||||
from cpython cimport *
|
||||
cdef extern from "Python.h":
|
||||
ctypedef char* const_char_ptr "const char*"
|
||||
ctypedef char* const_void_ptr "const void*"
|
||||
ctypedef struct PyObject
|
||||
cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
|
||||
char* __FILE__
|
||||
int __LINE__
|
||||
|
||||
from libc.stdlib cimport *
|
||||
from libc.string cimport *
|
||||
|
|
@ -139,11 +143,19 @@ cdef class Packer(object):
|
|||
ret = msgpack_pack_raw(&self.pk, len(o))
|
||||
if ret == 0:
|
||||
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
|
||||
elif PyDict_Check(o):
|
||||
elif PyDict_CheckExact(o):
|
||||
d = <dict>o
|
||||
ret = msgpack_pack_map(&self.pk, len(d))
|
||||
if ret == 0:
|
||||
for k,v in d.iteritems():
|
||||
for k, v in d.iteritems():
|
||||
ret = self._pack(k, nest_limit-1)
|
||||
if ret != 0: break
|
||||
ret = self._pack(v, nest_limit-1)
|
||||
if ret != 0: break
|
||||
elif PyDict_Check(o):
|
||||
ret = msgpack_pack_map(&self.pk, len(o))
|
||||
if ret == 0:
|
||||
for k, v in o.items():
|
||||
ret = self._pack(k, nest_limit-1)
|
||||
if ret != 0: break
|
||||
ret = self._pack(v, nest_limit-1)
|
||||
|
|
@ -170,6 +182,17 @@ cdef class Packer(object):
|
|||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
cpdef pack_array_header(self, size_t size):
|
||||
msgpack_pack_array(&self.pk, size)
|
||||
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
||||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
cpdef pack_map_header(self, size_t size):
|
||||
msgpack_pack_map(&self.pk, size)
|
||||
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
||||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
|
||||
"""
|
||||
|
|
@ -187,8 +210,9 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use
|
|||
|
||||
cdef extern from "unpack.h":
|
||||
ctypedef struct msgpack_user:
|
||||
int use_list
|
||||
bint use_list
|
||||
PyObject* object_hook
|
||||
bint has_pairs_hook # call object_hook with k-v pairs
|
||||
PyObject* list_hook
|
||||
char *encoding
|
||||
char *unicode_errors
|
||||
|
|
@ -200,71 +224,98 @@ cdef extern from "unpack.h":
|
|||
unsigned int ct
|
||||
PyObject* key
|
||||
|
||||
int template_execute(template_context* ctx, const_char_ptr data,
|
||||
size_t len, size_t* off) except -1
|
||||
ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
|
||||
size_t len, size_t* off) except -1
|
||||
execute_fn template_construct
|
||||
execute_fn template_skip
|
||||
execute_fn read_array_header
|
||||
execute_fn read_map_header
|
||||
void template_init(template_context* ctx)
|
||||
object template_data(template_context* ctx)
|
||||
|
||||
cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors):
|
||||
template_init(ctx)
|
||||
ctx.user.use_list = use_list
|
||||
ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
|
||||
|
||||
if object_hook is not None and object_pairs_hook is not None:
|
||||
raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
|
||||
|
||||
if object_hook is not None:
|
||||
if not PyCallable_Check(object_hook):
|
||||
raise TypeError("object_hook must be a callable.")
|
||||
ctx.user.object_hook = <PyObject*>object_hook
|
||||
|
||||
if object_pairs_hook is None:
|
||||
ctx.user.has_pairs_hook = False
|
||||
else:
|
||||
if not PyCallable_Check(object_pairs_hook):
|
||||
raise TypeError("object_pairs_hook must be a callable.")
|
||||
ctx.user.object_hook = <PyObject*>object_pairs_hook
|
||||
ctx.user.has_pairs_hook = True
|
||||
|
||||
if list_hook is not None:
|
||||
if not PyCallable_Check(list_hook):
|
||||
raise TypeError("list_hook must be a callable.")
|
||||
ctx.user.list_hook = <PyObject*>list_hook
|
||||
|
||||
if encoding is None:
|
||||
ctx.user.encoding = NULL
|
||||
ctx.user.unicode_errors = NULL
|
||||
else:
|
||||
if isinstance(encoding, unicode):
|
||||
_bencoding = encoding.encode('ascii')
|
||||
else:
|
||||
_bencoding = encoding
|
||||
ctx.user.encoding = PyBytes_AsString(_bencoding)
|
||||
if isinstance(unicode_errors, unicode):
|
||||
_berrors = unicode_errors.encode('ascii')
|
||||
else:
|
||||
_berrors = unicode_errors
|
||||
ctx.user.unicode_errors = PyBytes_AsString(_berrors)
|
||||
|
||||
def unpackb(object packed, object object_hook=None, object list_hook=None,
|
||||
bint use_list=0, encoding=None, unicode_errors="strict",
|
||||
bint use_list=1, encoding=None, unicode_errors="strict",
|
||||
object_pairs_hook=None,
|
||||
):
|
||||
"""Unpack packed_bytes to object. Returns an unpacked object.
|
||||
|
||||
Raises `ValueError` when `packed` contains extra bytes.
|
||||
"""
|
||||
Unpack packed_bytes to object. Returns an unpacked object."""
|
||||
cdef template_context ctx
|
||||
cdef size_t off = 0
|
||||
cdef int ret
|
||||
|
||||
cdef char* buf
|
||||
cdef Py_ssize_t buf_len
|
||||
|
||||
PyObject_AsReadBuffer(packed, <const_void_ptr*>&buf, &buf_len)
|
||||
|
||||
if encoding is None:
|
||||
enc = NULL
|
||||
err = NULL
|
||||
else:
|
||||
if isinstance(encoding, unicode):
|
||||
bencoding = encoding.encode('ascii')
|
||||
else:
|
||||
bencoding = encoding
|
||||
if isinstance(unicode_errors, unicode):
|
||||
berrors = unicode_errors.encode('ascii')
|
||||
else:
|
||||
berrors = unicode_errors
|
||||
enc = PyBytes_AsString(bencoding)
|
||||
err = PyBytes_AsString(berrors)
|
||||
|
||||
template_init(&ctx)
|
||||
ctx.user.use_list = use_list
|
||||
ctx.user.object_hook = ctx.user.list_hook = NULL
|
||||
ctx.user.encoding = <const_char_ptr>enc
|
||||
ctx.user.unicode_errors = <const_char_ptr>err
|
||||
if object_hook is not None:
|
||||
if not PyCallable_Check(object_hook):
|
||||
raise TypeError("object_hook must be a callable.")
|
||||
ctx.user.object_hook = <PyObject*>object_hook
|
||||
if list_hook is not None:
|
||||
if not PyCallable_Check(list_hook):
|
||||
raise TypeError("list_hook must be a callable.")
|
||||
ctx.user.list_hook = <PyObject*>list_hook
|
||||
ret = template_execute(&ctx, buf, buf_len, &off)
|
||||
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
|
||||
ret = template_construct(&ctx, buf, buf_len, &off)
|
||||
if ret == 1:
|
||||
return template_data(&ctx)
|
||||
obj = template_data(&ctx)
|
||||
if off < buf_len:
|
||||
raise ValueError("Extra data.")
|
||||
return obj
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def unpack(object stream, object object_hook=None, object list_hook=None,
|
||||
bint use_list=0, encoding=None, unicode_errors="strict",
|
||||
bint use_list=1, encoding=None, unicode_errors="strict",
|
||||
object_pairs_hook=None,
|
||||
):
|
||||
"""
|
||||
unpack an object from stream.
|
||||
"""Unpack an object from `stream`.
|
||||
|
||||
Raises `ValueError` when `stream` has extra bytes.
|
||||
"""
|
||||
return unpackb(stream.read(), use_list=use_list,
|
||||
object_hook=object_hook, list_hook=list_hook,
|
||||
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
|
||||
encoding=encoding, unicode_errors=unicode_errors,
|
||||
)
|
||||
|
||||
|
||||
cdef class Unpacker(object):
|
||||
"""
|
||||
Streaming unpacker.
|
||||
|
|
@ -277,10 +328,13 @@ cdef class Unpacker(object):
|
|||
(default: min(1024**2, max_buffer_size))
|
||||
|
||||
If `use_list` is true, msgpack list is deserialized to Python list.
|
||||
Otherwise, it is deserialized to Python tuple. (default: False)
|
||||
Otherwise, it is deserialized to Python tuple.
|
||||
|
||||
`object_hook` is same to simplejson. If it is not None, it should be callable
|
||||
and Unpacker calls it when deserializing key-value.
|
||||
and Unpacker calls it with a dict argument after deserializing a map.
|
||||
|
||||
`object_pairs_hook` is same to simplejson. If it is not None, it should be callable
|
||||
and Unpacker calls it with a list of key-value pairs after deserializing a map.
|
||||
|
||||
`encoding` is encoding used for decoding msgpack bytes. If it is None (default),
|
||||
msgpack bytes is deserialized to Python bytes.
|
||||
|
|
@ -315,7 +369,6 @@ cdef class Unpacker(object):
|
|||
cdef object file_like
|
||||
cdef object file_like_read
|
||||
cdef Py_ssize_t read_size
|
||||
cdef bint use_list
|
||||
cdef object object_hook
|
||||
cdef object _bencoding
|
||||
cdef object _berrors
|
||||
|
|
@ -330,10 +383,10 @@ cdef class Unpacker(object):
|
|||
free(self.buf)
|
||||
self.buf = NULL
|
||||
|
||||
def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
|
||||
object object_hook=None, object list_hook=None,
|
||||
encoding=None, unicode_errors='strict', int max_buffer_size=0):
|
||||
self.use_list = use_list
|
||||
def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
|
||||
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
|
||||
encoding=None, unicode_errors='strict', int max_buffer_size=0,
|
||||
):
|
||||
self.file_like = file_like
|
||||
if file_like:
|
||||
self.file_like_read = file_like.read
|
||||
|
|
@ -353,31 +406,7 @@ cdef class Unpacker(object):
|
|||
self.buf_size = read_size
|
||||
self.buf_head = 0
|
||||
self.buf_tail = 0
|
||||
template_init(&self.ctx)
|
||||
self.ctx.user.use_list = use_list
|
||||
self.ctx.user.object_hook = self.ctx.user.list_hook = <PyObject*>NULL
|
||||
if object_hook is not None:
|
||||
if not PyCallable_Check(object_hook):
|
||||
raise TypeError("object_hook must be a callable.")
|
||||
self.ctx.user.object_hook = <PyObject*>object_hook
|
||||
if list_hook is not None:
|
||||
if not PyCallable_Check(list_hook):
|
||||
raise TypeError("list_hook must be a callable.")
|
||||
self.ctx.user.list_hook = <PyObject*>list_hook
|
||||
if encoding is None:
|
||||
self.ctx.user.encoding = NULL
|
||||
self.ctx.user.unicode_errors = NULL
|
||||
else:
|
||||
if isinstance(encoding, unicode):
|
||||
self._bencoding = encoding.encode('ascii')
|
||||
else:
|
||||
self._bencoding = encoding
|
||||
self.ctx.user.encoding = PyBytes_AsString(self._bencoding)
|
||||
if isinstance(unicode_errors, unicode):
|
||||
self._berrors = unicode_errors.encode('ascii')
|
||||
else:
|
||||
self._berrors = unicode_errors
|
||||
self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors)
|
||||
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
|
||||
|
||||
def feed(self, object next_bytes):
|
||||
cdef char* buf
|
||||
|
|
@ -438,20 +467,20 @@ cdef class Unpacker(object):
|
|||
else:
|
||||
self.file_like = None
|
||||
|
||||
cpdef unpack(self):
|
||||
"""unpack one object"""
|
||||
cdef object _unpack(self, execute_fn execute):
|
||||
cdef int ret
|
||||
cdef object obj
|
||||
while 1:
|
||||
ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
|
||||
ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
|
||||
if ret == 1:
|
||||
o = template_data(&self.ctx)
|
||||
obj = template_data(&self.ctx)
|
||||
template_init(&self.ctx)
|
||||
return o
|
||||
return obj
|
||||
elif ret == 0:
|
||||
if self.file_like is not None:
|
||||
self.read_from_file()
|
||||
continue
|
||||
raise StopIteration("No more unpack data.")
|
||||
raise StopIteration("No more data to unpack.")
|
||||
else:
|
||||
raise ValueError("Unpack failed: error = %d" % (ret,))
|
||||
|
||||
|
|
@ -465,11 +494,27 @@ cdef class Unpacker(object):
|
|||
ret += self.file_like.read(nbytes - len(ret))
|
||||
return ret
|
||||
|
||||
def unpack(self):
|
||||
"""unpack one object"""
|
||||
return self._unpack(template_construct)
|
||||
|
||||
def skip(self):
|
||||
"""read and ignore one object, returning None"""
|
||||
return self._unpack(template_skip)
|
||||
|
||||
def read_array_header(self):
|
||||
"""assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents."""
|
||||
return self._unpack(read_array_header)
|
||||
|
||||
def read_map_header(self):
|
||||
"""assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs."""
|
||||
return self._unpack(read_map_header)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return self.unpack()
|
||||
return self._unpack(template_construct)
|
||||
|
||||
# for debug.
|
||||
#def _buf(self):
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = (0, 2, 2)
|
||||
version = (0, 3, 0, 'dev1')
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
typedef struct unpack_user {
|
||||
int use_list;
|
||||
PyObject *object_hook;
|
||||
bool has_pairs_hook;
|
||||
PyObject *list_hook;
|
||||
const char *encoding;
|
||||
const char *unicode_errors;
|
||||
|
|
@ -41,6 +42,7 @@ typedef struct unpack_user {
|
|||
|
||||
#define msgpack_unpack_user unpack_user
|
||||
|
||||
typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off);
|
||||
|
||||
struct template_context;
|
||||
typedef struct template_context template_context;
|
||||
|
|
@ -160,9 +162,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
|
|||
static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
|
||||
{
|
||||
if (u->list_hook) {
|
||||
PyObject *arglist = Py_BuildValue("(O)", *c);
|
||||
PyObject *new_c = PyEval_CallObject(u->list_hook, arglist);
|
||||
Py_DECREF(arglist);
|
||||
PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
|
||||
Py_DECREF(*c);
|
||||
*c = new_c;
|
||||
}
|
||||
|
|
@ -171,16 +171,31 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
|
|||
|
||||
static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
|
||||
{
|
||||
PyObject *p = PyDict_New();
|
||||
PyObject *p;
|
||||
if (u->has_pairs_hook) {
|
||||
p = PyList_New(n); // Or use tuple?
|
||||
}
|
||||
else {
|
||||
p = PyDict_New();
|
||||
}
|
||||
if (!p)
|
||||
return -1;
|
||||
*o = p;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
|
||||
static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
|
||||
{
|
||||
if (PyDict_SetItem(*c, k, v) == 0) {
|
||||
if (u->has_pairs_hook) {
|
||||
msgpack_unpack_object item = PyTuple_Pack(2, k, v);
|
||||
if (!item)
|
||||
return -1;
|
||||
Py_DECREF(k);
|
||||
Py_DECREF(v);
|
||||
PyList_SET_ITEM(*c, current, item);
|
||||
return 0;
|
||||
}
|
||||
else if (PyDict_SetItem(*c, k, v) == 0) {
|
||||
Py_DECREF(k);
|
||||
Py_DECREF(v);
|
||||
return 0;
|
||||
|
|
@ -191,9 +206,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
|
|||
static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
|
||||
{
|
||||
if (u->object_hook) {
|
||||
PyObject *arglist = Py_BuildValue("(O)", *c);
|
||||
PyObject *new_c = PyEval_CallObject(u->object_hook, arglist);
|
||||
Py_DECREF(arglist);
|
||||
PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
|
||||
Py_DECREF(*c);
|
||||
*c = new_c;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
|
|||
}
|
||||
|
||||
|
||||
template <bool construct>
|
||||
msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
|
||||
{
|
||||
assert(len >= *off);
|
||||
|
|
@ -117,14 +118,17 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
|
|||
|
||||
int ret;
|
||||
|
||||
#define construct_cb(name) \
|
||||
construct && msgpack_unpack_callback(name)
|
||||
|
||||
#define push_simple_value(func) \
|
||||
if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
|
||||
if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
#define push_fixed_value(func, arg) \
|
||||
if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
|
||||
if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
#define push_variable_value(func, base, pos, len) \
|
||||
if(msgpack_unpack_callback(func)(user, \
|
||||
if(construct_cb(func)(user, \
|
||||
(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
|
||||
|
|
@ -140,9 +144,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
|
|||
|
||||
#define start_container(func, count_, ct_) \
|
||||
if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
|
||||
if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
|
||||
if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
|
||||
if((count_) == 0) { obj = stack[top].obj; \
|
||||
msgpack_unpack_callback(func##_end)(user, &obj); \
|
||||
construct_cb(func##_end)(user, &obj); \
|
||||
goto _push; } \
|
||||
stack[top].ct = ct_; \
|
||||
stack[top].size = count_; \
|
||||
|
|
@ -340,10 +344,10 @@ _push:
|
|||
c = &stack[top-1];
|
||||
switch(c->ct) {
|
||||
case CT_ARRAY_ITEM:
|
||||
if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
|
||||
if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
|
||||
if(++c->count == c->size) {
|
||||
obj = c->obj;
|
||||
msgpack_unpack_callback(_array_end)(user, &obj);
|
||||
construct_cb(_array_end)(user, &obj);
|
||||
--top;
|
||||
/*printf("stack pop %d\n", top);*/
|
||||
goto _push;
|
||||
|
|
@ -354,10 +358,10 @@ _push:
|
|||
c->ct = CT_MAP_VALUE;
|
||||
goto _header_again;
|
||||
case CT_MAP_VALUE:
|
||||
if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
|
||||
if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
|
||||
if(++c->count == c->size) {
|
||||
obj = c->obj;
|
||||
msgpack_unpack_callback(_map_end)(user, &obj);
|
||||
construct_cb(_map_end)(user, &obj);
|
||||
--top;
|
||||
/*printf("stack pop %d\n", top);*/
|
||||
goto _push;
|
||||
|
|
@ -377,6 +381,8 @@ _header_again:
|
|||
|
||||
|
||||
_finish:
|
||||
if (!construct)
|
||||
msgpack_unpack_callback(_nil)(user, &obj);
|
||||
stack[0].obj = obj;
|
||||
++p;
|
||||
ret = 1;
|
||||
|
|
@ -399,15 +405,13 @@ _end:
|
|||
*off = p - (const unsigned char*)data;
|
||||
|
||||
return ret;
|
||||
#undef construct_cb
|
||||
}
|
||||
|
||||
|
||||
#undef msgpack_unpack_func
|
||||
#undef msgpack_unpack_callback
|
||||
#undef msgpack_unpack_struct
|
||||
#undef msgpack_unpack_object
|
||||
#undef msgpack_unpack_user
|
||||
|
||||
#undef SWITCH_RANGE_BEGIN
|
||||
#undef SWITCH_RANGE
|
||||
#undef SWITCH_RANGE_DEFAULT
|
||||
#undef SWITCH_RANGE_END
|
||||
#undef push_simple_value
|
||||
#undef push_fixed_value
|
||||
#undef push_variable_value
|
||||
|
|
@ -415,6 +419,74 @@ _end:
|
|||
#undef again_fixed_trail_if_zero
|
||||
#undef start_container
|
||||
|
||||
template <unsigned int fixed_offset, unsigned int var_offset>
|
||||
msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
|
||||
{
|
||||
assert(len >= *off);
|
||||
uint32_t size;
|
||||
const unsigned char *const p = (unsigned char*)data + *off;
|
||||
|
||||
#define inc_offset(inc) \
|
||||
if (len - *off < inc) \
|
||||
return 0; \
|
||||
*off += inc;
|
||||
|
||||
switch (*p) {
|
||||
case var_offset:
|
||||
inc_offset(3);
|
||||
size = _msgpack_load16(uint16_t, p + 1);
|
||||
break;
|
||||
case var_offset + 1:
|
||||
inc_offset(5);
|
||||
size = _msgpack_load32(uint32_t, p + 1);
|
||||
break;
|
||||
#ifdef USE_CASE_RANGE
|
||||
case fixed_offset + 0x0 ... fixed_offset + 0xf:
|
||||
#else
|
||||
case fixed_offset + 0x0:
|
||||
case fixed_offset + 0x1:
|
||||
case fixed_offset + 0x2:
|
||||
case fixed_offset + 0x3:
|
||||
case fixed_offset + 0x4:
|
||||
case fixed_offset + 0x5:
|
||||
case fixed_offset + 0x6:
|
||||
case fixed_offset + 0x7:
|
||||
case fixed_offset + 0x8:
|
||||
case fixed_offset + 0x9:
|
||||
case fixed_offset + 0xa:
|
||||
case fixed_offset + 0xb:
|
||||
case fixed_offset + 0xc:
|
||||
case fixed_offset + 0xd:
|
||||
case fixed_offset + 0xe:
|
||||
case fixed_offset + 0xf:
|
||||
#endif
|
||||
++*off;
|
||||
size = ((unsigned int)*p) & 0x0f;
|
||||
break;
|
||||
default:
|
||||
PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
|
||||
return -1;
|
||||
}
|
||||
msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
|
||||
return 1;
|
||||
}
|
||||
|
||||
#undef SWITCH_RANGE_BEGIN
|
||||
#undef SWITCH_RANGE
|
||||
#undef SWITCH_RANGE_DEFAULT
|
||||
#undef SWITCH_RANGE_END
|
||||
|
||||
static const execute_fn template_construct = &template_execute<true>;
|
||||
static const execute_fn template_skip = &template_execute<false>;
|
||||
static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>;
|
||||
static const execute_fn read_map_header = &template_container_header<0x80, 0xde>;
|
||||
|
||||
#undef msgpack_unpack_func
|
||||
#undef msgpack_unpack_callback
|
||||
#undef msgpack_unpack_struct
|
||||
#undef msgpack_unpack_object
|
||||
#undef msgpack_unpack_user
|
||||
|
||||
#undef NEXT_CS
|
||||
|
||||
/* vim: set ts=4 sw=4 noexpandtab */
|
||||
|
|
|
|||
11
setup.py
11
setup.py
|
|
@ -18,7 +18,7 @@ except ImportError:
|
|||
|
||||
def cythonize(src):
|
||||
sys.stderr.write("cythonize: %r\n" % (src,))
|
||||
cython_compiler.compile([src])
|
||||
cython_compiler.compile([src], cplus=True, emit_linenums=True)
|
||||
|
||||
def ensure_source(src):
|
||||
pyx = os.path.splitext(src)[0] + '.pyx'
|
||||
|
|
@ -34,13 +34,6 @@ Install Cython >= 0.16 or install msgpack from PyPI.
|
|||
os.stat(src).st_mtime < os.stat(pyx).st_mtime and
|
||||
have_cython):
|
||||
cythonize(pyx)
|
||||
|
||||
# Use C++ compiler on win32.
|
||||
# MSVC9 doesn't provide stdint.h when using C Compiler.
|
||||
if sys.platform == 'win32':
|
||||
cpp = src + 'pp'
|
||||
shutil.copy(src, cpp)
|
||||
return cpp
|
||||
else:
|
||||
return src
|
||||
|
||||
|
|
@ -67,7 +60,7 @@ if have_cython:
|
|||
else:
|
||||
Sdist = sdist
|
||||
|
||||
sources = ['msgpack/_msgpack.c']
|
||||
sources = ['msgpack/_msgpack.cpp']
|
||||
libraries = []
|
||||
if sys.platform == 'win32':
|
||||
libraries.append('ws2_32')
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ def test_unpack_buffer():
|
|||
from array import array
|
||||
buf = array('b')
|
||||
buf.fromstring(packb(('foo', 'bar')))
|
||||
obj = unpackb(buf)
|
||||
assert_equal((b'foo', b'bar'), obj)
|
||||
obj = unpackb(buf, use_list=1)
|
||||
assert_equal([b'foo', b'bar'], obj)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from msgpack import packb, unpackb
|
|||
def check(length, obj):
|
||||
v = packb(obj)
|
||||
assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
|
||||
assert_equal(unpackb(v), obj)
|
||||
assert_equal(unpackb(v, use_list=0), obj)
|
||||
|
||||
def test_1():
|
||||
for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
|
||||
|
|
@ -71,7 +71,7 @@ def test_array32():
|
|||
|
||||
def match(obj, buf):
|
||||
assert_equal(packb(obj), buf)
|
||||
assert_equal(unpackb(buf), obj)
|
||||
assert_equal(unpackb(buf, use_list=0), obj)
|
||||
|
||||
def test_match():
|
||||
cases = [
|
||||
|
|
@ -99,7 +99,7 @@ def test_match():
|
|||
match(v, p)
|
||||
|
||||
def test_unicode():
|
||||
assert_equal(b'foobar', unpackb(packb('foobar')))
|
||||
assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ from nose import main
|
|||
from nose.tools import *
|
||||
from msgpack import unpackb
|
||||
|
||||
def check(src, should):
|
||||
assert_equal(unpackb(src), should)
|
||||
def check(src, should, use_list=0):
|
||||
assert_equal(unpackb(src, use_list=use_list), should)
|
||||
|
||||
def testSimpleValue():
|
||||
check(b"\x93\xc0\xc2\xc3",
|
||||
|
|
|
|||
|
|
@ -18,25 +18,35 @@ def _encode_complex(obj):
|
|||
|
||||
def test_encode_hook():
|
||||
packed = packb([3, 1+2j], default=_encode_complex)
|
||||
unpacked = unpackb(packed)
|
||||
unpacked = unpackb(packed, use_list=1)
|
||||
eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
|
||||
|
||||
def test_decode_hook():
|
||||
packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
|
||||
unpacked = unpackb(packed, object_hook=_decode_complex)
|
||||
unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
|
||||
eq_(unpacked[1], 1+2j)
|
||||
|
||||
def test_decode_pairs_hook():
|
||||
packed = packb([3, {1: 2, 3: 4}])
|
||||
prod_sum = 1 * 2 + 3 * 4
|
||||
unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
|
||||
eq_(unpacked[1], prod_sum)
|
||||
|
||||
@raises(ValueError)
|
||||
def test_only_one_obj_hook():
|
||||
unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
|
||||
|
||||
@raises(ValueError)
|
||||
def test_bad_hook():
|
||||
packed = packb([3, 1+2j], default=lambda o: o)
|
||||
unpacked = unpackb(packed)
|
||||
unpacked = unpackb(packed, use_list=1)
|
||||
|
||||
def _arr_to_str(arr):
|
||||
return ''.join(str(c) for c in arr)
|
||||
|
||||
def test_array_hook():
|
||||
packed = packb([1,2,3])
|
||||
unpacked = unpackb(packed, list_hook=_arr_to_str)
|
||||
unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
|
||||
eq_(unpacked, '123')
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer
|
|||
|
||||
from io import BytesIO
|
||||
|
||||
def check(data):
|
||||
re = unpackb(packb(data))
|
||||
def check(data, use_list=False):
|
||||
re = unpackb(packb(data), use_list=use_list)
|
||||
assert_equal(re, data)
|
||||
|
||||
def testPack():
|
||||
|
|
@ -31,14 +31,14 @@ def testPack():
|
|||
|
||||
def testPackUnicode():
|
||||
test_data = [
|
||||
six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"),
|
||||
six.u(""), six.u("abcd"), [six.u("defgh")], six.u("Русский текст"),
|
||||
]
|
||||
for td in test_data:
|
||||
re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8')
|
||||
re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
|
||||
assert_equal(re, td)
|
||||
packer = Packer(encoding='utf-8')
|
||||
data = packer.pack(td)
|
||||
re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
|
||||
re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
|
||||
assert_equal(re, td)
|
||||
|
||||
def testPackUTF32():
|
||||
|
|
@ -46,11 +46,11 @@ def testPackUTF32():
|
|||
test_data = [
|
||||
six.u(""),
|
||||
six.u("abcd"),
|
||||
(six.u("defgh"),),
|
||||
[six.u("defgh")],
|
||||
six.u("Русский текст"),
|
||||
]
|
||||
for td in test_data:
|
||||
re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32')
|
||||
re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
|
||||
assert_equal(re, td)
|
||||
except LookupError:
|
||||
raise SkipTest
|
||||
|
|
@ -63,20 +63,19 @@ def testPackBytes():
|
|||
check(td)
|
||||
|
||||
def testIgnoreUnicodeErrors():
|
||||
re = unpackb(packb(b'abc\xeddef'),
|
||||
encoding='utf-8', unicode_errors='ignore')
|
||||
re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
|
||||
assert_equal(re, "abcdef")
|
||||
|
||||
@raises(UnicodeDecodeError)
|
||||
def testStrictUnicodeUnpack():
|
||||
unpackb(packb(b'abc\xeddef'), encoding='utf-8')
|
||||
unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
|
||||
|
||||
@raises(UnicodeEncodeError)
|
||||
def testStrictUnicodePack():
|
||||
packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
|
||||
|
||||
def testIgnoreErrorsPack():
|
||||
re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
|
||||
re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
|
||||
assert_equal(re, six.u("abcdef"))
|
||||
|
||||
@raises(TypeError)
|
||||
|
|
@ -84,12 +83,66 @@ def testNoEncoding():
|
|||
packb(six.u("abc"), encoding=None)
|
||||
|
||||
def testDecodeBinary():
|
||||
re = unpackb(packb("abc"), encoding=None)
|
||||
re = unpackb(packb("abc"), encoding=None, use_list=1)
|
||||
assert_equal(re, b"abc")
|
||||
|
||||
def testPackFloat():
|
||||
assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0))
|
||||
assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
|
||||
|
||||
def testArraySize(sizes=[0, 5, 50, 1000]):
|
||||
bio = six.BytesIO()
|
||||
packer = Packer()
|
||||
for size in sizes:
|
||||
bio.write(packer.pack_array_header(size))
|
||||
for i in range(size):
|
||||
bio.write(packer.pack(i))
|
||||
|
||||
bio.seek(0)
|
||||
unpacker = Unpacker(bio, use_list=1)
|
||||
for size in sizes:
|
||||
assert unpacker.unpack() == list(range(size))
|
||||
|
||||
def testMapSize(sizes=[0, 5, 50, 1000]):
|
||||
bio = six.BytesIO()
|
||||
packer = Packer()
|
||||
for size in sizes:
|
||||
bio.write(packer.pack_map_header(size))
|
||||
for i in range(size):
|
||||
bio.write(packer.pack(i)) # key
|
||||
bio.write(packer.pack(i * 2)) # value
|
||||
|
||||
bio.seek(0)
|
||||
unpacker = Unpacker(bio)
|
||||
for size in sizes:
|
||||
assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
|
||||
|
||||
|
||||
|
||||
|
||||
class odict(dict):
|
||||
'''Reimplement OrderedDict to run test on Python 2.6'''
|
||||
def __init__(self, seq):
|
||||
self._seq = seq
|
||||
dict.__init__(self, seq)
|
||||
|
||||
def items(self):
|
||||
return self._seq[:]
|
||||
|
||||
def iteritems(self):
|
||||
return iter(self._seq)
|
||||
|
||||
def keys(self):
|
||||
return [x[0] for x in self._seq]
|
||||
|
||||
def test_odict():
|
||||
seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
|
||||
od = odict(seq)
|
||||
assert_equal(unpackb(packb(od), use_list=1), dict(seq))
|
||||
def pair_hook(seq):
|
||||
return seq
|
||||
assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
66
test/test_read_size.py
Normal file
66
test/test_read_size.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
"""Test Unpacker's read_array_header and read_map_header methods"""
|
||||
from msgpack import packb, Unpacker
|
||||
UnexpectedTypeException = ValueError
|
||||
|
||||
def test_read_array_header():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb(['a', 'b', 'c']))
|
||||
assert unpacker.read_array_header() == 3
|
||||
assert unpacker.unpack() == b'a'
|
||||
assert unpacker.unpack() == b'b'
|
||||
assert unpacker.unpack() == b'c'
|
||||
try:
|
||||
unpacker.unpack()
|
||||
assert 0, 'should raise exception'
|
||||
except StopIteration:
|
||||
assert 1, 'okay'
|
||||
|
||||
|
||||
def test_read_map_header():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb({'a': 'A'}))
|
||||
assert unpacker.read_map_header() == 1
|
||||
assert unpacker.unpack() == B'a'
|
||||
assert unpacker.unpack() == B'A'
|
||||
try:
|
||||
unpacker.unpack()
|
||||
assert 0, 'should raise exception'
|
||||
except StopIteration:
|
||||
assert 1, 'okay'
|
||||
|
||||
def test_incorrect_type_array():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb(1))
|
||||
try:
|
||||
unpacker.read_array_header()
|
||||
assert 0, 'should raise exception'
|
||||
except UnexpectedTypeException:
|
||||
assert 1, 'okay'
|
||||
|
||||
def test_incorrect_type_map():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb(1))
|
||||
try:
|
||||
unpacker.read_map_header()
|
||||
assert 0, 'should raise exception'
|
||||
except UnexpectedTypeException:
|
||||
assert 1, 'okay'
|
||||
|
||||
def test_correct_type_nested_array():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb({'a': ['b', 'c', 'd']}))
|
||||
try:
|
||||
unpacker.read_array_header()
|
||||
assert 0, 'should raise exception'
|
||||
except UnexpectedTypeException:
|
||||
assert 1, 'okay'
|
||||
|
||||
def test_incorrect_type_nested_map():
|
||||
unpacker = Unpacker()
|
||||
unpacker.feed(packb([{'a': 'b'}]))
|
||||
try:
|
||||
unpacker.read_map_header()
|
||||
assert 0, 'should raise exception'
|
||||
except UnexpectedTypeException:
|
||||
assert 1, 'okay'
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size():
|
|||
f = io.BytesIO(dumpf.getvalue())
|
||||
dumpf.close()
|
||||
|
||||
unpacker = msgpack.Unpacker(f, read_size=read_size)
|
||||
unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
|
||||
|
||||
read_count = 0
|
||||
for idx, o in enumerate(unpacker):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from msgpack import Unpacker, BufferFull
|
|||
import nose
|
||||
|
||||
def test_foobar():
|
||||
unpacker = Unpacker(read_size=3)
|
||||
unpacker = Unpacker(read_size=3, use_list=1)
|
||||
unpacker.feed(b'foobar')
|
||||
assert unpacker.unpack() == ord(b'f')
|
||||
assert unpacker.unpack() == ord(b'o')
|
||||
|
|
@ -29,10 +29,24 @@ def test_foobar():
|
|||
k += 1
|
||||
assert k == len(b'foobar')
|
||||
|
||||
def test_foobar_skip():
|
||||
unpacker = Unpacker(read_size=3, use_list=1)
|
||||
unpacker.feed(b'foobar')
|
||||
assert unpacker.unpack() == ord(b'f')
|
||||
unpacker.skip()
|
||||
assert unpacker.unpack() == ord(b'o')
|
||||
unpacker.skip()
|
||||
assert unpacker.unpack() == ord(b'a')
|
||||
unpacker.skip()
|
||||
try:
|
||||
o = unpacker.unpack()
|
||||
assert 0, "should raise exception"
|
||||
except StopIteration:
|
||||
assert 1, "ok"
|
||||
|
||||
def test_maxbuffersize():
|
||||
nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
|
||||
unpacker = Unpacker(read_size=3, max_buffer_size=3)
|
||||
unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
|
||||
unpacker.feed(b'fo')
|
||||
nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
|
||||
unpacker.feed(b'o')
|
||||
|
|
|
|||
3
tox.ini
3
tox.ini
|
|
@ -1,5 +1,6 @@
|
|||
[tox]
|
||||
envlist = py26,py27,py32
|
||||
envlist = py26,py27,py32,py33
|
||||
|
||||
[testenv]
|
||||
deps=
|
||||
nose
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue