2009-05-22 14:31:20 +09:00
|
|
|
# coding: utf-8
|
2011-08-26 04:45:05 +09:00
|
|
|
#cython: embedsignature=True
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2010-11-03 03:11:00 +09:00
|
|
|
from cpython cimport *
|
|
|
|
from libc.stdlib cimport *
|
|
|
|
from libc.string cimport *
|
2012-07-13 21:28:16 +09:00
|
|
|
from libc.limits cimport *
|
|
|
|
|
2012-12-11 22:15:21 +09:00
|
|
|
from msgpack.exceptions import PackValueError
|
2012-07-13 21:28:16 +09:00
|
|
|
|
2009-06-08 00:23:38 +09:00
|
|
|
cdef extern from "pack.h":
|
2009-05-22 14:31:20 +09:00
|
|
|
struct msgpack_packer:
|
2009-07-01 00:57:46 +09:00
|
|
|
char* buf
|
|
|
|
size_t length
|
|
|
|
size_t buf_size
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2009-06-29 10:09:04 +09:00
|
|
|
int msgpack_pack_int(msgpack_packer* pk, int d)
|
|
|
|
int msgpack_pack_nil(msgpack_packer* pk)
|
|
|
|
int msgpack_pack_true(msgpack_packer* pk)
|
|
|
|
int msgpack_pack_false(msgpack_packer* pk)
|
|
|
|
int msgpack_pack_long(msgpack_packer* pk, long d)
|
|
|
|
int msgpack_pack_long_long(msgpack_packer* pk, long long d)
|
|
|
|
int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
|
2012-08-20 21:56:55 +02:00
|
|
|
int msgpack_pack_float(msgpack_packer* pk, float d)
|
2009-06-29 10:09:04 +09:00
|
|
|
int msgpack_pack_double(msgpack_packer* pk, double d)
|
|
|
|
int msgpack_pack_array(msgpack_packer* pk, size_t l)
|
|
|
|
int msgpack_pack_map(msgpack_packer* pk, size_t l)
|
|
|
|
int msgpack_pack_raw(msgpack_packer* pk, size_t l)
|
2013-10-17 08:35:08 +09:00
|
|
|
int msgpack_pack_bin(msgpack_packer* pk, size_t l)
|
2009-06-29 10:09:04 +09:00
|
|
|
int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2010-10-26 02:09:52 +09:00
|
|
|
cdef int DEFAULT_RECURSE_LIMIT=511
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2012-07-13 21:28:16 +09:00
|
|
|
|
|
|
|
|
2009-06-28 21:24:02 +09:00
|
|
|
cdef class Packer(object):
|
2013-02-26 09:20:44 +09:00
|
|
|
"""
|
|
|
|
MessagePack Packer
|
2011-04-15 17:36:17 +03:00
|
|
|
|
2013-02-26 09:20:44 +09:00
|
|
|
usage::
|
2009-06-08 01:30:43 +09:00
|
|
|
|
2009-07-01 20:55:24 +09:00
|
|
|
packer = Packer()
|
|
|
|
astream.write(packer.pack(a))
|
|
|
|
astream.write(packer.pack(b))
|
2012-12-10 21:47:18 +09:00
|
|
|
|
|
|
|
Packer's constructor has some keyword arguments:
|
|
|
|
|
2013-02-26 09:20:44 +09:00
|
|
|
:param callable default:
|
|
|
|
Convert user type to builtin type that Packer supports.
|
|
|
|
See also simplejson's document.
|
|
|
|
:param str encoding:
|
|
|
|
Convert unicode to bytes with this encoding. (default: 'utf-8')
|
2013-05-07 13:56:39 +09:00
|
|
|
:param str unicode_errors:
|
2013-02-26 09:20:44 +09:00
|
|
|
Error handler for encoding unicode. (default: 'strict')
|
|
|
|
:param bool use_single_float:
|
|
|
|
Use single precision float type for float. (default: False)
|
|
|
|
:param bool autoreset:
|
|
|
|
Reset buffer after each pack and return it's content as `bytes`. (default: True).
|
|
|
|
If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
|
2013-10-17 08:35:08 +09:00
|
|
|
:param bool use_bin_type:
|
|
|
|
Use bin type introduced in msgpack spec 2.0 for bytes.
|
|
|
|
It also enable str8 type for unicode.
|
2009-06-08 01:30:43 +09:00
|
|
|
"""
|
2009-05-22 14:31:20 +09:00
|
|
|
cdef msgpack_packer pk
|
2011-01-29 07:27:10 +09:00
|
|
|
cdef object _default
|
2011-04-15 17:36:17 +03:00
|
|
|
cdef object _bencoding
|
|
|
|
cdef object _berrors
|
|
|
|
cdef char *encoding
|
|
|
|
cdef char *unicode_errors
|
2012-08-20 21:56:55 +02:00
|
|
|
cdef bool use_float
|
2013-10-17 08:35:08 +09:00
|
|
|
cdef bool use_bin_type
|
2012-12-10 21:47:18 +09:00
|
|
|
cdef bint autoreset
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2009-07-01 20:55:24 +09:00
|
|
|
def __cinit__(self):
|
2009-07-01 00:57:46 +09:00
|
|
|
cdef int buf_size = 1024*1024
|
|
|
|
self.pk.buf = <char*> malloc(buf_size);
|
2011-01-10 20:47:23 +09:00
|
|
|
if self.pk.buf == NULL:
|
|
|
|
raise MemoryError("Unable to allocate internal buffer.")
|
2009-07-01 00:57:46 +09:00
|
|
|
self.pk.buf_size = buf_size
|
|
|
|
self.pk.length = 0
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2013-10-17 08:35:08 +09:00
|
|
|
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
|
|
|
|
use_single_float=False, bint autoreset=1, bint use_bin_type=0):
|
2013-02-26 09:20:44 +09:00
|
|
|
"""
|
|
|
|
"""
|
2012-09-21 14:08:34 +09:00
|
|
|
self.use_float = use_single_float
|
2012-12-10 21:47:18 +09:00
|
|
|
self.autoreset = autoreset
|
2013-10-17 08:35:08 +09:00
|
|
|
self.pk.use_bin_type = use_bin_type
|
2010-10-26 01:26:06 +09:00
|
|
|
if default is not None:
|
|
|
|
if not PyCallable_Check(default):
|
|
|
|
raise TypeError("default must be a callable.")
|
2011-01-29 07:27:10 +09:00
|
|
|
self._default = default
|
2011-04-15 17:36:17 +03:00
|
|
|
if encoding is None:
|
|
|
|
self.encoding = NULL
|
|
|
|
self.unicode_errors = NULL
|
|
|
|
else:
|
|
|
|
if isinstance(encoding, unicode):
|
|
|
|
self._bencoding = encoding.encode('ascii')
|
|
|
|
else:
|
|
|
|
self._bencoding = encoding
|
|
|
|
self.encoding = PyBytes_AsString(self._bencoding)
|
|
|
|
if isinstance(unicode_errors, unicode):
|
|
|
|
self._berrors = unicode_errors.encode('ascii')
|
|
|
|
else:
|
|
|
|
self._berrors = unicode_errors
|
|
|
|
self.unicode_errors = PyBytes_AsString(self._berrors)
|
2010-10-26 01:26:06 +09:00
|
|
|
|
2009-07-01 20:55:24 +09:00
|
|
|
def __dealloc__(self):
|
2009-07-01 00:57:46 +09:00
|
|
|
free(self.pk.buf);
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2011-01-29 07:27:10 +09:00
|
|
|
cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
|
2009-06-28 21:24:02 +09:00
|
|
|
cdef long long llval
|
2009-06-29 10:09:04 +09:00
|
|
|
cdef unsigned long long ullval
|
2009-06-28 21:24:02 +09:00
|
|
|
cdef long longval
|
2012-08-20 21:56:55 +02:00
|
|
|
cdef float fval
|
|
|
|
cdef double dval
|
2011-04-15 17:36:17 +03:00
|
|
|
cdef char* rawval
|
2009-07-01 20:55:24 +09:00
|
|
|
cdef int ret
|
2010-09-02 02:16:28 +09:00
|
|
|
cdef dict d
|
2013-10-17 08:35:08 +09:00
|
|
|
cdef size_t L
|
2009-05-22 14:31:20 +09:00
|
|
|
|
2010-10-26 01:49:00 +09:00
|
|
|
if nest_limit < 0:
|
2012-12-11 22:15:21 +09:00
|
|
|
raise PackValueError("recursion limit exceeded.")
|
2010-10-26 01:49:00 +09:00
|
|
|
|
2009-05-22 14:31:20 +09:00
|
|
|
if o is None:
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_nil(&self.pk)
|
2010-09-02 01:29:57 +09:00
|
|
|
elif isinstance(o, bool):
|
|
|
|
if o:
|
|
|
|
ret = msgpack_pack_true(&self.pk)
|
|
|
|
else:
|
|
|
|
ret = msgpack_pack_false(&self.pk)
|
2009-06-30 23:03:33 +09:00
|
|
|
elif PyLong_Check(o):
|
2009-06-29 10:09:04 +09:00
|
|
|
if o > 0:
|
2010-11-03 03:11:00 +09:00
|
|
|
ullval = o
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
|
2009-06-29 10:09:04 +09:00
|
|
|
else:
|
2010-11-03 03:11:00 +09:00
|
|
|
llval = o
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_long_long(&self.pk, llval)
|
2009-06-30 23:03:33 +09:00
|
|
|
elif PyInt_Check(o):
|
2009-06-28 21:24:02 +09:00
|
|
|
longval = o
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_long(&self.pk, longval)
|
2009-06-30 23:03:33 +09:00
|
|
|
elif PyFloat_Check(o):
|
2012-08-20 21:56:55 +02:00
|
|
|
if self.use_float:
|
|
|
|
fval = o
|
|
|
|
ret = msgpack_pack_float(&self.pk, fval)
|
|
|
|
else:
|
|
|
|
dval = o
|
|
|
|
ret = msgpack_pack_double(&self.pk, dval)
|
2010-09-02 01:29:57 +09:00
|
|
|
elif PyBytes_Check(o):
|
2009-05-22 14:31:20 +09:00
|
|
|
rawval = o
|
2013-10-17 08:35:08 +09:00
|
|
|
L = len(o)
|
|
|
|
ret = msgpack_pack_bin(&self.pk, L)
|
2009-07-01 20:55:24 +09:00
|
|
|
if ret == 0:
|
2013-10-17 08:35:08 +09:00
|
|
|
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
|
2009-06-30 23:03:33 +09:00
|
|
|
elif PyUnicode_Check(o):
|
2011-04-15 17:36:17 +03:00
|
|
|
if not self.encoding:
|
2012-02-10 15:08:49 -05:00
|
|
|
raise TypeError("Can't encode unicode string: no encoding is specified")
|
2011-04-15 17:36:17 +03:00
|
|
|
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
|
2009-06-08 00:23:38 +09:00
|
|
|
rawval = o
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_raw(&self.pk, len(o))
|
|
|
|
if ret == 0:
|
|
|
|
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
|
2012-09-23 02:13:32 +09:00
|
|
|
elif PyDict_CheckExact(o):
|
2012-06-16 13:56:46 +03:00
|
|
|
d = <dict>o
|
2010-09-02 02:16:28 +09:00
|
|
|
ret = msgpack_pack_map(&self.pk, len(d))
|
2009-07-01 20:55:24 +09:00
|
|
|
if ret == 0:
|
2012-09-23 02:13:32 +09:00
|
|
|
for k, v in d.iteritems():
|
|
|
|
ret = self._pack(k, nest_limit-1)
|
|
|
|
if ret != 0: break
|
|
|
|
ret = self._pack(v, nest_limit-1)
|
|
|
|
if ret != 0: break
|
|
|
|
elif PyDict_Check(o):
|
|
|
|
ret = msgpack_pack_map(&self.pk, len(o))
|
|
|
|
if ret == 0:
|
|
|
|
for k, v in o.items():
|
2011-01-29 07:27:10 +09:00
|
|
|
ret = self._pack(k, nest_limit-1)
|
2009-07-01 20:55:24 +09:00
|
|
|
if ret != 0: break
|
2011-01-29 07:27:10 +09:00
|
|
|
ret = self._pack(v, nest_limit-1)
|
2009-07-01 20:55:24 +09:00
|
|
|
if ret != 0: break
|
2012-02-16 17:15:04 +01:00
|
|
|
elif PyTuple_Check(o) or PyList_Check(o):
|
2009-07-01 20:55:24 +09:00
|
|
|
ret = msgpack_pack_array(&self.pk, len(o))
|
|
|
|
if ret == 0:
|
|
|
|
for v in o:
|
2011-01-29 07:27:10 +09:00
|
|
|
ret = self._pack(v, nest_limit-1)
|
2009-07-01 20:55:24 +09:00
|
|
|
if ret != 0: break
|
2011-01-29 23:22:41 +09:00
|
|
|
elif self._default:
|
2011-01-29 07:27:10 +09:00
|
|
|
o = self._default(o)
|
2011-01-29 23:22:41 +09:00
|
|
|
ret = self._pack(o, nest_limit-1)
|
2009-05-22 14:31:20 +09:00
|
|
|
else:
|
2010-10-26 01:26:06 +09:00
|
|
|
raise TypeError("can't serialize %r" % (o,))
|
2009-07-01 20:55:24 +09:00
|
|
|
return ret
|
|
|
|
|
2012-03-08 16:59:08 +09:00
|
|
|
cpdef pack(self, object obj):
|
2009-07-01 20:55:24 +09:00
|
|
|
cdef int ret
|
2011-01-29 07:27:10 +09:00
|
|
|
ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
|
2012-12-10 01:42:38 +09:00
|
|
|
if ret == -1:
|
|
|
|
raise MemoryError
|
|
|
|
elif ret: # should not happen.
|
2009-07-01 20:55:24 +09:00
|
|
|
raise TypeError
|
2012-12-10 21:47:18 +09:00
|
|
|
if self.autoreset:
|
|
|
|
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
|
|
|
self.pk.length = 0
|
|
|
|
return buf
|
2009-07-01 00:57:46 +09:00
|
|
|
|
2012-12-10 21:26:41 +09:00
|
|
|
def pack_array_header(self, size_t size):
|
|
|
|
cdef int ret = msgpack_pack_array(&self.pk, size)
|
|
|
|
if ret == -1:
|
|
|
|
raise MemoryError
|
|
|
|
elif ret: # should not happen
|
|
|
|
raise TypeError
|
2012-12-10 21:47:18 +09:00
|
|
|
if self.autoreset:
|
|
|
|
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
|
|
|
self.pk.length = 0
|
|
|
|
return buf
|
2012-12-10 21:26:41 +09:00
|
|
|
|
|
|
|
def pack_map_header(self, size_t size):
|
|
|
|
cdef int ret = msgpack_pack_map(&self.pk, size)
|
|
|
|
if ret == -1:
|
|
|
|
raise MemoryError
|
|
|
|
elif ret: # should not happen
|
|
|
|
raise TypeError
|
2012-12-10 21:47:18 +09:00
|
|
|
if self.autoreset:
|
|
|
|
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
|
|
|
self.pk.length = 0
|
|
|
|
return buf
|
2012-09-23 17:26:16 +10:00
|
|
|
|
2012-12-10 21:26:41 +09:00
|
|
|
def pack_map_pairs(self, object pairs):
|
|
|
|
"""
|
|
|
|
Pack *pairs* as msgpack map type.
|
|
|
|
|
|
|
|
*pairs* should sequence of pair.
|
2013-02-26 09:20:44 +09:00
|
|
|
(`len(pairs)` and `for k, v in pairs:` should be supported.)
|
2012-12-10 21:26:41 +09:00
|
|
|
"""
|
|
|
|
cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
|
|
|
|
if ret == 0:
|
|
|
|
for k, v in pairs:
|
|
|
|
ret = self._pack(k)
|
|
|
|
if ret != 0: break
|
|
|
|
ret = self._pack(v)
|
|
|
|
if ret != 0: break
|
|
|
|
if ret == -1:
|
|
|
|
raise MemoryError
|
|
|
|
elif ret: # should not happen
|
|
|
|
raise TypeError
|
2012-12-10 21:47:18 +09:00
|
|
|
if self.autoreset:
|
|
|
|
buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
|
|
|
self.pk.length = 0
|
|
|
|
return buf
|
|
|
|
|
|
|
|
def reset(self):
|
|
|
|
"""Clear internal buffer."""
|
2012-09-23 17:26:16 +10:00
|
|
|
self.pk.length = 0
|
2012-12-10 21:47:18 +09:00
|
|
|
|
|
|
|
def bytes(self):
|
|
|
|
"""Return buffer content."""
|
|
|
|
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
2009-06-26 14:10:20 +09:00
|
|
|
|
2012-12-10 21:26:41 +09:00
|
|
|
|
2013-10-17 08:35:08 +09:00
|
|
|
def pack(object o, object stream,
|
|
|
|
default=None, str encoding='utf-8', str unicode_errors='strict',
|
|
|
|
bint use_single_float=False, bint use_bin_type=False):
|
2013-02-26 09:20:44 +09:00
|
|
|
"""
|
|
|
|
pack an object `o` and write it to stream)
|
|
|
|
|
|
|
|
See :class:`Packer` for options.
|
2011-08-26 04:45:05 +09:00
|
|
|
"""
|
2013-10-17 08:35:08 +09:00
|
|
|
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
|
|
|
|
use_single_float=use_single_float, use_bin_type=use_bin_type)
|
2009-07-01 20:55:24 +09:00
|
|
|
stream.write(packer.pack(o))
|
2009-06-08 01:30:43 +09:00
|
|
|
|
2013-10-17 08:35:08 +09:00
|
|
|
def packb(object o,
|
|
|
|
default=None, str encoding='utf-8', str unicode_errors='strict',
|
|
|
|
bint use_single_float=False, bint use_bin_type=False):
|
2013-02-26 09:20:44 +09:00
|
|
|
"""
|
|
|
|
pack o and return packed bytes
|
|
|
|
|
|
|
|
See :class:`Packer` for options.
|
2011-08-26 04:45:05 +09:00
|
|
|
"""
|
2012-09-21 14:15:30 +09:00
|
|
|
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
|
2013-10-17 08:35:08 +09:00
|
|
|
use_single_float=use_single_float, use_bin_type=use_bin_type)
|
2009-07-01 20:55:24 +09:00
|
|
|
return packer.pack(o)
|