Stowage/msgpack-python
2
0
Fork 0
mirror of https://github.com/msgpack/msgpack-python.git synced 2025-10-31 17:40:54 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add bin type support.

Browse source
This commit is contained in:
INADA Naoki 2013-10-17 08:35:08 +09:00
parent f45d7b4e2d
commit da12e177a3
6 changed files with 81 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

28
msgpack/_packer.pyx
View file

@ -26,6 +26,7 @@ cdef extern from "pack.h":
int msgpack_pack_array(msgpack_packer* pk, size_t l)
int msgpack_pack_map(msgpack_packer* pk, size_t l)
int msgpack_pack_raw(msgpack_packer* pk, size_t l)
int msgpack_pack_bin(msgpack_packer* pk, size_t l)
int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
cdef int DEFAULT_RECURSE_LIMIT=511
@ -56,6 +57,9 @@ cdef class Packer(object):
:param bool autoreset:
Reset buffer after each pack and return it's content as `bytes`. (default: True).
If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
:param bool use_bin_type:
Use bin type introduced in msgpack spec 2.0 for bytes.
It also enable str8 type for unicode.
"""
cdef msgpack_packer pk
cdef object _default
@ -64,6 +68,7 @@ cdef class Packer(object):
cdef char *encoding
cdef char *unicode_errors
cdef bool use_float
cdef bool use_bin_type
cdef bint autoreset
def __cinit__(self):
@ -74,11 +79,13 @@ cdef class Packer(object):
self.pk.buf_size = buf_size
self.pk.length = 0
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
use_single_float=False, bint autoreset=1, bint use_bin_type=0):
"""
"""
self.use_float = use_single_float
self.autoreset = autoreset
self.pk.use_bin_type = use_bin_type
if default is not None:
if not PyCallable_Check(default):
raise TypeError("default must be a callable.")
@ -110,6 +117,7 @@ cdef class Packer(object):
cdef char* rawval
cdef int ret
cdef dict d
cdef size_t L
if nest_limit < 0:
raise PackValueError("recursion limit exceeded.")
@ -140,9 +148,10 @@ cdef class Packer(object):
ret = msgpack_pack_double(&self.pk, dval)
elif PyBytes_Check(o):
rawval = o
ret = msgpack_pack_raw(&self.pk, len(o))
L = len(o)
ret = msgpack_pack_bin(&self.pk, L)
if ret == 0:
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
elif PyUnicode_Check(o):
if not self.encoding:
raise TypeError("Can't encode unicode string: no encoding is specified")
@ -247,21 +256,26 @@ cdef class Packer(object):
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
def pack(object o, object stream,
default=None, str encoding='utf-8', str unicode_errors='strict',
bint use_single_float=False, bint use_bin_type=False):
"""
pack an object `o` and write it to stream)
See :class:`Packer` for options.
"""
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
use_single_float=use_single_float, use_bin_type=use_bin_type)
stream.write(packer.pack(o))
def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
def packb(object o,
default=None, str encoding='utf-8', str unicode_errors='strict',
bint use_single_float=False, bint use_bin_type=False):
"""
pack o and return packed bytes
See :class:`Packer` for options.
"""
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
use_single_float=use_single_float)
use_single_float=use_single_float, use_bin_type=use_bin_type)
return packer.pack(o)

3
msgpack/pack.h
View file

@ -34,11 +34,11 @@ typedef struct msgpack_packer {
char *buf;
size_t length;
size_t buf_size;
bool use_bin_type;
} msgpack_packer;
typedef struct Packer Packer;
static inline int msgpack_pack_short(msgpack_packer* pk, short d);
static inline int msgpack_pack_int(msgpack_packer* pk, int d);
static inline int msgpack_pack_long(msgpack_packer* pk, long d);
static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)

27
msgpack/pack_template.h
View file

@ -667,7 +667,10 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
if(l < 32) {
unsigned char d = 0xa0 | (uint8_t)l;
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
} else if(l < 65536) {
} else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin.
unsigned char buf[2] = {0xd9, (uint8_t)l};
msgpack_pack_append_buffer(x, buf, 2);
} else if (l < 65536) {
unsigned char buf[3];
buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
msgpack_pack_append_buffer(x, buf, 3);
@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
}
}
/*
* bin
*/
static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
{
if (!x->use_bin_type) {
return msgpack_pack_raw(x, l)
}
if (l < 256) {
unsigned char buf[2] = {0xc4, (unsigned char)l};
msgpack_pack_append_buffer(x, buf, 2);
} else if (l < 65536) {
unsigned char buf[3] = {0xc5};
_msgpack_store16(&buf[1], (uint16_t)l);
msgpack_pack_append_buffer(x, buf, 3);
} else {
unsigned char buf[5] = {0xc6};
_msgpack_store32(&buf[1], (uint32_t)l);
msgpack_pack_append_buffer(x, buf, 5);
}
}
static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
{
msgpack_pack_append_buffer(x, (const unsigned char*)b, l);

9
msgpack/unpack.h
View file

@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
return 0;
}
static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
{
PyObject *py = PyBytes_FromStringAndSize(p, l);
if (!py)
return -1;
*o = py;
return 0;
}
#include "unpack_template.h"

17
msgpack/unpack_define.h
View file

@ -34,6 +34,7 @@ extern "C" {
#endif
// CS is first byte & 0x1f
typedef enum {
CS_HEADER = 0x00, // nil
@ -41,9 +42,9 @@ typedef enum {
//CS_ = 0x02, // false
//CS_ = 0x03, // true
//CS_ = 0x04,
//CS_ = 0x05,
//CS_ = 0x06,
CS_BIN_8 = 0x04,
CS_BIN_16 = 0x05,
CS_BIN_32 = 0x06,
//CS_ = 0x07,
//CS_ = 0x08,
@ -59,12 +60,7 @@ typedef enum {
CS_INT_32 = 0x12,
CS_INT_64 = 0x13,
//CS_ = 0x14,
//CS_ = 0x15,
//CS_BIG_INT_16 = 0x16,
//CS_BIG_INT_32 = 0x17,
//CS_BIG_FLOAT_16 = 0x18,
//CS_BIG_FLOAT_32 = 0x19,
CS_RAW_8 = 0x19,
CS_RAW_16 = 0x1a,
CS_RAW_32 = 0x1b,
CS_ARRAY_16 = 0x1c,
@ -72,9 +68,8 @@ typedef enum {
CS_MAP_16 = 0x1e,
CS_MAP_32 = 0x1f,
//ACS_BIG_INT_VALUE,
//ACS_BIG_FLOAT_VALUE,
ACS_RAW_VALUE,
ACS_BIN_VALUE,
} msgpack_unpack_state;

23
msgpack/unpack_template.h
View file

@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
*/ \
goto _header_again
#define NEXT_CS(p) \
((unsigned int)*p & 0x1f)
#define NEXT_CS(p) ((unsigned int)*p & 0x1f)
#ifdef USE_CASE_RANGE
#define SWITCH_RANGE_BEGIN switch(*p) {
@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
push_simple_value(_false);
case 0xc3: // true
push_simple_value(_true);
//case 0xc4:
//case 0xc5:
//case 0xc6:
//case 0xc7:
//case 0xc8:
//case 0xc9:
@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
case 0xd2: // signed int 32
case 0xd3: // signed int 64
again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
case 0xc4: // bin 8
case 0xc5: // bin 16
case 0xc6: // bin 32
//case 0xd4:
//case 0xd5:
//case 0xd6: // big integer 16
//case 0xd7: // big integer 32
//case 0xd8: // big float 16
//case 0xd9: // big float 32
case 0xd9: // raw 8
case 0xda: // raw 16
case 0xdb: // raw 32
case 0xdc: // array 16
@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
// // FIXME
// push_variable_value(_big_float, data, n, trail);
case CS_BIN_8:
again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
case CS_BIN_16:
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
case CS_BIN_32:
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
case ACS_BIN_VALUE:
_bin_zero:
push_variable_value(_bin, data, n, trail);
case CS_RAW_8:
again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
case CS_RAW_16:
again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
case CS_RAW_32: