mirror of
https://github.com/msgpack/msgpack-python.git
synced 2025-10-31 17:40:54 +00:00
refactoring
This commit is contained in:
Naoki INADA
parent
342d3ca710
commit
5425b72247
3 changed files with 68 additions and 330 deletions
52
msgpack.pyx
52
msgpack.pyx
|
|
@ -1,5 +1,6 @@
|
|||
# coding: utf-8
|
||||
|
||||
from cStringIO import StringIO
|
||||
|
||||
cdef extern from "Python.h":
|
||||
ctypedef char* const_char_ptr "const char*"
|
||||
|
|
@ -32,13 +33,15 @@ cdef extern from "pack.h":
|
|||
void msgpack_pack_raw(msgpack_packer* pk, size_t l)
|
||||
void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
|
||||
|
||||
cdef extern from "unpack.h":
|
||||
ctypedef struct msgpack_unpacker
|
||||
|
||||
|
||||
cdef int BUFF_SIZE=2*1024
|
||||
|
||||
cdef class Packer:
|
||||
"""Packer that pack data into strm.
|
||||
|
||||
strm must have `write(bytes)` method.
|
||||
size specifies local buffer size.
|
||||
"""
|
||||
cdef char* buff
|
||||
cdef unsigned int length
|
||||
cdef unsigned int allocated
|
||||
|
|
@ -46,11 +49,6 @@ cdef class Packer:
|
|||
cdef object strm
|
||||
|
||||
def __init__(self, strm, int size=0):
|
||||
"""Make packer that pack data into strm.
|
||||
|
||||
strm must have `write(bytes)` method.
|
||||
size specifies local buffer size.
|
||||
"""
|
||||
if size <= 0:
|
||||
size = BUFF_SIZE
|
||||
|
||||
|
|
@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l):
|
|||
packer.length += l
|
||||
return 0
|
||||
|
||||
cdef extern from "msgpack/zone.h":
|
||||
ctypedef struct msgpack_zone
|
||||
def pack(object o, object stream):
|
||||
packer = Packer(stream)
|
||||
packer.pack(o)
|
||||
packer.flush()
|
||||
|
||||
cdef extern from "unpack.c":
|
||||
def packs(object o):
|
||||
buf = StringIO()
|
||||
packer = Packer(buf)
|
||||
packer.pack(o)
|
||||
packer.flush()
|
||||
return buf.getvalue()
|
||||
|
||||
cdef extern from "unpack.h":
|
||||
ctypedef struct template_context:
|
||||
pass
|
||||
int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off)
|
||||
int template_execute(template_context* ctx, const_char_ptr data,
|
||||
size_t len, size_t* off)
|
||||
void template_init(template_context* ctx)
|
||||
PyObject* template_data(template_context* ctx)
|
||||
|
||||
|
||||
cdef class Unpacker:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def unpack(self, bytes_):
|
||||
cdef const_char_ptr p = bytes_
|
||||
def unpacks(object packed_bytes):
|
||||
"""Unpack packed_bytes to object. Returns unpacked object."""
|
||||
cdef const_char_ptr p = packed_bytes
|
||||
cdef template_context ctx
|
||||
cdef size_t off = 0
|
||||
template_init(&ctx)
|
||||
template_execute(&ctx, p, len(bytes_), &off)
|
||||
template_execute(&ctx, p, len(packed_bytes), &off)
|
||||
return <object> template_data(&ctx)
|
||||
|
||||
def unpack(object stream):
|
||||
"""unpack from stream."""
|
||||
packed = stream.read()
|
||||
return unpacks(packed)
|
||||
|
||||
cdef class Unpacker:
|
||||
"""Do nothing. This function is for symmetric to Packer"""
|
||||
unpack = staticmethod(unpacks)
|
||||
|
|
|
|||
67
pack.h
67
pack.h
|
|
@ -15,9 +15,6 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MSGPACK_PACK_H__
|
||||
#define MSGPACK_PACK_H__
|
||||
|
||||
#if _MSC_VER
|
||||
typedef signed char uint8_t;
|
||||
typedef unsigned char uint8_t;
|
||||
|
|
@ -27,14 +24,13 @@ typedef int int32_t;
|
|||
typedef unsigned int uint32_t;
|
||||
typedef long long int64_t;
|
||||
typedef unsigned long long uint64_t;
|
||||
#elif
|
||||
#else
|
||||
#include <stdint.h>
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include "msgpack/pack_define.h"
|
||||
#include "msgpack/object.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -48,44 +44,42 @@ typedef struct msgpack_packer {
|
|||
msgpack_packer_write callback;
|
||||
} msgpack_packer;
|
||||
|
||||
static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback);
|
||||
static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback);
|
||||
|
||||
static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback);
|
||||
static void msgpack_packer_free(msgpack_packer* pk);
|
||||
static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback);
|
||||
static inline void msgpack_packer_free(msgpack_packer* pk);
|
||||
|
||||
static int msgpack_pack_short(msgpack_packer* pk, short d);
|
||||
static int msgpack_pack_int(msgpack_packer* pk, int d);
|
||||
static int msgpack_pack_long(msgpack_packer* pk, long d);
|
||||
static int msgpack_pack_long_long(msgpack_packer* pk, long long d);
|
||||
static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
|
||||
static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
|
||||
static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
|
||||
static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
|
||||
static inline int msgpack_pack_short(msgpack_packer* pk, short d);
|
||||
static inline int msgpack_pack_int(msgpack_packer* pk, int d);
|
||||
static inline int msgpack_pack_long(msgpack_packer* pk, long d);
|
||||
static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
|
||||
static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
|
||||
static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
|
||||
static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
|
||||
static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
|
||||
|
||||
static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
|
||||
static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
|
||||
static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
|
||||
static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
|
||||
static int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
|
||||
static int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
|
||||
static int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
|
||||
static int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
|
||||
static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
|
||||
static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
|
||||
static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
|
||||
static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
|
||||
static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
|
||||
static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
|
||||
static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
|
||||
static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
|
||||
|
||||
static int msgpack_pack_float(msgpack_packer* pk, float d);
|
||||
static int msgpack_pack_double(msgpack_packer* pk, double d);
|
||||
static inline int msgpack_pack_float(msgpack_packer* pk, float d);
|
||||
static inline int msgpack_pack_double(msgpack_packer* pk, double d);
|
||||
|
||||
static int msgpack_pack_nil(msgpack_packer* pk);
|
||||
static int msgpack_pack_true(msgpack_packer* pk);
|
||||
static int msgpack_pack_false(msgpack_packer* pk);
|
||||
static inline int msgpack_pack_nil(msgpack_packer* pk);
|
||||
static inline int msgpack_pack_true(msgpack_packer* pk);
|
||||
static inline int msgpack_pack_false(msgpack_packer* pk);
|
||||
|
||||
static int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
|
||||
static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
|
||||
|
||||
static int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
|
||||
static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
|
||||
|
||||
static int msgpack_pack_raw(msgpack_packer* pk, size_t l);
|
||||
static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
|
||||
|
||||
int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
|
||||
static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
|
||||
static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
|
||||
|
||||
|
||||
|
||||
|
|
@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk)
|
|||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* msgpack/pack.h */
|
||||
|
||||
|
|
|
|||
273
unpack.h
273
unpack.h
|
|
@ -15,12 +15,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "msgpack/unpack.h"
|
||||
#include "msgpack/unpack_define.h"
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "Python.h"
|
||||
|
||||
|
||||
typedef struct {
|
||||
int reserved;
|
||||
|
|
@ -44,11 +39,11 @@ typedef struct {
|
|||
struct template_context;
|
||||
typedef struct template_context template_context;
|
||||
|
||||
static void template_init(template_context* ctx);
|
||||
static inline void template_init(template_context* ctx);
|
||||
|
||||
static msgpack_unpack_object template_data(template_context* ctx);
|
||||
static inline msgpack_unpack_object template_data(template_context* ctx);
|
||||
|
||||
static int template_execute(template_context* ctx,
|
||||
static inline int template_execute(template_context* ctx,
|
||||
const char* data, size_t len, size_t* off);
|
||||
|
||||
|
||||
|
|
@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
|
|||
}
|
||||
|
||||
#include "msgpack/unpack_template.h"
|
||||
|
||||
|
||||
#if 0
|
||||
#define CTX_CAST(m) ((template_context*)(m))
|
||||
#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced
|
||||
|
||||
|
||||
static const size_t COUNTER_SIZE = sizeof(unsigned int);
|
||||
|
||||
static inline void init_count(void* buffer)
|
||||
{
|
||||
*(volatile unsigned int*)buffer = 1;
|
||||
}
|
||||
|
||||
static inline void decl_count(void* buffer)
|
||||
{
|
||||
//if(--*(unsigned int*)buffer == 0) {
|
||||
if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) {
|
||||
free(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void incr_count(void* buffer)
|
||||
{
|
||||
//++*(unsigned int*)buffer;
|
||||
__sync_add_and_fetch((unsigned int*)buffer, 1);
|
||||
}
|
||||
|
||||
static inline unsigned int get_count(void* buffer)
|
||||
{
|
||||
return *(volatile unsigned int*)buffer;
|
||||
}
|
||||
|
||||
|
||||
|
||||
bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size)
|
||||
{
|
||||
if(initial_buffer_size < COUNTER_SIZE) {
|
||||
initial_buffer_size = COUNTER_SIZE;
|
||||
}
|
||||
|
||||
char* buffer = (char*)malloc(initial_buffer_size);
|
||||
if(buffer == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void* ctx = malloc(sizeof(template_context));
|
||||
if(ctx == NULL) {
|
||||
free(buffer);
|
||||
return false;
|
||||
}
|
||||
|
||||
msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
|
||||
if(z == NULL) {
|
||||
free(ctx);
|
||||
free(buffer);
|
||||
return false;
|
||||
}
|
||||
|
||||
mpac->buffer = buffer;
|
||||
mpac->used = COUNTER_SIZE;
|
||||
mpac->free = initial_buffer_size - mpac->used;
|
||||
mpac->off = COUNTER_SIZE;
|
||||
mpac->parsed = 0;
|
||||
mpac->initial_buffer_size = initial_buffer_size;
|
||||
mpac->z = z;
|
||||
mpac->ctx = ctx;
|
||||
|
||||
init_count(mpac->buffer);
|
||||
|
||||
template_init(CTX_CAST(mpac->ctx));
|
||||
CTX_CAST(mpac->ctx)->user.z = mpac->z;
|
||||
CTX_CAST(mpac->ctx)->user.referenced = false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void msgpack_unpacker_destroy(msgpack_unpacker* mpac)
|
||||
{
|
||||
msgpack_zone_free(mpac->z);
|
||||
free(mpac->ctx);
|
||||
decl_count(mpac->buffer);
|
||||
}
|
||||
|
||||
|
||||
msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size)
|
||||
{
|
||||
msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker));
|
||||
if(mpac == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(!msgpack_unpacker_init(mpac, initial_buffer_size)) {
|
||||
free(mpac);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mpac;
|
||||
}
|
||||
|
||||
void msgpack_unpacker_free(msgpack_unpacker* mpac)
|
||||
{
|
||||
msgpack_unpacker_destroy(mpac);
|
||||
free(mpac);
|
||||
}
|
||||
|
||||
bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size)
|
||||
{
|
||||
if(mpac->used == mpac->off && get_count(mpac->buffer) == 1
|
||||
&& !CTX_REFERENCED(mpac)) {
|
||||
// rewind buffer
|
||||
mpac->free += mpac->used - COUNTER_SIZE;
|
||||
mpac->used = COUNTER_SIZE;
|
||||
mpac->off = COUNTER_SIZE;
|
||||
|
||||
if(mpac->free >= size) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if(mpac->off == COUNTER_SIZE) {
|
||||
size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE
|
||||
while(next_size < size + mpac->used) {
|
||||
next_size *= 2;
|
||||
}
|
||||
|
||||
char* tmp = (char*)realloc(mpac->buffer, next_size);
|
||||
if(tmp == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
mpac->buffer = tmp;
|
||||
mpac->free = next_size - mpac->used;
|
||||
|
||||
} else {
|
||||
size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE
|
||||
size_t not_parsed = mpac->used - mpac->off;
|
||||
while(next_size < size + not_parsed + COUNTER_SIZE) {
|
||||
next_size *= 2;
|
||||
}
|
||||
|
||||
char* tmp = (char*)malloc(next_size);
|
||||
if(tmp == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
init_count(tmp);
|
||||
|
||||
memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed);
|
||||
|
||||
if(CTX_REFERENCED(mpac)) {
|
||||
if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
|
||||
free(tmp);
|
||||
return false;
|
||||
}
|
||||
CTX_REFERENCED(mpac) = false;
|
||||
} else {
|
||||
decl_count(mpac->buffer);
|
||||
}
|
||||
|
||||
mpac->buffer = tmp;
|
||||
mpac->used = not_parsed + COUNTER_SIZE;
|
||||
mpac->free = next_size - mpac->used;
|
||||
mpac->off = COUNTER_SIZE;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int msgpack_unpacker_execute(msgpack_unpacker* mpac)
|
||||
{
|
||||
size_t off = mpac->off;
|
||||
int ret = template_execute(CTX_CAST(mpac->ctx),
|
||||
mpac->buffer, mpac->used, &mpac->off);
|
||||
if(mpac->off > off) {
|
||||
mpac->parsed += mpac->off - off;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac)
|
||||
{
|
||||
return template_data(CTX_CAST(mpac->ctx));
|
||||
}
|
||||
|
||||
msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac)
|
||||
{
|
||||
if(!msgpack_unpacker_flush_zone(mpac)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
|
||||
if(r == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
msgpack_zone* old = mpac->z;
|
||||
mpac->z = r;
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac)
|
||||
{
|
||||
msgpack_zone_clear(mpac->z);
|
||||
}
|
||||
|
||||
bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac)
|
||||
{
|
||||
if(CTX_REFERENCED(mpac)) {
|
||||
if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
|
||||
return false;
|
||||
}
|
||||
CTX_REFERENCED(mpac) = false;
|
||||
|
||||
incr_count(mpac->buffer);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void msgpack_unpacker_reset(msgpack_unpacker* mpac)
|
||||
{
|
||||
template_init(CTX_CAST(mpac->ctx));
|
||||
// don't reset referenced flag
|
||||
mpac->parsed = 0;
|
||||
}
|
||||
|
||||
|
||||
msgpack_unpack_return
|
||||
msgpack_unpack(const char* data, size_t len, size_t* off,
|
||||
msgpack_zone* z, msgpack_unpack_object* result)
|
||||
{
|
||||
template_context ctx;
|
||||
template_init(&ctx);
|
||||
|
||||
ctx.user.z = z;
|
||||
ctx.user.referenced = false;
|
||||
|
||||
size_t noff = 0;
|
||||
if(off != NULL) { noff = *off; }
|
||||
|
||||
int ret = template_execute(&ctx, data, len, &noff);
|
||||
if(ret < 0) {
|
||||
return MSGPACK_UNPACK_PARSE_ERROR;
|
||||
}
|
||||
|
||||
if(off != NULL) { *off = noff; }
|
||||
|
||||
if(ret == 0) {
|
||||
return MSGPACK_UNPACK_CONTINUE;
|
||||
}
|
||||
|
||||
*result = template_data(&ctx);
|
||||
|
||||
if(noff < len) {
|
||||
return MSGPACK_UNPACK_EXTRA_BYTES;
|
||||
}
|
||||
|
||||
return MSGPACK_UNPACK_SUCCESS;
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue