Stowage/msgpack-python
2
0
Fork 0
mirror of https://github.com/msgpack/msgpack-python.git synced 2025-10-31 17:40:54 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactoring

Browse source
This commit is contained in:
Naoki INADA 2009-06-08 01:30:43 +09:00
parent 342d3ca710
commit 5425b72247
3 changed files with 68 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

58
msgpack.pyx
View file

@ -1,5 +1,6 @@
# coding: utf-8 # coding: utf-8
from cStringIO import StringIO
cdef extern from "Python.h": cdef extern from "Python.h":
ctypedef char* const_char_ptr "const char*" ctypedef char* const_char_ptr "const char*"
@ -32,13 +33,15 @@ cdef extern from "pack.h":
void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw(msgpack_packer* pk, size_t l)
void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
cdef extern from "unpack.h":
ctypedef struct msgpack_unpacker
cdef int BUFF_SIZE=2*1024 cdef int BUFF_SIZE=2*1024
cdef class Packer: cdef class Packer:
"""Packer that pack data into strm.
strm must have `write(bytes)` method.
size specifies local buffer size.
"""
cdef char* buff cdef char* buff
cdef unsigned int length cdef unsigned int length
cdef unsigned int allocated cdef unsigned int allocated
@ -46,11 +49,6 @@ cdef class Packer:
cdef object strm cdef object strm
def __init__(self, strm, int size=0): def __init__(self, strm, int size=0):
"""Make packer that pack data into strm.
strm must have `write(bytes)` method.
size specifies local buffer size.
"""
if size <= 0: if size <= 0:
size = BUFF_SIZE size = BUFF_SIZE
@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l):
packer.length += l packer.length += l
return 0 return 0
cdef extern from "msgpack/zone.h": def pack(object o, object stream):
ctypedef struct msgpack_zone packer = Packer(stream)
packer.pack(o)
packer.flush()
cdef extern from "unpack.c": def packs(object o):
buf = StringIO()
packer = Packer(buf)
packer.pack(o)
packer.flush()
return buf.getvalue()
cdef extern from "unpack.h":
ctypedef struct template_context: ctypedef struct template_context:
pass pass
int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) int template_execute(template_context* ctx, const_char_ptr data,
size_t len, size_t* off)
void template_init(template_context* ctx) void template_init(template_context* ctx)
PyObject* template_data(template_context* ctx) PyObject* template_data(template_context* ctx)
cdef class Unpacker: def unpacks(object packed_bytes):
def __init__(self): """Unpack packed_bytes to object. Returns unpacked object."""
pass cdef const_char_ptr p = packed_bytes
cdef template_context ctx
cdef size_t off = 0
template_init(&ctx)
template_execute(&ctx, p, len(packed_bytes), &off)
return <object> template_data(&ctx)
def unpack(self, bytes_): def unpack(object stream):
cdef const_char_ptr p = bytes_ """unpack from stream."""
cdef template_context ctx packed = stream.read()
cdef size_t off = 0 return unpacks(packed)
template_init(&ctx)
template_execute(&ctx, p, len(bytes_), &off) cdef class Unpacker:
return <object> template_data(&ctx) """Do nothing. This function is for symmetric to Packer"""
unpack = staticmethod(unpacks)

67
pack.h
View file

@ -15,9 +15,6 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#ifndef MSGPACK_PACK_H__
#define MSGPACK_PACK_H__
#if _MSC_VER #if _MSC_VER
typedef signed char uint8_t; typedef signed char uint8_t;
typedef unsigned char uint8_t; typedef unsigned char uint8_t;
@ -27,14 +24,13 @@ typedef int int32_t;
typedef unsigned int uint32_t; typedef unsigned int uint32_t;
typedef long long int64_t; typedef long long int64_t;
typedef unsigned long long uint64_t; typedef unsigned long long uint64_t;
#elif #else
#include <stdint.h> #include <stdint.h>
#endif #endif
#include <stddef.h> #include <stddef.h>
#include <stdlib.h> #include <stdlib.h>
#include "msgpack/pack_define.h" #include "msgpack/pack_define.h"
#include "msgpack/object.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -48,44 +44,42 @@ typedef struct msgpack_packer {
msgpack_packer_write callback; msgpack_packer_write callback;
} msgpack_packer; } msgpack_packer;
static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback);
static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback);
static void msgpack_packer_free(msgpack_packer* pk); static inline void msgpack_packer_free(msgpack_packer* pk);
static int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_short(msgpack_packer* pk, short d);
static int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_int(msgpack_packer* pk, int d);
static int msgpack_pack_long(msgpack_packer* pk, long d); static inline int msgpack_pack_long(msgpack_packer* pk, long d);
static int msgpack_pack_long_long(msgpack_packer* pk, long long d); static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
static int msgpack_pack_float(msgpack_packer* pk, float d); static inline int msgpack_pack_float(msgpack_packer* pk, float d);
static int msgpack_pack_double(msgpack_packer* pk, double d); static inline int msgpack_pack_double(msgpack_packer* pk, double d);
static int msgpack_pack_nil(msgpack_packer* pk); static inline int msgpack_pack_nil(msgpack_packer* pk);
static int msgpack_pack_true(msgpack_packer* pk); static inline int msgpack_pack_true(msgpack_packer* pk);
static int msgpack_pack_false(msgpack_packer* pk); static inline int msgpack_pack_false(msgpack_packer* pk);
static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
static int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /* msgpack/pack.h */

273
unpack.h
View file

@ -15,12 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#include "msgpack/unpack.h"
#include "msgpack/unpack_define.h" #include "msgpack/unpack_define.h"
#include <stdlib.h>
#include "Python.h"
typedef struct { typedef struct {
int reserved; int reserved;
@ -44,11 +39,11 @@ typedef struct {
struct template_context; struct template_context;
typedef struct template_context template_context; typedef struct template_context template_context;
static void template_init(template_context* ctx); static inline void template_init(template_context* ctx);
static msgpack_unpack_object template_data(template_context* ctx); static inline msgpack_unpack_object template_data(template_context* ctx);
static int template_execute(template_context* ctx, static inline int template_execute(template_context* ctx,
const char* data, size_t len, size_t* off); const char* data, size_t len, size_t* off);
@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
} }
#include "msgpack/unpack_template.h" #include "msgpack/unpack_template.h"
#if 0
#define CTX_CAST(m) ((template_context*)(m))
#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced
static const size_t COUNTER_SIZE = sizeof(unsigned int);
static inline void init_count(void* buffer)
{
*(volatile unsigned int*)buffer = 1;
}
static inline void decl_count(void* buffer)
{
//if(--*(unsigned int*)buffer == 0) {
if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) {
free(buffer);
}
}
static inline void incr_count(void* buffer)
{
//++*(unsigned int*)buffer;
__sync_add_and_fetch((unsigned int*)buffer, 1);
}
static inline unsigned int get_count(void* buffer)
{
return *(volatile unsigned int*)buffer;
}
bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size)
{
if(initial_buffer_size < COUNTER_SIZE) {
initial_buffer_size = COUNTER_SIZE;
}
char* buffer = (char*)malloc(initial_buffer_size);
if(buffer == NULL) {
return false;
}
void* ctx = malloc(sizeof(template_context));
if(ctx == NULL) {
free(buffer);
return false;
}
msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
if(z == NULL) {
free(ctx);
free(buffer);
return false;
}
mpac->buffer = buffer;
mpac->used = COUNTER_SIZE;
mpac->free = initial_buffer_size - mpac->used;
mpac->off = COUNTER_SIZE;
mpac->parsed = 0;
mpac->initial_buffer_size = initial_buffer_size;
mpac->z = z;
mpac->ctx = ctx;
init_count(mpac->buffer);
template_init(CTX_CAST(mpac->ctx));
CTX_CAST(mpac->ctx)->user.z = mpac->z;
CTX_CAST(mpac->ctx)->user.referenced = false;
return true;
}
void msgpack_unpacker_destroy(msgpack_unpacker* mpac)
{
msgpack_zone_free(mpac->z);
free(mpac->ctx);
decl_count(mpac->buffer);
}
msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size)
{
msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker));
if(mpac == NULL) {
return NULL;
}
if(!msgpack_unpacker_init(mpac, initial_buffer_size)) {
free(mpac);
return NULL;
}
return mpac;
}
void msgpack_unpacker_free(msgpack_unpacker* mpac)
{
msgpack_unpacker_destroy(mpac);
free(mpac);
}
bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size)
{
if(mpac->used == mpac->off && get_count(mpac->buffer) == 1
&& !CTX_REFERENCED(mpac)) {
// rewind buffer
mpac->free += mpac->used - COUNTER_SIZE;
mpac->used = COUNTER_SIZE;
mpac->off = COUNTER_SIZE;
if(mpac->free >= size) {
return true;
}
}
if(mpac->off == COUNTER_SIZE) {
size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE
while(next_size < size + mpac->used) {
next_size *= 2;
}
char* tmp = (char*)realloc(mpac->buffer, next_size);
if(tmp == NULL) {
return false;
}
mpac->buffer = tmp;
mpac->free = next_size - mpac->used;
} else {
size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE
size_t not_parsed = mpac->used - mpac->off;
while(next_size < size + not_parsed + COUNTER_SIZE) {
next_size *= 2;
}
char* tmp = (char*)malloc(next_size);
if(tmp == NULL) {
return false;
}
init_count(tmp);
memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed);
if(CTX_REFERENCED(mpac)) {
if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
free(tmp);
return false;
}
CTX_REFERENCED(mpac) = false;
} else {
decl_count(mpac->buffer);
}
mpac->buffer = tmp;
mpac->used = not_parsed + COUNTER_SIZE;
mpac->free = next_size - mpac->used;
mpac->off = COUNTER_SIZE;
}
return true;
}
int msgpack_unpacker_execute(msgpack_unpacker* mpac)
{
size_t off = mpac->off;
int ret = template_execute(CTX_CAST(mpac->ctx),
mpac->buffer, mpac->used, &mpac->off);
if(mpac->off > off) {
mpac->parsed += mpac->off - off;
}
return ret;
}
msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac)
{
return template_data(CTX_CAST(mpac->ctx));
}
msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac)
{
if(!msgpack_unpacker_flush_zone(mpac)) {
return false;
}
msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
if(r == NULL) {
return NULL;
}
msgpack_zone* old = mpac->z;
mpac->z = r;
return old;
}
void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac)
{
msgpack_zone_clear(mpac->z);
}
bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac)
{
if(CTX_REFERENCED(mpac)) {
if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) {
return false;
}
CTX_REFERENCED(mpac) = false;
incr_count(mpac->buffer);
}
return true;
}
void msgpack_unpacker_reset(msgpack_unpacker* mpac)
{
template_init(CTX_CAST(mpac->ctx));
// don't reset referenced flag
mpac->parsed = 0;
}
msgpack_unpack_return
msgpack_unpack(const char* data, size_t len, size_t* off,
msgpack_zone* z, msgpack_unpack_object* result)
{
template_context ctx;
template_init(&ctx);
ctx.user.z = z;
ctx.user.referenced = false;
size_t noff = 0;
if(off != NULL) { noff = *off; }
int ret = template_execute(&ctx, data, len, &noff);
if(ret < 0) {
return MSGPACK_UNPACK_PARSE_ERROR;
}
if(off != NULL) { *off = noff; }
if(ret == 0) {
return MSGPACK_UNPACK_CONTINUE;
}
*result = template_data(&ctx);
if(noff < len) {
return MSGPACK_UNPACK_EXTRA_BYTES;
}
return MSGPACK_UNPACK_SUCCESS;
}
#endif